Introduce option to checkpoint database on Flush() (#1104)

The way SQLite is initialized (synchronous=NORMAL config) introduces potential issues with durability of changes/updates in some corner cases.

iOS Outlook logs telemetry in scenarios where some critical validation/check failed and cannot be handled and process self-terminates. In between logging the event and terminating we call Flush() to avoid lost events. If we checkpoint the DB before returning from the Flush() call app can safely assume no events will be lost. This change introduces an option that enables such behavior.

Alternative solution would be to use the default configuration (with synchronous=FULL) which doesn't have that problem but has overhead of filesystem syncing for each transaction which makes it a bad idea from the perf point of view.

Author: dawidk-msft

lib/include/public/ILogConfiguration.hpp

@@ -144,6 +144,11 @@ namespace MAT_NS_BEGIN
     /// </summary>
     static constexpr const char* const CFG_INT_RAM_QUEUE_BUFFERS = "maxDBFlushQueues";
 
+    /// <summary>
+    /// SQLite DB will be checkpointed when flushing.
+    /// </summary>
+    static constexpr const char* const CFG_BOOL_CHECKPOINT_DB_ON_FLUSH = "checkpointDBOnFlush";
+
     /// <summary>
     /// The trace level mask.
     /// </summary>

lib/offline/ISqlite3Proxy.hpp

@@ -55,6 +55,7 @@ class ISqlite3Proxy {
     virtual const void*          sqlite3_value_blob(sqlite3_value* value) = 0;
     virtual int                  sqlite3_value_bytes(sqlite3_value* value) = 0;
     virtual sqlite3_vfs*         sqlite3_vfs_find(char const* zVfsName) = 0;
+    virtual void                 sqlite3_wal_checkpoint(sqlite3* db) = 0;
 };
 
 extern ISqlite3Proxy* g_sqlite3Proxy;

lib/offline/OfflineStorageHandler.cpp

@@ -199,6 +199,12 @@ namespace MAT_NS_BEGIN {
             }
         }
 
+        // Checkpoint DB
+        if (m_config.HasConfig(CFG_BOOL_CHECKPOINT_DB_ON_FLUSH) && m_config[CFG_BOOL_CHECKPOINT_DB_ON_FLUSH]) 
+        {
+            m_offlineStorageDisk->Flush();
+        }
+
         m_isStorageFullNotificationSend = false;
 
         // Flush is done, notify the waiters

lib/offline/OfflineStorage_SQLite.cpp

@@ -134,6 +134,12 @@ namespace MAT_NS_BEGIN {
         }
     }
 
+    void OfflineStorage_SQLite::Flush() 
+    {
+        if (m_db)
+            m_db->flush();
+    }
+    
     void OfflineStorage_SQLite::Execute(std::string command)
     {
         if (m_db)

lib/offline/OfflineStorage_SQLite.hpp

@@ -31,7 +31,7 @@ namespace MAT_NS_BEGIN {
         virtual ~OfflineStorage_SQLite() override;
         virtual void Initialize(IOfflineStorageObserver& observer) override;
         virtual void Shutdown() override;
-        virtual void Flush() override {};
+        virtual void Flush() override;
         virtual void Execute(std::string command);
         virtual bool StoreRecord(StorageRecord const& record) override;
         virtual size_t StoreRecords(std::vector<StorageRecord> & records) override;

lib/offline/SQLiteWrapper.hpp

@@ -184,6 +184,11 @@ namespace MAT_NS_BEGIN {
         {
             return ::sqlite3_vfs_find(zVfsName);
         }
+                
+        void sqlite3_wal_checkpoint(sqlite3* db) override 
+        {
+            ::sqlite3_wal_checkpoint_v2(db, NULL, SQLITE_CHECKPOINT_FULL, NULL, NULL);
+        }
     } g_realSqlite3Proxy;
 
     ISqlite3Proxy* g_sqlite3Proxy = &g_realSqlite3Proxy;
@@ -462,6 +467,11 @@ namespace MAT_NS_BEGIN {
             return records;
         }
 
+        void flush()
+        {
+            g_sqlite3Proxy->sqlite3_wal_checkpoint(m_db);
+        }
+
     protected:
 
         static void sqliteFunc_tokenize(sqlite3_context* ctx, int argc, sqlite3_value** argv)

wrappers/obj-c/ODWLogConfiguration.h

@@ -391,6 +391,17 @@ extern NSString * _Nonnull const ODWCFG_BOOL_SESSION_RESET_ENABLED;
 */
 +(nullable NSString *)cacheFilePath;
 
+/*!
+@brief Controls if DB will be checkpointed when flushing
+@param enableDbCheckpointOnFlush True if DB should be checkpointed when flushing.
+*/
++(void)setEnableDbCheckpointOnFlush:(bool)enableDbCheckpointOnFlush;
+
+/*!
+@brief Returns true if DB will be checkpointed when flushing.
+*/
++(bool)enableDbCheckpointOnFlush;
+
 /*!
 @brief Sets a config key to a string value for the copied config
 @param key A key.

wrappers/obj-c/ODWLogConfiguration.mm

@@ -431,6 +431,22 @@ +(nullable NSString *)cacheFilePath
     return [NSString stringWithCString:strCacheFilePath.c_str() encoding:NSUTF8StringEncoding];
 }
 
++(void)setEnableDbCheckpointOnFlush:(bool)enabled
+{
+    auto& config = LogManager::GetLogConfiguration();
+    config[CFG_BOOL_CHECKPOINT_DB_ON_FLUSH] = enabled;
+}
+
++(bool)enableDbCheckpointOnFlush
+{
+    auto& config = LogManager::GetLogConfiguration();
+    if (config.HasConfig(CFG_BOOL_CHECKPOINT_DB_ON_FLUSH)) {
+        return config[CFG_BOOL_CHECKPOINT_DB_ON_FLUSH];
+    } else {
+        return false;
+    }
+}
+
 -(void)set:(nonnull NSString *)key withValue:(nonnull NSString *)value
 {
     (*_wrappedConfiguration)[[key UTF8String]] = [value UTF8String];