From b1bed18331af6e8eb27d4b6978b8d71b95859741 Mon Sep 17 00:00:00 2001
From: zadam <adam.zivner@gmail.com>
Date: Sat, 4 Apr 2020 21:49:57 +0200
Subject: [PATCH] significantly faster sync with transactions

---
 src/routes/api/sync.js     |  2 +-
 src/services/sql_init.js   |  3 +++
 src/services/sync.js       | 30 ++++++++++++------------------
 src/services/sync_table.js | 24 +++++++++++++++---------
 4 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js
index 081e3e7cf..f72943303 100644
--- a/src/routes/api/sync.js
+++ b/src/routes/api/sync.js
@@ -126,7 +126,7 @@ async function getChanged(req) {
     };
 
     if (ret.syncs.length > 0) {
-        log.info(`Returning ${ret.syncs.length} in ${Date.now() - startTime}ms`);
+        log.info(`Returning ${ret.syncs.length} sync records in ${Date.now() - startTime}ms`);
     }
 
     return ret;
diff --git a/src/services/sql_init.js b/src/services/sql_init.js
index 572d31e83..40ea158b2 100644
--- a/src/services/sql_init.js
+++ b/src/services/sql_init.js
@@ -18,6 +18,9 @@ async function createConnection() {
 const dbConnection = new Promise(async (resolve, reject) => {
     // no need to create new connection now since DB stays the same all the time
     const db = await createConnection();
+
+    db.run('PRAGMA journal_mode = WAL;');
+
     sql.setDbConnection(db);
 
     resolve();
diff --git a/src/services/sync.js b/src/services/sync.js
index c76b179ff..1687c9cca 100644
--- a/src/services/sync.js
+++ b/src/services/sync.js
@@ -145,29 +145,25 @@ async function pullSync(syncContext) {
             break;
         }
 
-        log.info(`Pulled ${rows.length} changes from ${changesUri} in ${Date.now() - startDate}ms`);
+        await sql.transactional(async () => {
+            for (const {sync, entity} of rows) {
+                if (!sourceIdService.isLocalSourceId(sync.sourceId)) {
+                    if (appliedPulls === 0 && sync.entity !== 'recent_notes') { // send only for first
+                        ws.syncPullInProgress();
 
-        for (const {sync, entity} of rows) {
-            if (!sourceIdService.isLocalSourceId(sync.sourceId)) {
-                if (appliedPulls === 0 && sync.entity !== 'recent_notes') { // send only for first
-                    ws.syncPullInProgress();
+                        appliedPulls++;
+                    }
 
-                    appliedPulls++;
+                    await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
                 }
 
-                const startTime = Date.now();
-
-                const updated = await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
-
-                if (updated) {
-                    log.info(`Updated ${sync.entityName} ${sync.entityId} in ${Date.now() - startTime}ms`);
-                }
+                stats.outstandingPulls = resp.maxSyncId - sync.id;
             }
 
-            stats.outstandingPulls = resp.maxSyncId - sync.id;
-        }
+            await setLastSyncedPull(rows[rows.length - 1].sync.id);
+        });
 
-        await setLastSyncedPull(rows[rows.length - 1].sync.id);
+        log.info(`Pulled and updated ${rows.length} changes from ${changesUri} in ${Date.now() - startDate}ms`);
     }
 
     if (appliedPulls > 0) {
@@ -257,8 +253,6 @@ async function checkContentHash(syncContext) {
         await syncTableService.addEntitySyncsForSector(entityName, entityPrimaryKey, sector);
 
         await syncRequest(syncContext, 'POST', `/api/sync/queue-sector/${entityName}/${sector}`);
-
-        log.info(`Added sector ${sector} of ${entityName} to sync queue.`);
     }
 
     return failedChecks.length > 0;
diff --git a/src/services/sync_table.js b/src/services/sync_table.js
index dec81fd19..6b063aa21 100644
--- a/src/services/sync_table.js
+++ b/src/services/sync_table.js
@@ -33,19 +33,25 @@ async function addEntitySync(entityName, entityId, sourceId, isSynced) {
 }
 
 async function addEntitySyncsForSector(entityName, entityPrimaryKey, sector) {
-    const entityIds = await sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]);
+    const startTime = Date.now();
 
-    for (const entityId of entityIds) {
-        if (entityName === 'options') {
-            const isSynced = await sql.getValue(`SELECT isSynced FROM options WHERE name = ?`, [entityId]);
+    await sql.transactional(async () => {
+        const entityIds = await sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]);
 
-            if (!isSynced) {
-                continue;
+        for (const entityId of entityIds) {
+            if (entityName === 'options') {
+                const isSynced = await sql.getValue(`SELECT isSynced FROM options WHERE name = ?`, [entityId]);
+
+                if (!isSynced) {
+                    continue;
+                }
             }
-        }
 
-        await insertEntitySync(entityName, entityId, 'content-check', true);
-    }
+            await insertEntitySync(entityName, entityId, 'content-check', true);
+        }
+    });
+
+    log.info(`Added sector ${sector} of ${entityName} to sync queue in ${Date.now() - startTime}ms.`);
 }
 
 async function cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey) {