From 64336ffbeec30aa25b5b223cb97aedadd9e9ffb0 Mon Sep 17 00:00:00 2001
From: azivner <adam.zivner@gmail.com>
Date: Sat, 7 Apr 2018 21:53:42 -0400
Subject: [PATCH] implemented bulk sync pull for increased performance

---
 src/routes/api/sync.js | 56 ++++++++++++++++++++++++++++++++++++------
 src/services/sync.js   | 30 +++++++++++-----------
 2 files changed, 62 insertions(+), 24 deletions(-)

diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js
index 730753ec8..453e7dfa7 100644
--- a/src/routes/api/sync.js
+++ b/src/routes/api/sync.js
@@ -58,7 +58,52 @@ async function forceNoteSync(req) {
 async function getChanged(req) {
     const lastSyncId = parseInt(req.query.lastSyncId);
 
-    return await sql.getRows("SELECT * FROM sync WHERE id > ?", [lastSyncId]);
+    const records = [];
+    let length = 0;
+
+    for (const sync of await sql.getRows("SELECT * FROM sync WHERE id > ?", [lastSyncId])) {
+        const record = {
+            sync: sync,
+            entity: await getEntityRow(sync.entityName, sync.entityId)
+        };
+
+        records.push(record);
+
+        length += JSON.stringify(record).length;
+
+        if (length > 1000000) {
+            break;
+        }
+    }
+
+    return records;
+}
+
+const primaryKeys = {
+    "notes": "noteId",
+    "branches": "branchId",
+    "note_revisions": "noteRevisionId",
+    "option": "name",
+    "recent_notes": "branchId",
+    "images": "imageId",
+    "note_images": "noteImageId",
+    "labels": "labelId",
+    "api_tokens": "apiTokenId"
+};
+
+async function getEntityRow(entityName, entityId) {
+    if (entityName === 'note_reordering') {
+        return await getNoteReordering(entityId);
+    }
+    else {
+        const primaryKey = primaryKeys[entityName];
+
+        if (!primaryKey) {
+            throw new Error("Unknown entity " + entityName);
+        }
+
+        return await sql.getRow(`SELECT * FROM ${entityName} WHERE ${primaryKey} = ?`, [entityId]);
+    }
 }
 
 async function getNote(req) {
@@ -96,13 +141,8 @@ async function getOption(req) {
     }
 }
 
-async function getNoteReordering(req) {
-    const parentNoteId = req.params.parentNoteId;
-
-    return {
-        parentNoteId: parentNoteId,
-        ordering: await sql.getMap("SELECT branchId, notePosition FROM branches WHERE parentNoteId = ? AND isDeleted = 0", [parentNoteId])
-    };
+async function getNoteReordering(parentNoteId) {
+    return await sql.getMap("SELECT branchId, notePosition FROM branches WHERE parentNoteId = ? AND isDeleted = 0", [parentNoteId])
 }
 
 async function getRecentNote(req) {
diff --git a/src/services/sync.js b/src/services/sync.js
index fe1c0c445..b9bf1f4ea 100644
--- a/src/services/sync.js
+++ b/src/services/sync.js
@@ -104,11 +104,11 @@ async function pullSync(syncContext) {
 
     const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull;
 
-    const syncRows = await syncRequest(syncContext, 'GET', changesUri);
+    const rows = await syncRequest(syncContext, 'GET', changesUri);
 
-    log.info("Pulled " + syncRows.length + " changes from " + changesUri);
+    log.info("Pulled " + rows.length + " changes from " + changesUri);
 
-    for (const sync of syncRows) {
+    for (const {sync, entity} of rows) {
         if (sourceIdService.isLocalSourceId(sync.sourceId)) {
             log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
 
@@ -117,40 +117,38 @@ async function pullSync(syncContext) {
             continue;
         }
 
-        const resp = await syncRequest(syncContext, 'GET', "/api/sync/" + sync.entityName + "/" + encodeURIComponent(sync.entityId));
-
-        if (!resp || (sync.entityName === 'notes' && !resp.entity)) {
+        if (!entity) {
             log.error(`Empty response to pull for sync #${sync.id} ${sync.entityName}, id=${sync.entityId}`);
         }
         else if (sync.entityName === 'notes') {
-            await syncUpdateService.updateNote(resp.entity, syncContext.sourceId);
+            await syncUpdateService.updateNote(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'branches') {
-            await syncUpdateService.updateBranch(resp, syncContext.sourceId);
+            await syncUpdateService.updateBranch(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'note_revisions') {
-            await syncUpdateService.updateNoteRevision(resp, syncContext.sourceId);
+            await syncUpdateService.updateNoteRevision(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'note_reordering') {
-            await syncUpdateService.updateNoteReordering(resp, syncContext.sourceId);
+            await syncUpdateService.updateNoteReordering(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'options') {
-            await syncUpdateService.updateOptions(resp, syncContext.sourceId);
+            await syncUpdateService.updateOptions(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'recent_notes') {
-            await syncUpdateService.updateRecentNotes(resp, syncContext.sourceId);
+            await syncUpdateService.updateRecentNotes(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'images') {
-            await syncUpdateService.updateImage(resp, syncContext.sourceId);
+            await syncUpdateService.updateImage(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'note_images') {
-            await syncUpdateService.updateNoteImage(resp, syncContext.sourceId);
+            await syncUpdateService.updateNoteImage(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'labels') {
-            await syncUpdateService.updateLabel(resp, syncContext.sourceId);
+            await syncUpdateService.updateLabel(entity, syncContext.sourceId);
         }
         else if (sync.entityName === 'api_tokens') {
-            await syncUpdateService.updateApiToken(resp, syncContext.sourceId);
+            await syncUpdateService.updateApiToken(entity, syncContext.sourceId);
         }
         else {
             throw new Error(`Unrecognized entity type ${sync.entityName} in sync #${sync.id}`);