Merge remote-tracking branch 'origin/stable'

# Conflicts:
#	package-lock.json
#	package.json

src/public/app/services/ws.js

@@ -181,9 +181,9 @@ async function consumeFrontendUpdateData() {
 
         for (const entityChange of nonProcessedEntityChanges) {
             processedEntityChangeIds.add(entityChange.id);
-        }
 
-        lastProcessedEntityChangeId = Math.max(lastProcessedEntityChangeId, allEntityChanges[allEntityChanges.length - 1].id);
+            lastProcessedEntityChangeId = Math.max(lastProcessedEntityChangeId, entityChange.id);
+        }
     }
 
     checkEntityChangeIdListeners();

src/routes/api/sync.js

@@ -204,6 +204,11 @@ function queueSector(req) {
     entityChangesService.addEntityChangesForSector(entityName, sector);
 }
 
+function checkEntityChanges() {
+    const consistencyChecks = require("../../services/consistency_checks");
+    consistencyChecks.runEntityChangesChecks();
+}
+
 module.exports = {
     testSync,
     checkSync,
@@ -215,5 +220,6 @@ module.exports = {
     update,
     getStats,
     syncFinished,
-    queueSector
+    queueSector,
+    checkEntityChanges
 };

src/routes/routes.js

@@ -295,6 +295,7 @@ function register(app) {
     route(GET, '/api/sync/changed', [auth.checkApiAuth], syncApiRoute.getChanged, apiResultHandler);
     route(PUT, '/api/sync/update', [auth.checkApiAuth], syncApiRoute.update, apiResultHandler);
     route(POST, '/api/sync/finished', [auth.checkApiAuth], syncApiRoute.syncFinished, apiResultHandler);
+    route(POST, '/api/sync/check-entity-changes', [auth.checkApiAuth], syncApiRoute.checkEntityChanges, apiResultHandler);
     route(POST, '/api/sync/queue-sector/:entityName/:sector', [auth.checkApiAuth], syncApiRoute.queueSector, apiResultHandler);
     route(GET, '/api/sync/stats', [], syncApiRoute.getStats, apiResultHandler);
 

src/services/consistency_checks.js

@@ -701,6 +701,11 @@ function runOnDemandChecks(autoFix) {
     consistencyChecks.runChecks();
 }
 
+function runEntityChangesChecks() {
+    const consistencyChecks = new ConsistencyChecks(true);
+    consistencyChecks.findEntityChangeIssues();
+}
+
 sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(runPeriodicChecks), 60 * 60 * 1000);
 
@@ -709,5 +714,6 @@ sqlInit.dbReady.then(() => {
 });
 
 module.exports = {
-    runOnDemandChecks
+    runOnDemandChecks,
+    runEntityChangesChecks
 };

src/services/entity_changes.js

@@ -7,12 +7,10 @@ const becca = require("../becca/becca");
 
 let maxEntityChangeId = 0;
 
-function addEntityChange(origEntityChange, keepOriginalId = false) {
+function addEntityChange(origEntityChange) {
     const ec = {...origEntityChange};
 
-    if (!keepOriginalId) {
+    delete ec.id;
-        delete ec.id;
-    }
 
     ec.sourceId = ec.sourceId || cls.getSourceId() || sourceIdService.getCurrentSourceId();
     ec.isSynced = ec.isSynced ? 1 : 0;

src/services/sync.js

@@ -149,7 +149,10 @@ async function pullChanges(syncContext) {
 
         sql.transactional(() => {
             for (const {entityChange, entity} of entityChanges) {
-                if (!sourceIdService.isLocalSourceId(entityChange.sourceId)) {
+                // FIXME: temporary fix
+                const existsAlready = !!sql.getValue("SELECT id FROM entity_changes WHERE entityName = ? AND entityId = ? AND utcDateChanged = ? AND hash = ?", [entityChange.entityName, entityChange.entityId, entityChange.utcDateChanged, entityChange.hash]);
+
+                if (!existsAlready && !sourceIdService.isLocalSourceId(entityChange.sourceId)) {
                     if (!atLeastOnePullApplied) { // send only for first
                         ws.syncPullInProgress();
 
@@ -249,6 +252,14 @@ async function checkContentHash(syncContext) {
 
     const failedChecks = contentHashService.checkContentHashes(resp.entityHashes);
 
+    if (failedChecks.length > 0) {
+        // before requeuing sectors make sure the entity changes are correct
+        const consistencyChecks = require("./consistency_checks");
+        consistencyChecks.runEntityChangesChecks();
+
+        await syncRequest(syncContext, 'POST', `/api/sync/check-entity-changes`);
+    }
+
     for (const {entityName, sector} of failedChecks) {
         entityChangesService.addEntityChangesForSector(entityName, sector);
 

src/services/sync_update.js

@@ -54,7 +54,7 @@ function updateNormalEntity(remoteEntityChange, entity) {
 
             sql.execute(`DELETE FROM ${remoteEntityChange.entityName} WHERE ${primaryKey} = ?`, remoteEntityChange.entityId);
 
-            entityChangesService.addEntityChange(remoteEntityChange, true);
+            entityChangesService.addEntityChange(remoteEntityChange);
         });
 
         return true;
@@ -71,7 +71,7 @@ function updateNormalEntity(remoteEntityChange, entity) {
         sql.transactional(() => {
             sql.replace(remoteEntityChange.entityName, entity);
 
-            entityChangesService.addEntityChange(remoteEntityChange, true);
+            entityChangesService.addEntityChange(remoteEntityChange);
         });
 
         return true;
@@ -86,7 +86,7 @@ function updateNoteReordering(entityChange, entity) {
             sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]);
         }
 
-        entityChangesService.addEntityChange(entityChange, true);
+        entityChangesService.addEntityChange(entityChange);
     });
 
     return true;