sync status widget

src/public/app/services/ws.js

@@ -12,9 +12,10 @@ const messageHandlers = [];
 
 let ws;
 let lastAcceptedEntityChangeId = window.glob.maxEntityChangeIdAtLoad;
+let lastAcceptedEntityChangeSyncId = window.glob.maxEntityChangeSyncIdAtLoad;
 let lastProcessedEntityChangeId = window.glob.maxEntityChangeIdAtLoad;
 let lastPingTs;
-let syncDataQueue = [];
+let frontendUpdateDataQueue = [];
 
 function logError(message) {
     console.error(utils.now(), message); // needs to be separate from .trace()
@@ -34,7 +35,7 @@ function subscribeToMessages(messageHandler) {
     messageHandlers.push(messageHandler);
 }
 
-// used to serialize sync operations
+// used to serialize frontend update operations
 let consumeQueuePromise = null;
 
 // to make sure each change event is processed only once. Not clear if this is still necessary
@@ -46,7 +47,7 @@ function logRows(entityChanges) {
         && (row.entityName !== 'options' || row.entityId !== 'openTabs'));
 
     if (filteredRows.length > 0) {
-        console.debug(utils.now(), "Sync data: ", filteredRows);
+        console.debug(utils.now(), "Frontend update data: ", filteredRows);
     }
 }
 
@@ -57,17 +58,24 @@ async function handleMessage(event) {
         messageHandler(message);
     }
 
-    if (message.type === 'sync') {
+    if (message.type === 'frontend-update') {
-        let entityChanges = message.data;
+        let {entityChanges, lastSyncedPush} = message.data;
         lastPingTs = Date.now();
 
         if (entityChanges.length > 0) {
             logRows(entityChanges);
 
-            syncDataQueue.push(...entityChanges);
+            frontendUpdateDataQueue.push(...entityChanges);
 
-            // we set lastAcceptedEntityChangeId even before sync processing and send ping so that backend can start sending more updates
+            // we set lastAcceptedEntityChangeId even before frontend update processing and send ping so that backend can start sending more updates
             lastAcceptedEntityChangeId = Math.max(lastAcceptedEntityChangeId, entityChanges[entityChanges.length - 1].id);
+
+            const lastSyncEntityChange = entityChanges.slice().reverse().find(ec => ec.isSynced);
+
+            if (lastSyncEntityChange) {
+                lastAcceptedEntityChangeSyncId = Math.max(lastAcceptedEntityChangeSyncId, lastSyncEntityChange.id);
+            }
+
             sendPing();
 
             // first wait for all the preceding consumers to finish
@@ -77,7 +85,7 @@ async function handleMessage(event) {
 
             try {
                 // it's my turn so start it up
-                consumeQueuePromise = consumeSyncData();
+                consumeQueuePromise = consumeFrontendUpdateData();
 
                 await consumeQueuePromise;
             }
@@ -129,19 +137,10 @@ function checkEntityChangeIdListeners() {
         .forEach(l => console.log(`Waiting for entityChangeId ${l.desiredEntityChangeId} while last processed is ${lastProcessedEntityChangeId} (last accepted ${lastAcceptedEntityChangeId}) for ${Math.floor((Date.now() - l.start) / 1000)}s`));
 }
 
-async function runSafely(syncHandler, syncData) {
+async function consumeFrontendUpdateData() {
-    try {
+    if (frontendUpdateDataQueue.length > 0) {
-        return await syncHandler(syncData);
+        const allEntityChanges = frontendUpdateDataQueue;
-    }
+        frontendUpdateDataQueue = [];
-    catch (e) {
-        console.log(`Sync handler failed with ${e.message}: ${e.stack}`);
-    }
-}
-
-async function consumeSyncData() {
-    if (syncDataQueue.length > 0) {
-        const allEntityChanges = syncDataQueue;
-        syncDataQueue = [];
 
         const nonProcessedEntityChanges = allEntityChanges.filter(ec => !processedEntityChangeIds.has(ec.id));
 
@@ -213,30 +212,6 @@ setTimeout(() => {
     setInterval(sendPing, 1000);
 }, 0);
 
-subscribeToMessages(async message => {
-    const appContext = (await import("./app_context.js")).default;
-
-    if (message.type === 'sync-pull-in-progress') {
-        toastService.showPersistent({
-            id: 'sync',
-            title: "Sync status",
-            message: "Sync update in progress",
-            icon: "refresh"
-        });
-
-        appContext.triggerEvent('syncInProgress');
-    }
-    else if (message.type === 'sync-finished') {
-        // this gives user a chance to see the toast in case of fast sync finish
-        setTimeout(() => toastService.closePersistent('sync'), 1000);
-
-        appContext.triggerEvent('syncFinished');
-    }
-    else if (message.type === 'sync-failed') {
-        appContext.triggerEvent('syncFailed');
-    }
-});
-
 async function processEntityChanges(entityChanges) {
     const loadResults = new LoadResults(treeCache);
 
@@ -413,5 +388,6 @@ export default {
     logError,
     subscribeToMessages,
     waitForEntityChangeId,
-    waitForMaxKnownEntityChangeId
+    waitForMaxKnownEntityChangeId,
+    getMaxKnownEntityChangeSyncId: () => lastAcceptedEntityChangeSyncId
 };

src/public/app/widgets/global_menu.js

@@ -1,6 +1,5 @@
 import BasicWidget from "./basic_widget.js";
 import utils from "../services/utils.js";
-import syncService from "../services/sync.js";
 
 const TPL = `
 <div class="global-menu-wrapper">
@@ -45,11 +44,6 @@ const TPL = `
                 Options
             </a>
 
-            <a class="dropdown-item sync-now-button" title="Trigger sync">
-                <span class="bx bx-refresh"></span>
-                Sync now
-            </a>
-
             <a class="dropdown-item" data-trigger-command="openNewWindow">
                 <span class="bx bx-window-open"></span>
                 Open new window
@@ -121,8 +115,6 @@ export default class GlobalMenuWidget extends BasicWidget {
         this.$widget.find(".show-about-dialog-button").on('click',
             () => import("../dialogs/about.js").then(d => d.showDialog()));
 
-        this.$widget.find(".sync-now-button").on('click', () => syncService.syncNow());
-
         this.$widget.find(".logout-button").toggle(!utils.isElectron());
 
         this.$widget.find(".open-dev-tools-button").toggle(utils.isElectron());

src/public/app/widgets/sync_status.js

@@ -1,9 +1,13 @@
 import BasicWidget from "./basic_widget.js";
+import toastService from "../services/toast.js";
+import ws from "../services/ws.js";
+import options from "../services/options.js";
+import syncService from "../services/sync.js";
 
 const TPL = `
-<div class="sync-status-wrapper">
+<div class="sync-status-widget">
     <style>
-    .sync-status-wrapper {
+    .sync-status-widget {
         height: 35px;
         box-sizing: border-box;
         border-bottom: 1px solid var(--main-border-color);
@@ -12,78 +16,127 @@ const TPL = `
     .sync-status {
         height: 34px;
         box-sizing: border-box;
-    }
-    
-    .sync-status button {
-        height: 34px;
-        border: none;
-        font-size: 180%;
         padding-left: 10px;
         padding-right: 10px;
     }
     
-    .sync-status button > span {
+    .sync-status .sync-status-icon {
-        display: inline-block; 
+        height: 34px;
+        font-size: 180%;
+        display: inline-block;
         position: relative;
-        top: -5px;
     }
     
-    .sync-status button:hover {
+    .sync-status .sync-status-icon span {
+        border: none !important;
+    }
+    
+    .sync-status-icon:not(.sync-status-in-progress):hover {
         background-color: var(--hover-item-background-color);
-    }
+        cursor: pointer;
-    
-    .sync-status .dropdown-menu {
-        width: 20em;
     }
     </style>
 
     <div class="sync-status">
-        <button type="button" class="btn btn-sm" title="Sync status">
+        <span class="sync-status-icon sync-status-connected-with-changes" title="<p>Connected to the sync server. <br>There are some outstanding changes yet to be synced.</p><p>Click to trigger sync.</p>">
-            <span class="sync-status-icon sync-status-online-with-changes" title="Connected to the sync server. There are some outstanding changes yet to be synced.">
+            <span class="bx bx-wifi"></span>
-                <span class="bx bx-wifi"></span>
+            <span class="bx bxs-star" style="font-size: 40%; position: absolute; left: -3px; top: 20px;"></span>
-                <span class="bx bxs-star" style="font-size: 40%; position: absolute; left: -3px; top: 20px;"></span>
+        </span>
-            </span>
+        <span class="sync-status-icon sync-status-connected-no-changes" 
-            <span class="sync-status-icon sync-status-online-no-changes" title="Connected to the sync server. All changes have been already synced.">
+              data-toggle="tooltip" 
-                <span class="bx bx-wifi"></span>
+              title="<p>Connected to the sync server.<br>All changes have been already synced.</p><p>Click to trigger sync.</p>">
-            </span>
+            <span class="bx bx-wifi"></span>
-            <span class="sync-status-icon sync-status-offline-with-changes" title="Establishing the connection to the sync server was unsuccessful. There are some outstanding changes yet to be synced.">
+        </span>
-                <span class="bx bx-wifi-off"></span>
+        <span class="sync-status-icon sync-status-disconnected-with-changes"
-                <span class="bx bxs-star" style="font-size: 40%; position: absolute; left: -3px; top: 20px;"></span>
+              data-toggle="tooltip" 
-            </span>
+              title="<p>Establishing the connection to the sync server was unsuccessful.<br>There are some outstanding changes yet to be synced.</p><p>Click to trigger sync.</p>">
-            <span class="sync-status-icon sync-status-offline-no-changes" title="Establishing the connection to the sync server was unsuccessful. All known changes have been synced.">
+            <span class="bx bx-wifi-off"></span>
-                <span class="bx bx-wifi-off"></span>
+            <span class="bx bxs-star" style="font-size: 40%; position: absolute; left: -3px; top: 20px;"></span>
-            </span>
+        </span>
-            <span class="sync-status-icon sync-status-in-progress" title="Sync with the server is in progress.">
+        <span class="sync-status-icon sync-status-disconnected-no-changes" 
-                <span class="bx bx-analyse bx-spin"></span>
+              data-toggle="tooltip"
-            </span>
+              title="<p>Establishing the connection to the sync server was unsuccessful.<br>All known changes have been synced.</p><p>Click to trigger sync.</p>">
-        </button>
+            <span class="bx bx-wifi-off"></span>
+        </span>
+        <span class="sync-status-icon sync-status-in-progress" 
+              data-toggle="tooltip"
+              title="Sync with the server is in progress.">
+            <span class="bx bx-analyse bx-spin"></span>
+        </span>
     </div>
 </div>
 `;
 
 export default class SyncStatusWidget extends BasicWidget {
+    constructor() {
+        super();
+
+        ws.subscribeToMessages(message => this.processMessage(message));
+
+        this.syncState = 'disconnected';
+        this.allChangesPushed = false;
+    }
+
     doRender() {
         this.$widget = $(TPL);
         this.$widget.hide();
 
+        this.$widget.find('[data-toggle="tooltip"]').tooltip({
+            html: true
+        });
+
+        this.$widget.find('.sync-status-icon:not(.sync-status-in-progress)')
+            .on('click', () => syncService.syncNow())
+
         this.overflowing();
     }
 
-    syncInProgressEvent() {
-        this.showIcon('in-progress');
-    }
-
-    syncFinishedEvent() {
-        this.showIcon('online-no-changes');
-    }
-
-    syncFailedEvent() {
-        this.showIcon('offline-no-changes');
-    }
-
     showIcon(className) {
+        if (!options.get('syncServerHost')) {
+            this.$widget.hide();
+            return;
+        }
+
         this.$widget.show();
         this.$widget.find('.sync-status-icon').hide();
         this.$widget.find('.sync-status-' + className).show();
     }
+
+    processMessage(message) {
+        if (message.type === 'sync-pull-in-progress') {
+            toastService.showPersistent({
+                id: 'sync',
+                title: "Sync status",
+                message: "Sync update in progress",
+                icon: "refresh"
+            });
+
+            this.syncState = 'in-progress';
+            this.allChangesPushed = false;
+        }
+        else if (message.type === 'sync-push-in-progress') {
+            this.syncState = 'in-progress';
+            this.allChangesPushed = false;
+        }
+        else if (message.type === 'sync-finished') {
+            // this gives user a chance to see the toast in case of fast sync finish
+            setTimeout(() => toastService.closePersistent('sync'), 1000);
+
+            this.syncState = 'connected';
+        }
+        else if (message.type === 'sync-failed') {
+            this.syncState = 'disconnected';
+        }
+        else if (message.type === 'frontend-update') {
+            const {lastSyncedPush} = message.data;
+
+            this.allChangesPushed = lastSyncedPush === ws.getMaxKnownEntityChangeSyncId();
+        }
+
+        if (this.syncState === 'in-progress') {
+            this.showIcon('in-progress');
+        } else {
+            this.showIcon(this.syncState + '-' + (this.allChangesPushed ? 'no-changes' : 'with-changes'));
+        }
+    }
 }

src/routes/index.js

@@ -25,6 +25,7 @@ function index(req, res) {
         detailFontSize: parseInt(options.detailFontSize),
         sourceId: sourceIdService.generateSourceId(),
         maxEntityChangeIdAtLoad: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes"),
+        maxEntityChangeSyncIdAtLoad: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1"),
         instanceName: config.General ? config.General.instanceName : null,
         appCssNoteIds: getAppCssNoteIds(),
         isDev: env.isDev(),

src/services/sql.js

@@ -234,7 +234,7 @@ function transactional(func) {
         const ret = dbConnection.transaction(func).deferred();
 
         if (!dbConnection.inTransaction) { // i.e. transaction was really committed (and not just savepoint released)
-            require('./ws.js').sendTransactionSyncsToAllClients();
+            require('./ws.js').sendTransactionEntityChangesToAllClients();
         }
 
         return ret;

src/services/sync.js

@@ -363,10 +363,16 @@ function setLastSyncedPull(entityChangeId) {
 }
 
 function getLastSyncedPush() {
-    return parseInt(optionService.getOption('lastSyncedPush'));
+    const lastSyncedPush = parseInt(optionService.getOption('lastSyncedPush'));
+
+    ws.setLastSyncedPush(lastSyncedPush);
+
+    return lastSyncedPush;
 }
 
 function setLastSyncedPush(entityChangeId) {
+    ws.setLastSyncedPush(entityChangeId);
+
     optionService.setOption('lastSyncedPush', entityChangeId);
 }
 
@@ -382,9 +388,12 @@ sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(sync), 60000);
 
     // kickoff initial sync immediately
-    setTimeout(cls.wrap(sync), 3000);
+    setTimeout(cls.wrap(sync), 5000);
 });
 
+// called just so ws.setLastSyncedPush() is called
+getLastSyncedPush();
+
 module.exports = {
     sync,
     login,

src/services/ws.js

@@ -8,6 +8,7 @@ const syncMutexService = require('./sync_mutex');
 const protectedSessionService = require('./protected_session');
 
 let webSocketServer;
+let lastSyncedPush = null;
 
 function init(httpServer, sessionParser) {
     webSocketServer = new WebSocket.Server({
@@ -61,7 +62,9 @@ function sendMessageToAllClients(message) {
     const jsonStr = JSON.stringify(message);
 
     if (webSocketServer) {
-        log.info("Sending message to all clients: " + jsonStr);
+        if (message.type !== 'sync-failed') {
+            log.info("Sending message to all clients: " + jsonStr);
+        }
 
         webSocketServer.clients.forEach(function each(client) {
             if (client.readyState === WebSocket.OPEN) {
@@ -96,23 +99,26 @@ function fillInAdditionalProperties(entityChange) {
 }
 
 function sendPing(client, entityChanges = []) {
-    for (const sync of entityChanges) {
+    for (const entityChange of entityChanges) {
         try {
-            fillInAdditionalProperties(sync);
+            fillInAdditionalProperties(entityChange);
         }
         catch (e) {
-            log.error("Could not fill additional properties for sync " + JSON.stringify(sync)
+            log.error("Could not fill additional properties for entity change " + JSON.stringify(entityChange)
                 + " because of error: " + e.message + ": " + e.stack);
         }
     }
 
     sendMessage(client, {
-        type: 'sync',
+        type: 'frontend-update',
-        data: entityChanges
+        data: {
+            lastSyncedPush,
+            entityChanges
+        }
     });
 }
 
-function sendTransactionSyncsToAllClients() {
+function sendTransactionEntityChangesToAllClients() {
     if (webSocketServer) {
         const entityChanges = cls.getAndClearEntityChanges();
 
@@ -136,6 +142,10 @@ function syncFailed() {
     sendMessageToAllClients({ type: 'sync-failed' });
 }
 
+function setLastSyncedPush(entityChangeId) {
+    lastSyncedPush = entityChangeId;
+}
+
 module.exports = {
     init,
     sendMessageToAllClients,
@@ -143,5 +153,6 @@ module.exports = {
     syncPullInProgress,
     syncFinished,
     syncFailed,
-    sendTransactionSyncsToAllClients
+    sendTransactionEntityChangesToAllClients,
+    setLastSyncedPush
 };

src/views/desktop.ejs

@@ -49,6 +49,7 @@
         activeDialog: null,
         sourceId: '<%= sourceId %>',
         maxEntityChangeIdAtLoad: <%= maxEntityChangeIdAtLoad %>,
+        maxEntityChangeSyncIdAtLoad: <%= maxEntityChangeSyncIdAtLoad %>,
         instanceName: '<%= instanceName %>',
         csrfToken: '<%= csrfToken %>',
         isDev: <%= isDev %>,

src/views/mobile.ejs

@@ -111,6 +111,7 @@
         activeDialog: null,
         sourceId: '<%= sourceId %>',
         maxEntityChangeIdAtLoad: <%= maxEntityChangeIdAtLoad %>,
+        maxEntityChangeSyncIdAtLoad: <%= maxEntityChangeSyncIdAtLoad %>,
         instanceName: '<%= instanceName %>',
         csrfToken: '<%= csrfToken %>',
         isDev: <%= isDev %>,