From 392a00ac179dd61b28d98b85a0fb626471ff5d9e Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 21 Mar 2021 22:43:41 +0100 Subject: [PATCH] sync status widget --- src/public/app/services/ws.js | 68 ++++-------- src/public/app/widgets/global_menu.js | 8 -- src/public/app/widgets/sync_status.js | 147 ++++++++++++++++++-------- src/routes/index.js | 1 + src/services/sql.js | 2 +- src/services/sync.js | 13 ++- src/services/ws.js | 27 +++-- src/views/desktop.ejs | 1 + src/views/mobile.ejs | 1 + 9 files changed, 156 insertions(+), 112 deletions(-) diff --git a/src/public/app/services/ws.js b/src/public/app/services/ws.js index 82f56a346..e2b171566 100644 --- a/src/public/app/services/ws.js +++ b/src/public/app/services/ws.js @@ -12,9 +12,10 @@ const messageHandlers = []; let ws; let lastAcceptedEntityChangeId = window.glob.maxEntityChangeIdAtLoad; +let lastAcceptedEntityChangeSyncId = window.glob.maxEntityChangeSyncIdAtLoad; let lastProcessedEntityChangeId = window.glob.maxEntityChangeIdAtLoad; let lastPingTs; -let syncDataQueue = []; +let frontendUpdateDataQueue = []; function logError(message) { console.error(utils.now(), message); // needs to be separate from .trace() @@ -34,7 +35,7 @@ function subscribeToMessages(messageHandler) { messageHandlers.push(messageHandler); } -// used to serialize sync operations +// used to serialize frontend update operations let consumeQueuePromise = null; // to make sure each change event is processed only once. Not clear if this is still necessary @@ -46,7 +47,7 @@ function logRows(entityChanges) { && (row.entityName !== 'options' || row.entityId !== 'openTabs')); if (filteredRows.length > 0) { - console.debug(utils.now(), "Sync data: ", filteredRows); + console.debug(utils.now(), "Frontend update data: ", filteredRows); } } @@ -57,17 +58,24 @@ async function handleMessage(event) { messageHandler(message); } - if (message.type === 'sync') { - let entityChanges = message.data; + if (message.type === 'frontend-update') { + let {entityChanges, lastSyncedPush} = message.data; lastPingTs = Date.now(); if (entityChanges.length > 0) { logRows(entityChanges); - syncDataQueue.push(...entityChanges); + frontendUpdateDataQueue.push(...entityChanges); - // we set lastAcceptedEntityChangeId even before sync processing and send ping so that backend can start sending more updates + // we set lastAcceptedEntityChangeId even before frontend update processing and send ping so that backend can start sending more updates lastAcceptedEntityChangeId = Math.max(lastAcceptedEntityChangeId, entityChanges[entityChanges.length - 1].id); + + const lastSyncEntityChange = entityChanges.slice().reverse().find(ec => ec.isSynced); + + if (lastSyncEntityChange) { + lastAcceptedEntityChangeSyncId = Math.max(lastAcceptedEntityChangeSyncId, lastSyncEntityChange.id); + } + sendPing(); // first wait for all the preceding consumers to finish @@ -77,7 +85,7 @@ async function handleMessage(event) { try { // it's my turn so start it up - consumeQueuePromise = consumeSyncData(); + consumeQueuePromise = consumeFrontendUpdateData(); await consumeQueuePromise; } @@ -129,19 +137,10 @@ function checkEntityChangeIdListeners() { .forEach(l => console.log(`Waiting for entityChangeId ${l.desiredEntityChangeId} while last processed is ${lastProcessedEntityChangeId} (last accepted ${lastAcceptedEntityChangeId}) for ${Math.floor((Date.now() - l.start) / 1000)}s`)); } -async function runSafely(syncHandler, syncData) { - try { - return await syncHandler(syncData); - } - catch (e) { - console.log(`Sync handler failed with ${e.message}: ${e.stack}`); - } -} - -async function consumeSyncData() { - if (syncDataQueue.length > 0) { - const allEntityChanges = syncDataQueue; - syncDataQueue = []; +async function consumeFrontendUpdateData() { + if (frontendUpdateDataQueue.length > 0) { + const allEntityChanges = frontendUpdateDataQueue; + frontendUpdateDataQueue = []; const nonProcessedEntityChanges = allEntityChanges.filter(ec => !processedEntityChangeIds.has(ec.id)); @@ -213,30 +212,6 @@ setTimeout(() => { setInterval(sendPing, 1000); }, 0); -subscribeToMessages(async message => { - const appContext = (await import("./app_context.js")).default; - - if (message.type === 'sync-pull-in-progress') { - toastService.showPersistent({ - id: 'sync', - title: "Sync status", - message: "Sync update in progress", - icon: "refresh" - }); - - appContext.triggerEvent('syncInProgress'); - } - else if (message.type === 'sync-finished') { - // this gives user a chance to see the toast in case of fast sync finish - setTimeout(() => toastService.closePersistent('sync'), 1000); - - appContext.triggerEvent('syncFinished'); - } - else if (message.type === 'sync-failed') { - appContext.triggerEvent('syncFailed'); - } -}); - async function processEntityChanges(entityChanges) { const loadResults = new LoadResults(treeCache); @@ -413,5 +388,6 @@ export default { logError, subscribeToMessages, waitForEntityChangeId, - waitForMaxKnownEntityChangeId + waitForMaxKnownEntityChangeId, + getMaxKnownEntityChangeSyncId: () => lastAcceptedEntityChangeSyncId }; diff --git a/src/public/app/widgets/global_menu.js b/src/public/app/widgets/global_menu.js index 260e6bd52..4565039cb 100644 --- a/src/public/app/widgets/global_menu.js +++ b/src/public/app/widgets/global_menu.js @@ -1,6 +1,5 @@ import BasicWidget from "./basic_widget.js"; import utils from "../services/utils.js"; -import syncService from "../services/sync.js"; const TPL = `
@@ -45,11 +44,6 @@ const TPL = ` Options - - - Sync now - - Open new window @@ -121,8 +115,6 @@ export default class GlobalMenuWidget extends BasicWidget { this.$widget.find(".show-about-dialog-button").on('click', () => import("../dialogs/about.js").then(d => d.showDialog())); - this.$widget.find(".sync-now-button").on('click', () => syncService.syncNow()); - this.$widget.find(".logout-button").toggle(!utils.isElectron()); this.$widget.find(".open-dev-tools-button").toggle(utils.isElectron()); diff --git a/src/public/app/widgets/sync_status.js b/src/public/app/widgets/sync_status.js index 7da5f8f42..c3bc3a7e2 100644 --- a/src/public/app/widgets/sync_status.js +++ b/src/public/app/widgets/sync_status.js @@ -1,9 +1,13 @@ import BasicWidget from "./basic_widget.js"; +import toastService from "../services/toast.js"; +import ws from "../services/ws.js"; +import options from "../services/options.js"; +import syncService from "../services/sync.js"; const TPL = ` -
+
- + + + + + + + + + + + + + + + + +
`; export default class SyncStatusWidget extends BasicWidget { + constructor() { + super(); + + ws.subscribeToMessages(message => this.processMessage(message)); + + this.syncState = 'disconnected'; + this.allChangesPushed = false; + } + doRender() { this.$widget = $(TPL); this.$widget.hide(); + this.$widget.find('[data-toggle="tooltip"]').tooltip({ + html: true + }); + + this.$widget.find('.sync-status-icon:not(.sync-status-in-progress)') + .on('click', () => syncService.syncNow()) + this.overflowing(); } - syncInProgressEvent() { - this.showIcon('in-progress'); - } - - syncFinishedEvent() { - this.showIcon('online-no-changes'); - } - - syncFailedEvent() { - this.showIcon('offline-no-changes'); - } - showIcon(className) { + if (!options.get('syncServerHost')) { + this.$widget.hide(); + return; + } + this.$widget.show(); this.$widget.find('.sync-status-icon').hide(); this.$widget.find('.sync-status-' + className).show(); } + + processMessage(message) { + if (message.type === 'sync-pull-in-progress') { + toastService.showPersistent({ + id: 'sync', + title: "Sync status", + message: "Sync update in progress", + icon: "refresh" + }); + + this.syncState = 'in-progress'; + this.allChangesPushed = false; + } + else if (message.type === 'sync-push-in-progress') { + this.syncState = 'in-progress'; + this.allChangesPushed = false; + } + else if (message.type === 'sync-finished') { + // this gives user a chance to see the toast in case of fast sync finish + setTimeout(() => toastService.closePersistent('sync'), 1000); + + this.syncState = 'connected'; + } + else if (message.type === 'sync-failed') { + this.syncState = 'disconnected'; + } + else if (message.type === 'frontend-update') { + const {lastSyncedPush} = message.data; + + this.allChangesPushed = lastSyncedPush === ws.getMaxKnownEntityChangeSyncId(); + } + + if (this.syncState === 'in-progress') { + this.showIcon('in-progress'); + } else { + this.showIcon(this.syncState + '-' + (this.allChangesPushed ? 'no-changes' : 'with-changes')); + } + } } diff --git a/src/routes/index.js b/src/routes/index.js index c76e494fa..0de11466a 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -25,6 +25,7 @@ function index(req, res) { detailFontSize: parseInt(options.detailFontSize), sourceId: sourceIdService.generateSourceId(), maxEntityChangeIdAtLoad: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes"), + maxEntityChangeSyncIdAtLoad: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1"), instanceName: config.General ? config.General.instanceName : null, appCssNoteIds: getAppCssNoteIds(), isDev: env.isDev(), diff --git a/src/services/sql.js b/src/services/sql.js index 69cc8b99a..a043898ef 100644 --- a/src/services/sql.js +++ b/src/services/sql.js @@ -234,7 +234,7 @@ function transactional(func) { const ret = dbConnection.transaction(func).deferred(); if (!dbConnection.inTransaction) { // i.e. transaction was really committed (and not just savepoint released) - require('./ws.js').sendTransactionSyncsToAllClients(); + require('./ws.js').sendTransactionEntityChangesToAllClients(); } return ret; diff --git a/src/services/sync.js b/src/services/sync.js index 82c08f1a5..3a6c838b1 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -363,10 +363,16 @@ function setLastSyncedPull(entityChangeId) { } function getLastSyncedPush() { - return parseInt(optionService.getOption('lastSyncedPush')); + const lastSyncedPush = parseInt(optionService.getOption('lastSyncedPush')); + + ws.setLastSyncedPush(lastSyncedPush); + + return lastSyncedPush; } function setLastSyncedPush(entityChangeId) { + ws.setLastSyncedPush(entityChangeId); + optionService.setOption('lastSyncedPush', entityChangeId); } @@ -382,9 +388,12 @@ sqlInit.dbReady.then(() => { setInterval(cls.wrap(sync), 60000); // kickoff initial sync immediately - setTimeout(cls.wrap(sync), 3000); + setTimeout(cls.wrap(sync), 5000); }); +// called just so ws.setLastSyncedPush() is called +getLastSyncedPush(); + module.exports = { sync, login, diff --git a/src/services/ws.js b/src/services/ws.js index f234f0dd8..0fbddf4bd 100644 --- a/src/services/ws.js +++ b/src/services/ws.js @@ -8,6 +8,7 @@ const syncMutexService = require('./sync_mutex'); const protectedSessionService = require('./protected_session'); let webSocketServer; +let lastSyncedPush = null; function init(httpServer, sessionParser) { webSocketServer = new WebSocket.Server({ @@ -61,7 +62,9 @@ function sendMessageToAllClients(message) { const jsonStr = JSON.stringify(message); if (webSocketServer) { - log.info("Sending message to all clients: " + jsonStr); + if (message.type !== 'sync-failed') { + log.info("Sending message to all clients: " + jsonStr); + } webSocketServer.clients.forEach(function each(client) { if (client.readyState === WebSocket.OPEN) { @@ -96,23 +99,26 @@ function fillInAdditionalProperties(entityChange) { } function sendPing(client, entityChanges = []) { - for (const sync of entityChanges) { + for (const entityChange of entityChanges) { try { - fillInAdditionalProperties(sync); + fillInAdditionalProperties(entityChange); } catch (e) { - log.error("Could not fill additional properties for sync " + JSON.stringify(sync) + log.error("Could not fill additional properties for entity change " + JSON.stringify(entityChange) + " because of error: " + e.message + ": " + e.stack); } } sendMessage(client, { - type: 'sync', - data: entityChanges + type: 'frontend-update', + data: { + lastSyncedPush, + entityChanges + } }); } -function sendTransactionSyncsToAllClients() { +function sendTransactionEntityChangesToAllClients() { if (webSocketServer) { const entityChanges = cls.getAndClearEntityChanges(); @@ -136,6 +142,10 @@ function syncFailed() { sendMessageToAllClients({ type: 'sync-failed' }); } +function setLastSyncedPush(entityChangeId) { + lastSyncedPush = entityChangeId; +} + module.exports = { init, sendMessageToAllClients, @@ -143,5 +153,6 @@ module.exports = { syncPullInProgress, syncFinished, syncFailed, - sendTransactionSyncsToAllClients + sendTransactionEntityChangesToAllClients, + setLastSyncedPush }; diff --git a/src/views/desktop.ejs b/src/views/desktop.ejs index 4ec2acdd7..de9adfd94 100644 --- a/src/views/desktop.ejs +++ b/src/views/desktop.ejs @@ -49,6 +49,7 @@ activeDialog: null, sourceId: '<%= sourceId %>', maxEntityChangeIdAtLoad: <%= maxEntityChangeIdAtLoad %>, + maxEntityChangeSyncIdAtLoad: <%= maxEntityChangeSyncIdAtLoad %>, instanceName: '<%= instanceName %>', csrfToken: '<%= csrfToken %>', isDev: <%= isDev %>, diff --git a/src/views/mobile.ejs b/src/views/mobile.ejs index bb7c14353..25c2a9d41 100644 --- a/src/views/mobile.ejs +++ b/src/views/mobile.ejs @@ -111,6 +111,7 @@ activeDialog: null, sourceId: '<%= sourceId %>', maxEntityChangeIdAtLoad: <%= maxEntityChangeIdAtLoad %>, + maxEntityChangeSyncIdAtLoad: <%= maxEntityChangeSyncIdAtLoad %>, instanceName: '<%= instanceName %>', csrfToken: '<%= csrfToken %>', isDev: <%= isDev %>,