From 1fe7c62f5a90e8dbf6e93f268069dbf87fc59d74 Mon Sep 17 00:00:00 2001 From: azivner Date: Mon, 23 Jul 2018 21:15:32 +0200 Subject: [PATCH] #98, sync setup now doesn't copy the whole DB file, but sets up minimal database and starts off sync --- db/migrations/0102__fix_sync_entityIds.sql | 2 + package.json | 1 - src/public/javascripts/setup.js | 27 +++++-- src/routes/api/setup.js | 88 ++++++++++------------ src/routes/api/sync.js | 23 ++++-- src/routes/routes.js | 3 +- src/services/app_info.js | 2 +- src/services/note_cache.js | 11 +++ src/services/options_init.js | 35 +++++---- src/services/sql_init.js | 39 +++++++--- src/services/sync.js | 44 ++++++++--- src/views/setup.ejs | 10 +++ 12 files changed, 185 insertions(+), 100 deletions(-) create mode 100644 db/migrations/0102__fix_sync_entityIds.sql diff --git a/db/migrations/0102__fix_sync_entityIds.sql b/db/migrations/0102__fix_sync_entityIds.sql new file mode 100644 index 000000000..15e565340 --- /dev/null +++ b/db/migrations/0102__fix_sync_entityIds.sql @@ -0,0 +1,2 @@ +DELETE FROM sync WHERE entityName = 'note_tree'; +DELETE FROM sync WHERE entityName = 'attributes'; diff --git a/package.json b/package.json index 1aa92c01d..1e8640463 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,6 @@ "simple-node-logger": "^0.93.37", "sqlite": "^2.9.2", "tar-stream": "^1.6.1", - "tmp-promise": "^1.0.5", "unescape": "^1.0.1", "ws": "^5.2.1", "xml2js": "^0.4.19" diff --git a/src/public/javascripts/setup.js b/src/public/javascripts/setup.js index 7fdafff24..1cf530e20 100644 --- a/src/public/javascripts/setup.js +++ b/src/public/javascripts/setup.js @@ -34,7 +34,7 @@ function SetupModel() { this.setupSyncFromDesktop(false); }; - this.finish = () => { + this.finish = async () => { if (this.setupNewDocument()) { const username = this.username(); const password1 = this.password1(); @@ -84,20 +84,33 @@ function SetupModel() { } // not using server.js because it loads too many dependencies - $.post('/api/setup/sync-from-server', { + const resp = await $.post('/api/setup/sync-from-server', { serverAddress: serverAddress, username: username, password: password - }).then(() => { - window.location.replace("/"); - }).catch((err) => { - alert("Error, see dev console for details."); - console.error(err); }); + + if (resp.result === 'success') { + this.step('sync-in-progress'); + + checkOutstandingSyncs(); + + setInterval(checkOutstandingSyncs, 1000); + } + else { + showAlert('Sync setup failed: ', resp.error); + } } }; } +async function checkOutstandingSyncs() { + const stats = await $.get('/api/sync/stats'); + const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls; + + $("#outstanding-syncs").html(totalOutstandingSyncs); +} + function showAlert(message) { $("#alert").html(message); $("#alert").show(); diff --git a/src/routes/api/setup.js b/src/routes/api/setup.js index a07674169..48c7c5cd4 100644 --- a/src/routes/api/setup.js +++ b/src/routes/api/setup.js @@ -2,14 +2,10 @@ const sqlInit = require('../../services/sql_init'); const sql = require('../../services/sql'); -const cls = require('../../services/cls'); -const tmp = require('tmp-promise'); -const http = require('http'); -const fs = require('fs'); +const rp = require('request-promise'); +const Option = require('../../entities/option'); +const syncService = require('../../services/sync'); const log = require('../../services/log'); -const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH; -const sourceIdService = require('../../services/source_id'); -const url = require('url'); async function setupNewDocument(req) { const { username, password } = req.body; @@ -20,52 +16,44 @@ async function setupNewDocument(req) { async function setupSyncFromServer(req) { const { serverAddress, username, password } = req.body; - const tempFile = await tmp.file(); + try { + log.info("Getting document options from sync server."); - await new Promise((resolve, reject) => { - const file = fs.createWriteStream(tempFile.path); - const parsedAddress = url.parse(serverAddress); + // response is expected to contain documentId and documentSecret options + const options = await rp.get({ + uri: serverAddress + '/api/sync/document', + auth: { + 'user': username, + 'pass': password + }, + json: true + }); - const options = { - method: 'GET', - protocol: parsedAddress.protocol, - host: parsedAddress.hostname, - port: parsedAddress.port, - path: '/api/sync/document', - auth: username + ':' + password + log.info("Creating database for sync"); + + await sql.transactional(async () => { + await sqlInit.createDatabaseForSync(serverAddress); + + for (const opt of options) { + await new Option(opt).save(); + } + }); + + log.info("Triggering sync."); + + // it's ok to not wait for it here + syncService.sync(); + + return { result: 'success' }; + } + catch (e) { + log.error("Sync failed: " + e.message); + + return { + result: 'failure', + error: e.message }; - - log.info("Getting document from: " + serverAddress); - - http.request(options, function(response) { - response.pipe(file); - - file.on('finish', function() { - log.info("Document download finished, closing & renaming."); - - file.close(() => { // close() is async, call after close completes. - fs.rename(tempFile.path, DOCUMENT_PATH, async () => { - cls.reset(); - - await sqlInit.initDbConnection(); - - // we need to generate new source ID for this instance, otherwise it will - // match the original server one - await sql.transactional(async () => { - await sourceIdService.generateSourceId(); - }); - - resolve(); - }); - }); - }); - }).on('error', function(err) { // Handle errors - fs.unlink(tempFile.path); // Delete the file async. (But we don't check the result) - - reject(err.message); - log.error(err.message); - }).end(); - }); + } } module.exports = { diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js index 1400b0037..b37d2f954 100644 --- a/src/routes/api/sync.js +++ b/src/routes/api/sync.js @@ -7,7 +7,7 @@ const sql = require('../../services/sql'); const optionService = require('../../services/options'); const contentHashService = require('../../services/content_hash'); const log = require('../../services/log'); -const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH; +const repository = require('../../services/repository'); async function testSync() { try { @@ -23,6 +23,10 @@ async function testSync() { } } +async function getStats() { + return syncService.stats; +} + async function checkSync() { return { hashes: await contentHashService.getHashes(), @@ -75,7 +79,10 @@ async function getChanged(req) { const syncs = await sql.getRows("SELECT * FROM sync WHERE id > ? LIMIT 1000", [lastSyncId]); - return await syncService.getSyncRecords(syncs); + return { + syncs: await syncService.getSyncRecords(syncs), + maxSyncId: await sql.getValue('SELECT MAX(id) FROM sync') + }; } async function update(req) { @@ -87,10 +94,13 @@ async function update(req) { } } -async function getDocument(req, resp) { - log.info("Serving document."); +async function getDocument() { + log.info("Serving document options."); - resp.sendFile(DOCUMENT_PATH); + return [ + await repository.getOption('documentId'), + await repository.getOption('documentSecret') + ]; } module.exports = { @@ -102,5 +112,6 @@ module.exports = { forceNoteSync, getChanged, update, - getDocument + getDocument, + getStats }; \ No newline at end of file diff --git a/src/routes/routes.js b/src/routes/routes.js index e1b3f5942..0d6cec705 100644 --- a/src/routes/routes.js +++ b/src/routes/routes.js @@ -156,7 +156,8 @@ function register(app) { apiRoute(POST, '/api/sync/force-note-sync/:noteId', syncApiRoute.forceNoteSync); apiRoute(GET, '/api/sync/changed', syncApiRoute.getChanged); apiRoute(PUT, '/api/sync/update', syncApiRoute.update); - route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument); + route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument, apiResultHandler); + route(GET, '/api/sync/stats', [], syncApiRoute.getStats, apiResultHandler); apiRoute(GET, '/api/event-log', eventLogRoute.getEventLog); diff --git a/src/services/app_info.js b/src/services/app_info.js index ef78a4a78..879f53c78 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -3,7 +3,7 @@ const build = require('./build'); const packageJson = require('../../package'); -const APP_DB_VERSION = 101; +const APP_DB_VERSION = 102; const SYNC_VERSION = 1; module.exports = { diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 9f4105e99..dacdae569 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -5,6 +5,7 @@ const repository = require('./repository'); const protectedSessionService = require('./protected_session'); const utils = require('./utils'); +let loaded = false; let noteTitles; let protectedNoteTitles; let noteIds; @@ -34,6 +35,8 @@ async function load() { for (const noteId of hiddenLabels) { archived[noteId] = true; } + + loaded = true; } function findNotes(query) { @@ -226,6 +229,10 @@ function getNotePath(noteId) { } eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entityId}) => { + if (!loaded) { + return; + } + if (entityName === 'notes') { const note = await repository.getNote(entityId); @@ -277,6 +284,10 @@ eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entityId }); eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, async () => { + if (!loaded) { + return; + } + protectedNoteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 1`); for (const noteId in protectedNoteTitles) { diff --git a/src/services/options_init.js b/src/services/options_init.js index 72232080a..8029c495e 100644 --- a/src/services/options_init.js +++ b/src/services/options_init.js @@ -5,21 +5,14 @@ const appInfo = require('./app_info'); const utils = require('./utils'); const dateUtils = require('./date_utils'); -async function initOptions(startNotePath, username, password) { +async function initDocumentOptions() { await optionService.createOption('documentId', utils.randomSecureToken(16), false); await optionService.createOption('documentSecret', utils.randomSecureToken(16), false); +} - await optionService.createOption('startNotePath', startNotePath, false); - await optionService.createOption('protectedSessionTimeout', 600, true); - await optionService.createOption('noteRevisionSnapshotTimeInterval', 600, true); - await optionService.createOption('lastBackupDate', dateUtils.nowDate(), false); - await optionService.createOption('dbVersion', appInfo.dbVersion, false); - - await optionService.createOption('lastSyncedPull', appInfo.dbVersion, false); - await optionService.createOption('lastSyncedPush', 0, false); - - await optionService.createOption('zoomFactor', 1.0, false); - await optionService.createOption('theme', 'white', false); +async function initSyncedOptions(username, password) { + await optionService.createOption('protectedSessionTimeout', 600); + await optionService.createOption('noteRevisionSnapshotTimeInterval', 600); await optionService.createOption('username', username); @@ -34,12 +27,26 @@ async function initOptions(startNotePath, username, password) { await optionService.createOption('encryptedDataKeyIv', ''); await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16)); +} - await optionService.createOption('syncServerHost', '', false); +async function initNotSyncedOptions(startNotePath = '', syncServerHost = '') { + await optionService.createOption('startNotePath', startNotePath, false); + await optionService.createOption('lastBackupDate', dateUtils.nowDate(), false); + await optionService.createOption('dbVersion', appInfo.dbVersion, false); + + await optionService.createOption('lastSyncedPull', appInfo.dbVersion, false); + await optionService.createOption('lastSyncedPush', 0, false); + + await optionService.createOption('zoomFactor', 1.0, false); + await optionService.createOption('theme', 'white', false); + + await optionService.createOption('syncServerHost', syncServerHost, false); await optionService.createOption('syncServerTimeout', 5000, false); await optionService.createOption('syncProxy', '', false); } module.exports = { - initOptions + initDocumentOptions, + initSyncedOptions, + initNotSyncedOptions }; \ No newline at end of file diff --git a/src/services/sql_init.js b/src/services/sql_init.js index d16cf7cce..e013f5be7 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -12,9 +12,13 @@ async function createConnection() { } let dbReadyResolve = null; -const dbReady = new Promise((resolve, reject) => { +const dbReady = new Promise(async (resolve, reject) => { dbReadyResolve = resolve; + // no need to create new connection now since DB stays the same all the time + const db = await createConnection(); + sql.setDbConnection(db); + initDbConnection(); }); @@ -26,9 +30,6 @@ async function isDbInitialized() { async function initDbConnection() { await cls.init(async () => { - const db = await createConnection(); - sql.setDbConnection(db); - await sql.execute("PRAGMA foreign_keys = ON"); if (!await isDbInitialized()) { @@ -45,12 +46,12 @@ async function initDbConnection() { } log.info("DB ready."); - dbReadyResolve(db); + dbReadyResolve(); }); } async function createInitialDatabase(username, password) { - log.info("Connected to db, but schema doesn't exist. Initializing schema ..."); + log.info("Creating initial database ..."); const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8'); const notesSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_notes.sql', 'UTF-8'); @@ -67,15 +68,34 @@ async function createInitialDatabase(username, password) { const startNoteId = await sql.getValue("SELECT noteId FROM branches WHERE parentNoteId = 'root' AND isDeleted = 0 ORDER BY notePosition"); - await require('./options_init').initOptions(startNoteId, username, password); + const optionsInitService = require('./options_init'); + + await optionsInitService.initDocumentOptions(); + await optionsInitService.initSyncedOptions(username, password); + await optionsInitService.initNotSyncedOptions(startNoteId); + await require('./sync_table').fillAllSyncRows(); }); - log.info("Schema and initial content generated. Waiting for user to enter username/password to finish setup."); + log.info("Schema and initial content generated."); await initDbConnection(); } +async function createDatabaseForSync(syncServerHost) { + log.info("Creating database for sync with server ..."); + + const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8'); + + await sql.transactional(async () => { + await sql.executeScript(schema); + + await require('./options_init').initNotSyncedOptions('', syncServerHost); + }); + + log.info("Schema and not synced options generated."); +} + async function isDbUpToDate() { const dbVersion = parseInt(await sql.getValue("SELECT value FROM options WHERE name = 'dbVersion'")); @@ -93,5 +113,6 @@ module.exports = { isDbInitialized, initDbConnection, isDbUpToDate, - createInitialDatabase + createInitialDatabase, + createDatabaseForSync }; \ No newline at end of file diff --git a/src/services/sync.js b/src/services/sync.js index fb40abbcf..8ece64357 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -18,6 +18,11 @@ const cls = require('./cls'); let proxyToggle = true; +const stats = { + outstandingPushes: 0, + outstandingPulls: 0 +}; + async function sync() { try { await syncMutexService.doExclusively(async () => { @@ -82,21 +87,33 @@ async function login() { } async function pullSync(syncContext) { - const changesUri = '/api/sync/changed?lastSyncId=' + await getLastSyncedPull(); + while (true) { + const lastSyncedPull = await getLastSyncedPull(); + const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull; - const rows = await syncRequest(syncContext, 'GET', changesUri); + const resp = await syncRequest(syncContext, 'GET', changesUri); + stats.outstandingPulls = resp.maxSyncId - lastSyncedPull; - log.info("Pulled " + rows.length + " changes from " + changesUri); + const rows = resp.syncs; - for (const {sync, entity} of rows) { - if (sourceIdService.isLocalSourceId(sync.sourceId)) { - log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`); - } - else { - await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId); + if (rows.length === 0) { + break; } - await setLastSyncedPull(sync.id); + log.info("Pulled " + rows.length + " changes from " + changesUri); + + for (const {sync, entity} of rows) { + if (sourceIdService.isLocalSourceId(sync.sourceId)) { + log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`); + } + else { + await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId); + } + + stats.outstandingPulls = resp.maxSyncId - sync.id; + + await setLastSyncedPull(sync.id); + } } log.info("Finished pull"); @@ -127,6 +144,8 @@ async function pushSync(syncContext) { if (filteredSyncs.length === 0) { log.info("Nothing to push"); + stats.outstandingPushes = 0; + await setLastSyncedPush(lastSyncedPush); break; @@ -144,6 +163,8 @@ async function pushSync(syncContext) { lastSyncedPush = syncRecords[syncRecords.length - 1].sync.id; await setLastSyncedPush(lastSyncedPush); + + stats.outstandingPushes = await sql.getValue(`SELECT MAX(id) FROM sync`) - lastSyncedPush; } } @@ -290,5 +311,6 @@ sqlInit.dbReady.then(async () => { module.exports = { sync, login, - getSyncRecords + getSyncRecords, + stats }; \ No newline at end of file diff --git a/src/views/setup.ejs b/src/views/setup.ejs index 1ac37d28e..8590e63df 100644 --- a/src/views/setup.ejs +++ b/src/views/setup.ejs @@ -86,6 +86,16 @@ + +
+

Sync in progress

+ +
Sync has been correctly set up. It will take some time for the initial sync to finish. Once it's done, you'll be redirected to the login page.
+ +
+ Outstanding sync items: N/A +
+