#98, sync to server now works as well + a lot of related changes

db/migrations/0104__fill_sync_rows_for_options.js

@@ -0,0 +1,4 @@
+const syncTableService = require('../../src/services/sync_table');
+
+// options has not been filled so far which caused problems with clean-slate sync.
+module.exports = async () => await syncTableService.fillAllSyncRows();

src/entities/entity.js

@@ -1,7 +1,6 @@
 "use strict";
 
 const utils = require('../services/utils');
-const repository = require('../services/repository');
 
 class Entity {
     constructor(row = {}) {
@@ -25,7 +24,7 @@ class Entity {
     }
 
     async save() {
-        await repository.updateEntity(this);
+        await require('../services/repository').updateEntity(this);
 
         return this;
     }

src/public/javascripts/dialogs/options.js

@@ -193,6 +193,7 @@ addTabHandler((function() {
     const $syncServerTimeout = $("#sync-server-timeout");
     const $syncProxy = $("#sync-proxy");
     const $testSyncButton = $("#test-sync-button");
+    const $syncToServerButton = $("#sync-to-server-button");
 
     function optionsLoaded(options) {
         $syncServerHost.val(options['syncServerHost']);
@@ -221,6 +222,12 @@ addTabHandler((function() {
         }
     });
 
+    $syncToServerButton.click(async () => {
+        await server.post("sync/sync-to-server");
+
+        infoService.showMessage("Sync has been established to the server instance. It will take some time to finish.");
+    });
+
     return {
         optionsLoaded
     };

src/routes/api/login.js

@@ -9,8 +9,13 @@ const protectedSessionService = require('../../services/protected_session');
 const appInfo = require('../../services/app_info');
 const eventService = require('../../services/events');
 const cls = require('../../services/cls');
+const sqlInit = require('../../services/sql_init');
 
 async function loginSync(req) {
+    if (!await sqlInit.schemaExists()) {
+        return [400, { message: "DB schema does not exist, can't sync." }];
+    }
+
     const timestampStr = req.body.timestamp;
 
     const timestamp = dateUtils.parseDateTime(timestampStr);

src/routes/api/setup.js

@@ -15,7 +15,14 @@ async function setupSyncFromServer(req) {
     return await setupService.setupSyncFromSyncServer(serverAddress, username, password);
 }
 
+async function setupSyncFromClient(req) {
+    const options = req.body.options;
+
+    await sqlInit.createDatabaseForSync(options);
+}
+
 module.exports = {
     setupNewDocument,
-    setupSyncFromServer
+    setupSyncFromServer,
+    setupSyncFromClient
 };

src/routes/api/sync.js

@@ -4,10 +4,12 @@ const syncService = require('../../services/sync');
 const syncUpdateService = require('../../services/sync_update');
 const syncTableService = require('../../services/sync_table');
 const sql = require('../../services/sql');
+const sqlInit = require('../../services/sql_init');
 const optionService = require('../../services/options');
 const contentHashService = require('../../services/content_hash');
 const log = require('../../services/log');
 const repository = require('../../services/repository');
+const rp = require('request-promise');
 
 async function testSync() {
     try {
@@ -97,15 +99,50 @@ async function update(req) {
     }
 }
 
-async function getDocument() {
-    log.info("Serving document options.");
-
+async function getDocumentOptions() {
     return [
         await repository.getOption('documentId'),
         await repository.getOption('documentSecret')
     ];
 }
 
+async function getDocument() {
+    log.info("Serving document options.");
+
+    return await getDocumentOptions();
+}
+
+async function syncToServer() {
+    log.info("Initiating sync to server");
+
+    // FIXME: add proxy support
+    const syncServerHost = await optionService.getOption('syncServerHost');
+
+    const payload = {
+        options: await getDocumentOptions()
+    };
+
+    await rp({
+        uri: syncServerHost + '/api/setup/sync-from-client',
+        method: 'POST',
+        json: true,
+        body: payload
+    });
+
+    // this is completely new sync, need to reset counters. If this would not be new sync,
+    // the previous request would have failed.
+    await optionService.setOption('lastSyncedPush', 0);
+    await optionService.setOption('lastSyncedPull', 0);
+
+    syncService.sync();
+}
+
+async function syncFinished() {
+    // after first sync finishes, the application is ready to be used
+    // this is meaningless but at the same time harmless (idempotent) for further syncs
+    await sqlInit.dbInitialized();
+}
+
 module.exports = {
     testSync,
     checkSync,
@@ -116,5 +153,7 @@ module.exports = {
     getChanged,
     update,
     getDocument,
-    getStats
+    getStats,
+    syncToServer,
+    syncFinished
 };

src/routes/routes.js

@@ -96,7 +96,7 @@ const uploadMiddleware = multer.single('upload');
 
 function register(app) {
     route(GET, '/', [auth.checkAuth], indexRoute.index);
-    route(GET, '/login', [], loginRoute.loginPage);
+    route(GET, '/login', [auth.checkAppInitialized], loginRoute.loginPage);
     route(POST, '/login', [], loginRoute.login);
     route(POST, '/logout', [auth.checkAuth], loginRoute.logout);
     route(GET, '/setup', [auth.checkAppNotInitialized], setupRoute.setupPage);
@@ -158,6 +158,8 @@ function register(app) {
     apiRoute(PUT, '/api/sync/update', syncApiRoute.update);
     route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument, apiResultHandler);
     route(GET, '/api/sync/stats', [], syncApiRoute.getStats, apiResultHandler);
+    apiRoute(POST, '/api/sync/sync-to-server', syncApiRoute.syncToServer);
+    apiRoute(POST, '/api/sync/finished', syncApiRoute.syncFinished);
 
     apiRoute(GET, '/api/event-log', eventLogRoute.getEventLog);
 
@@ -167,6 +169,7 @@ function register(app) {
 
     route(POST, '/api/setup/new-document', [auth.checkAppNotInitialized], setupApiRoute.setupNewDocument, apiResultHandler);
     route(POST, '/api/setup/sync-from-server', [auth.checkAppNotInitialized], setupApiRoute.setupSyncFromServer, apiResultHandler, false);
+    route(POST, '/api/setup/sync-from-client', [auth.checkAppNotInitialized], setupApiRoute.setupSyncFromClient, apiResultHandler, false);
 
     apiRoute(POST, '/api/sql/execute', sqlRoute.execute);
     apiRoute(POST, '/api/anonymization/anonymize', anonymizationRoute.anonymize);

src/services/app_info.js

@@ -3,7 +3,7 @@
 const build = require('./build');
 const packageJson = require('../../package');
 
-const APP_DB_VERSION = 103;
+const APP_DB_VERSION = 104;
 const SYNC_VERSION = 1;
 
 module.exports = {

src/services/auth.js

@@ -38,6 +38,15 @@ async function checkApiAuth(req, res, next) {
     }
 }
 
+async function checkAppInitialized(req, res, next) {
+    if (!await sqlInit.isDbInitialized()) {
+        res.redirect("setup");
+    }
+    else {
+        next();
+    }
+}
+
 async function checkAppNotInitialized(req, res, next) {
     if (await sqlInit.isDbInitialized()) {
         res.status(400).send("App already initialized.");
@@ -77,6 +86,7 @@ async function checkBasicAuth(req, res, next) {
 module.exports = {
     checkAuth,
     checkApiAuth,
+    checkAppInitialized,
     checkAppNotInitialized,
     checkApiAuthOrElectron,
     checkSenderToken,

src/services/migration.js

@@ -56,28 +56,23 @@ async function migrate() {
                 else if (mig.type === 'js') {
                     console.log("Migration with JS module");
 
-                    const migrationModule = require("../" + resourceDir.MIGRATIONS_DIR + "/" + mig.file);
-                    await migrationModule(db);
+                    const migrationModule = require(resourceDir.MIGRATIONS_DIR + "/" + mig.file);
+                    await migrationModule();
                 }
                 else {
                     throw new Error("Unknown migration type " + mig.type);
                 }
 
                 await optionService.setOption("dbVersion", mig.dbVersion);
-
             });
 
             log.info("Migration to version " + mig.dbVersion + " has been successful.");
-
-            mig['success'] = true;
         }
         catch (e) {
-            mig['success'] = false;
-            mig['error'] = e.stack;
-
             log.error("error during migration to version " + mig.dbVersion + ": " + e.stack);
+            log.error("migration failed, crashing hard"); // this is not very user friendly :-/
 
-            break;
+            process.exit(1);
         }
         finally {
             // make sure foreign keys are enabled even if migration script disables them
@@ -88,8 +83,6 @@ async function migrate() {
     if (await sqlInit.isDbUpToDate()) {
         await sqlInit.initDbConnection();
     }
-
-    return migrations;
 }
 
 module.exports = {

src/services/setup.js

@@ -1,15 +1,16 @@
-const sqlInit = require('./sql_init');
-const sql = require('./sql');
 const rp = require('request-promise');
-const Option = require('../entities/option');
 const syncService = require('./sync');
 const log = require('./log');
-const optionService = require('./options');
+const sqlInit = require('./sql_init');
 
 function triggerSync() {
-// it's ok to not wait for it here
-    syncService.sync().then(async () => {
-        await optionService.setOption('initialized', 'true');
+    log.info("Triggering sync.");
+
+    // it's ok to not wait for it here
+    syncService.sync().then(async res => {
+        if (res.success) {
+            await sqlInit.dbInitialized();
+        }
     });
 }
 
@@ -34,17 +35,7 @@ async function setupSyncFromSyncServer(serverAddress, username, password) {
             json: true
         });
 
-        log.info("Creating database for sync");
-
-        await sql.transactional(async () => {
-            await sqlInit.createDatabaseForSync(serverAddress);
-
-            for (const opt of options) {
-                await new Option(opt).save();
-            }
-        });
-
-        log.info("Triggering sync.");
+        await sqlInit.createDatabaseForSync(options, serverAddress);
 
         triggerSync();
 

src/services/sql_init.js

@@ -6,6 +6,8 @@ const resourceDir = require('./resource_dir');
 const appInfo = require('./app_info');
 const sql = require('./sql');
 const cls = require('./cls');
+const optionService = require('./options');
+const Option = require('../entities/option');
 
 async function createConnection() {
     return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise});
@@ -35,19 +37,20 @@ async function isDbInitialized() {
 
     const initialized = await sql.getValue("SELECT value FROM options WHERE name = 'initialized'");
 
-    return initialized === 'true';
+    // !initialized may be removed in the future, required only for migration
+    return !initialized || initialized === 'true';
 }
 
 async function initDbConnection() {
     await cls.init(async () => {
-        await sql.execute("PRAGMA foreign_keys = ON");
-
         if (!await isDbInitialized()) {
             log.info("DB not initialized, please visit setup page to initialize Trilium.");
 
             return;
         }
 
+        await sql.execute("PRAGMA foreign_keys = ON");
+
         if (!await isDbUpToDate()) {
             // avoiding circular dependency
             const migrationService = require('./migration');
@@ -96,8 +99,12 @@ async function createInitialDatabase(username, password) {
     await initDbConnection();
 }
 
-async function createDatabaseForSync(syncServerHost) {
-    log.info("Creating database for sync with server ...");
+async function createDatabaseForSync(options, syncServerHost = '') {
+    log.info("Creating database for sync");
+
+    if (await isDbInitialized()) {
+        throw new Error("DB is already initialized");
+    }
 
     const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
 
@@ -105,6 +112,11 @@ async function createDatabaseForSync(syncServerHost) {
         await sql.executeScript(schema);
 
         await require('./options_init').initNotSyncedOptions(false, '', syncServerHost);
+
+        // document options required for sync to kick off
+        for (const opt of options) {
+            await new Option(opt).save();
+        }
     });
 
     log.info("Schema and not synced options generated.");
@@ -122,6 +134,12 @@ async function isDbUpToDate() {
     return upToDate;
 }
 
+async function dbInitialized() {
+    await optionService.setOption('initialized', 'true');
+
+    await initDbConnection();
+}
+
 module.exports = {
     dbReady,
     schemaExists,
@@ -129,5 +147,6 @@ module.exports = {
     initDbConnection,
     isDbUpToDate,
     createInitialDatabase,
-    createDatabaseForSync
+    createDatabaseForSync,
+    dbInitialized
 };

src/services/sync.js

@@ -10,7 +10,6 @@ const sourceIdService = require('./source_id');
 const dateUtils = require('./date_utils');
 const syncUpdateService = require('./sync_update');
 const contentHashService = require('./content_hash');
-const fs = require('fs');
 const appInfo = require('./app_info');
 const syncSetup = require('./sync_setup');
 const syncMutexService = require('./sync_mutex');
@@ -25,7 +24,11 @@ const stats = {
 
 async function sync() {
     try {
-        await syncMutexService.doExclusively(async () => {
+        return await syncMutexService.doExclusively(async () => {
+            if (!await syncSetup.isSyncSetup()) {
+                return { success: false, message: 'Sync not configured' };
+            }
+
             const syncContext = await login();
 
             await pushSync(syncContext);
@@ -34,12 +37,14 @@ async function sync() {
 
             await pushSync(syncContext);
 
-            await checkContentHash(syncContext);
-        });
+            await syncFinished(syncContext);
 
-        return {
-            success: true
-        };
+            await checkContentHash(syncContext);
+
+            return {
+                success: true
+            };
+        });
     }
     catch (e) {
         proxyToggle = !proxyToggle;
@@ -53,7 +58,7 @@ async function sync() {
             };
         }
         else {
-            log.info("sync failed: " + e.stack);
+            log.info("sync failed: " + e.message);
 
             return {
                 success: false,
@@ -104,7 +109,8 @@ async function pullSync(syncContext) {
 
         for (const {sync, entity} of rows) {
             if (sourceIdService.isLocalSourceId(sync.sourceId)) {
-                log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
+                // too noisy
+                //log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
             }
             else {
                 await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
@@ -127,7 +133,8 @@ async function pushSync(syncContext) {
 
         const filteredSyncs = syncs.filter(sync => {
             if (sync.sourceId === syncContext.sourceId) {
-                log.info(`Skipping push #${sync.id} ${sync.entityName} ${sync.entityId} because it originates from sync target`);
+                // too noisy
+                //log.info(`Skipping push #${sync.id} ${sync.entityName} ${sync.entityId} because it originates from sync target`);
 
                 // this may set lastSyncedPush beyond what's actually sent (because of size limit)
                 // so this is applied to the database only if there's no actual update
@@ -168,6 +175,10 @@ async function pushSync(syncContext) {
     }
 }
 
+async function syncFinished(syncContext) {
+    await syncRequest(syncContext, 'POST', '/api/sync/finished');
+}
+
 async function checkContentHash(syncContext) {
     const resp = await syncRequest(syncContext, 'GET', '/api/sync/check');
 
@@ -210,7 +221,7 @@ async function syncRequest(syncContext, method, uri, body) {
         return await rp(options);
     }
     catch (e) {
-        throw new Error(`Request to ${method} ${fullUri} failed, inner exception: ${e.stack}`);
+        throw new Error(`Request to ${method} ${fullUri} failed, error: ${e.message}`);
     }
 }
 

src/services/sync_mutex.js

@@ -10,12 +10,11 @@ async function doExclusively(func) {
     const releaseMutex = await instance.acquire();
 
     try {
-        await func();
+        return await func();
     }
     finally {
         releaseMutex();
     }
-
 }
 
 module.exports = {

src/services/sync_setup.js

@@ -1,6 +1,5 @@
 "use strict";
 
-const config = require('./config');
 const optionService = require('./options');
 
 module.exports = {

src/services/sync_table.js

@@ -74,10 +74,11 @@ async function cleanupSyncRowsForMissingEntities(entityName, entityKey) {
         AND sync.entityId NOT IN (SELECT ${entityKey} FROM ${entityName})`);
 }
 
-async function fillSyncRows(entityName, entityKey) {
+async function fillSyncRows(entityName, entityKey, condition = '') {
     await cleanupSyncRowsForMissingEntities(entityName, entityKey);
 
-    const entityIds = await sql.getColumn(`SELECT ${entityKey} FROM ${entityName}`);
+    const entityIds = await sql.getColumn(`SELECT ${entityKey} FROM ${entityName}`
+        + (condition ? ` WHERE ${condition}` : ''));
 
     for (const entityId of entityIds) {
         const existingRows = await sql.getValue("SELECT COUNT(id) FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
@@ -107,6 +108,7 @@ async function fillAllSyncRows() {
     await fillSyncRows("note_images", "noteImageId");
     await fillSyncRows("labels", "labelId");
     await fillSyncRows("api_tokens", "apiTokenId");
+    await fillSyncRows("options", "name", 'isSynced = 1');
 }
 
 module.exports = {

src/services/sync_update.js

@@ -109,7 +109,7 @@ async function updateNoteReordering(entityId, entity, sourceId) {
 async function updateOptions(entity, sourceId) {
     const orig = await sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [entity.name]);
 
-    if (!orig.isSynced) {
+    if (orig && !orig.isSynced) {
         return;
     }
 

src/views/index.ejs

@@ -433,11 +433,11 @@
 
           <button id="test-sync-button" class="btn btn-sm">Test sync</button>
 
-          <h4>Copy document to the server instance</h4>
+          <h4>Sync document to the server instance</h4>
 
-          <p>This is used when you're setting up server instance of Trilium. After the installation the databa</p>
+          <p>This is used when you want to sync your local document to the server instance configured above. This is a one time action after which the documents are synced automatically and transparently.</p>
 
-          <button id="test-sync-button" class="btn btn-sm">Copy document to the sync server</button>
+          <button id="sync-to-server-button" class="btn btn-sm">Sync local document to the server instance</button>
         </div>
         <div id="advanced">
           <h4 style="margin-top: 0px;">Sync</h4>

src/views/setup.ejs

@@ -92,7 +92,7 @@
 
         <div class="alert alert-success">Sync has been correctly set up. It will take some time for the initial sync to finish. Once it's done, you'll be redirected to the login page.</div>
 
-        <div>
+        <div data-bind="if: instanceType == 'desktop'">
             Outstanding sync items: <strong id="outstanding-syncs">N/A</strong>
         </div>
     </div>