diff --git a/db/migrations/0104__fill_sync_rows_for_options.js b/db/migrations/0104__fill_sync_rows_for_options.js
new file mode 100644
index 000000000..69d1586e7
--- /dev/null
+++ b/db/migrations/0104__fill_sync_rows_for_options.js
@@ -0,0 +1,4 @@
+const syncTableService = require('../../src/services/sync_table');
+
+// options has not been filled so far which caused problems with clean-slate sync.
+module.exports = async () => await syncTableService.fillAllSyncRows();
\ No newline at end of file
diff --git a/src/entities/entity.js b/src/entities/entity.js
index 395cbae9c..8908ae6c2 100644
--- a/src/entities/entity.js
+++ b/src/entities/entity.js
@@ -1,7 +1,6 @@
"use strict";
const utils = require('../services/utils');
-const repository = require('../services/repository');
class Entity {
constructor(row = {}) {
@@ -25,7 +24,7 @@ class Entity {
}
async save() {
- await repository.updateEntity(this);
+ await require('../services/repository').updateEntity(this);
return this;
}
diff --git a/src/public/javascripts/dialogs/options.js b/src/public/javascripts/dialogs/options.js
index e0fa9d5f6..312557a7b 100644
--- a/src/public/javascripts/dialogs/options.js
+++ b/src/public/javascripts/dialogs/options.js
@@ -193,6 +193,7 @@ addTabHandler((function() {
const $syncServerTimeout = $("#sync-server-timeout");
const $syncProxy = $("#sync-proxy");
const $testSyncButton = $("#test-sync-button");
+ const $syncToServerButton = $("#sync-to-server-button");
function optionsLoaded(options) {
$syncServerHost.val(options['syncServerHost']);
@@ -221,6 +222,12 @@ addTabHandler((function() {
}
});
+ $syncToServerButton.click(async () => {
+ await server.post("sync/sync-to-server");
+
+ infoService.showMessage("Sync has been established to the server instance. It will take some time to finish.");
+ });
+
return {
optionsLoaded
};
diff --git a/src/routes/api/login.js b/src/routes/api/login.js
index 0c7f11634..e6866d836 100644
--- a/src/routes/api/login.js
+++ b/src/routes/api/login.js
@@ -9,8 +9,13 @@ const protectedSessionService = require('../../services/protected_session');
const appInfo = require('../../services/app_info');
const eventService = require('../../services/events');
const cls = require('../../services/cls');
+const sqlInit = require('../../services/sql_init');
async function loginSync(req) {
+ if (!await sqlInit.schemaExists()) {
+ return [400, { message: "DB schema does not exist, can't sync." }];
+ }
+
const timestampStr = req.body.timestamp;
const timestamp = dateUtils.parseDateTime(timestampStr);
diff --git a/src/routes/api/setup.js b/src/routes/api/setup.js
index cbc0f5e7d..297a60002 100644
--- a/src/routes/api/setup.js
+++ b/src/routes/api/setup.js
@@ -15,7 +15,14 @@ async function setupSyncFromServer(req) {
return await setupService.setupSyncFromSyncServer(serverAddress, username, password);
}
+async function setupSyncFromClient(req) {
+ const options = req.body.options;
+
+ await sqlInit.createDatabaseForSync(options);
+}
+
module.exports = {
setupNewDocument,
- setupSyncFromServer
+ setupSyncFromServer,
+ setupSyncFromClient
};
\ No newline at end of file
diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js
index 7cfc2b469..cbe6e74a7 100644
--- a/src/routes/api/sync.js
+++ b/src/routes/api/sync.js
@@ -4,10 +4,12 @@ const syncService = require('../../services/sync');
const syncUpdateService = require('../../services/sync_update');
const syncTableService = require('../../services/sync_table');
const sql = require('../../services/sql');
+const sqlInit = require('../../services/sql_init');
const optionService = require('../../services/options');
const contentHashService = require('../../services/content_hash');
const log = require('../../services/log');
const repository = require('../../services/repository');
+const rp = require('request-promise');
async function testSync() {
try {
@@ -97,15 +99,50 @@ async function update(req) {
}
}
-async function getDocument() {
- log.info("Serving document options.");
-
+async function getDocumentOptions() {
return [
await repository.getOption('documentId'),
await repository.getOption('documentSecret')
];
}
+async function getDocument() {
+ log.info("Serving document options.");
+
+ return await getDocumentOptions();
+}
+
+async function syncToServer() {
+ log.info("Initiating sync to server");
+
+ // FIXME: add proxy support
+ const syncServerHost = await optionService.getOption('syncServerHost');
+
+ const payload = {
+ options: await getDocumentOptions()
+ };
+
+ await rp({
+ uri: syncServerHost + '/api/setup/sync-from-client',
+ method: 'POST',
+ json: true,
+ body: payload
+ });
+
+ // this is completely new sync, need to reset counters. If this would not be new sync,
+ // the previous request would have failed.
+ await optionService.setOption('lastSyncedPush', 0);
+ await optionService.setOption('lastSyncedPull', 0);
+
+ syncService.sync();
+}
+
+async function syncFinished() {
+ // after first sync finishes, the application is ready to be used
+ // this is meaningless but at the same time harmless (idempotent) for further syncs
+ await sqlInit.dbInitialized();
+}
+
module.exports = {
testSync,
checkSync,
@@ -116,5 +153,7 @@ module.exports = {
getChanged,
update,
getDocument,
- getStats
+ getStats,
+ syncToServer,
+ syncFinished
};
\ No newline at end of file
diff --git a/src/routes/routes.js b/src/routes/routes.js
index 0d6cec705..fc150c16f 100644
--- a/src/routes/routes.js
+++ b/src/routes/routes.js
@@ -96,7 +96,7 @@ const uploadMiddleware = multer.single('upload');
function register(app) {
route(GET, '/', [auth.checkAuth], indexRoute.index);
- route(GET, '/login', [], loginRoute.loginPage);
+ route(GET, '/login', [auth.checkAppInitialized], loginRoute.loginPage);
route(POST, '/login', [], loginRoute.login);
route(POST, '/logout', [auth.checkAuth], loginRoute.logout);
route(GET, '/setup', [auth.checkAppNotInitialized], setupRoute.setupPage);
@@ -158,6 +158,8 @@ function register(app) {
apiRoute(PUT, '/api/sync/update', syncApiRoute.update);
route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument, apiResultHandler);
route(GET, '/api/sync/stats', [], syncApiRoute.getStats, apiResultHandler);
+ apiRoute(POST, '/api/sync/sync-to-server', syncApiRoute.syncToServer);
+ apiRoute(POST, '/api/sync/finished', syncApiRoute.syncFinished);
apiRoute(GET, '/api/event-log', eventLogRoute.getEventLog);
@@ -167,6 +169,7 @@ function register(app) {
route(POST, '/api/setup/new-document', [auth.checkAppNotInitialized], setupApiRoute.setupNewDocument, apiResultHandler);
route(POST, '/api/setup/sync-from-server', [auth.checkAppNotInitialized], setupApiRoute.setupSyncFromServer, apiResultHandler, false);
+ route(POST, '/api/setup/sync-from-client', [auth.checkAppNotInitialized], setupApiRoute.setupSyncFromClient, apiResultHandler, false);
apiRoute(POST, '/api/sql/execute', sqlRoute.execute);
apiRoute(POST, '/api/anonymization/anonymize', anonymizationRoute.anonymize);
diff --git a/src/services/app_info.js b/src/services/app_info.js
index 65dd9c0b5..e7835d71f 100644
--- a/src/services/app_info.js
+++ b/src/services/app_info.js
@@ -3,7 +3,7 @@
const build = require('./build');
const packageJson = require('../../package');
-const APP_DB_VERSION = 103;
+const APP_DB_VERSION = 104;
const SYNC_VERSION = 1;
module.exports = {
diff --git a/src/services/auth.js b/src/services/auth.js
index ea5d604fb..f9ed65356 100644
--- a/src/services/auth.js
+++ b/src/services/auth.js
@@ -38,6 +38,15 @@ async function checkApiAuth(req, res, next) {
}
}
+async function checkAppInitialized(req, res, next) {
+ if (!await sqlInit.isDbInitialized()) {
+ res.redirect("setup");
+ }
+ else {
+ next();
+ }
+}
+
async function checkAppNotInitialized(req, res, next) {
if (await sqlInit.isDbInitialized()) {
res.status(400).send("App already initialized.");
@@ -77,6 +86,7 @@ async function checkBasicAuth(req, res, next) {
module.exports = {
checkAuth,
checkApiAuth,
+ checkAppInitialized,
checkAppNotInitialized,
checkApiAuthOrElectron,
checkSenderToken,
diff --git a/src/services/migration.js b/src/services/migration.js
index 006a66dc8..8498cb820 100644
--- a/src/services/migration.js
+++ b/src/services/migration.js
@@ -56,28 +56,23 @@ async function migrate() {
else if (mig.type === 'js') {
console.log("Migration with JS module");
- const migrationModule = require("../" + resourceDir.MIGRATIONS_DIR + "/" + mig.file);
- await migrationModule(db);
+ const migrationModule = require(resourceDir.MIGRATIONS_DIR + "/" + mig.file);
+ await migrationModule();
}
else {
throw new Error("Unknown migration type " + mig.type);
}
await optionService.setOption("dbVersion", mig.dbVersion);
-
});
log.info("Migration to version " + mig.dbVersion + " has been successful.");
-
- mig['success'] = true;
}
catch (e) {
- mig['success'] = false;
- mig['error'] = e.stack;
-
log.error("error during migration to version " + mig.dbVersion + ": " + e.stack);
+ log.error("migration failed, crashing hard"); // this is not very user friendly :-/
- break;
+ process.exit(1);
}
finally {
// make sure foreign keys are enabled even if migration script disables them
@@ -88,8 +83,6 @@ async function migrate() {
if (await sqlInit.isDbUpToDate()) {
await sqlInit.initDbConnection();
}
-
- return migrations;
}
module.exports = {
diff --git a/src/services/setup.js b/src/services/setup.js
index 7e6a3a03b..2093a312d 100644
--- a/src/services/setup.js
+++ b/src/services/setup.js
@@ -1,15 +1,16 @@
-const sqlInit = require('./sql_init');
-const sql = require('./sql');
const rp = require('request-promise');
-const Option = require('../entities/option');
const syncService = require('./sync');
const log = require('./log');
-const optionService = require('./options');
+const sqlInit = require('./sql_init');
function triggerSync() {
-// it's ok to not wait for it here
- syncService.sync().then(async () => {
- await optionService.setOption('initialized', 'true');
+ log.info("Triggering sync.");
+
+ // it's ok to not wait for it here
+ syncService.sync().then(async res => {
+ if (res.success) {
+ await sqlInit.dbInitialized();
+ }
});
}
@@ -34,17 +35,7 @@ async function setupSyncFromSyncServer(serverAddress, username, password) {
json: true
});
- log.info("Creating database for sync");
-
- await sql.transactional(async () => {
- await sqlInit.createDatabaseForSync(serverAddress);
-
- for (const opt of options) {
- await new Option(opt).save();
- }
- });
-
- log.info("Triggering sync.");
+ await sqlInit.createDatabaseForSync(options, serverAddress);
triggerSync();
diff --git a/src/services/sql_init.js b/src/services/sql_init.js
index b6e9508ba..3e06184e6 100644
--- a/src/services/sql_init.js
+++ b/src/services/sql_init.js
@@ -6,6 +6,8 @@ const resourceDir = require('./resource_dir');
const appInfo = require('./app_info');
const sql = require('./sql');
const cls = require('./cls');
+const optionService = require('./options');
+const Option = require('../entities/option');
async function createConnection() {
return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise});
@@ -35,19 +37,20 @@ async function isDbInitialized() {
const initialized = await sql.getValue("SELECT value FROM options WHERE name = 'initialized'");
- return initialized === 'true';
+ // !initialized may be removed in the future, required only for migration
+ return !initialized || initialized === 'true';
}
async function initDbConnection() {
await cls.init(async () => {
- await sql.execute("PRAGMA foreign_keys = ON");
-
if (!await isDbInitialized()) {
log.info("DB not initialized, please visit setup page to initialize Trilium.");
return;
}
+ await sql.execute("PRAGMA foreign_keys = ON");
+
if (!await isDbUpToDate()) {
// avoiding circular dependency
const migrationService = require('./migration');
@@ -96,8 +99,12 @@ async function createInitialDatabase(username, password) {
await initDbConnection();
}
-async function createDatabaseForSync(syncServerHost) {
- log.info("Creating database for sync with server ...");
+async function createDatabaseForSync(options, syncServerHost = '') {
+ log.info("Creating database for sync");
+
+ if (await isDbInitialized()) {
+ throw new Error("DB is already initialized");
+ }
const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
@@ -105,6 +112,11 @@ async function createDatabaseForSync(syncServerHost) {
await sql.executeScript(schema);
await require('./options_init').initNotSyncedOptions(false, '', syncServerHost);
+
+ // document options required for sync to kick off
+ for (const opt of options) {
+ await new Option(opt).save();
+ }
});
log.info("Schema and not synced options generated.");
@@ -122,6 +134,12 @@ async function isDbUpToDate() {
return upToDate;
}
+async function dbInitialized() {
+ await optionService.setOption('initialized', 'true');
+
+ await initDbConnection();
+}
+
module.exports = {
dbReady,
schemaExists,
@@ -129,5 +147,6 @@ module.exports = {
initDbConnection,
isDbUpToDate,
createInitialDatabase,
- createDatabaseForSync
+ createDatabaseForSync,
+ dbInitialized
};
\ No newline at end of file
diff --git a/src/services/sync.js b/src/services/sync.js
index 8ece64357..986458e63 100644
--- a/src/services/sync.js
+++ b/src/services/sync.js
@@ -10,7 +10,6 @@ const sourceIdService = require('./source_id');
const dateUtils = require('./date_utils');
const syncUpdateService = require('./sync_update');
const contentHashService = require('./content_hash');
-const fs = require('fs');
const appInfo = require('./app_info');
const syncSetup = require('./sync_setup');
const syncMutexService = require('./sync_mutex');
@@ -25,7 +24,11 @@ const stats = {
async function sync() {
try {
- await syncMutexService.doExclusively(async () => {
+ return await syncMutexService.doExclusively(async () => {
+ if (!await syncSetup.isSyncSetup()) {
+ return { success: false, message: 'Sync not configured' };
+ }
+
const syncContext = await login();
await pushSync(syncContext);
@@ -34,12 +37,14 @@ async function sync() {
await pushSync(syncContext);
- await checkContentHash(syncContext);
- });
+ await syncFinished(syncContext);
- return {
- success: true
- };
+ await checkContentHash(syncContext);
+
+ return {
+ success: true
+ };
+ });
}
catch (e) {
proxyToggle = !proxyToggle;
@@ -53,7 +58,7 @@ async function sync() {
};
}
else {
- log.info("sync failed: " + e.stack);
+ log.info("sync failed: " + e.message);
return {
success: false,
@@ -104,7 +109,8 @@ async function pullSync(syncContext) {
for (const {sync, entity} of rows) {
if (sourceIdService.isLocalSourceId(sync.sourceId)) {
- log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
+ // too noisy
+ //log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
}
else {
await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
@@ -127,7 +133,8 @@ async function pushSync(syncContext) {
const filteredSyncs = syncs.filter(sync => {
if (sync.sourceId === syncContext.sourceId) {
- log.info(`Skipping push #${sync.id} ${sync.entityName} ${sync.entityId} because it originates from sync target`);
+ // too noisy
+ //log.info(`Skipping push #${sync.id} ${sync.entityName} ${sync.entityId} because it originates from sync target`);
// this may set lastSyncedPush beyond what's actually sent (because of size limit)
// so this is applied to the database only if there's no actual update
@@ -168,6 +175,10 @@ async function pushSync(syncContext) {
}
}
+async function syncFinished(syncContext) {
+ await syncRequest(syncContext, 'POST', '/api/sync/finished');
+}
+
async function checkContentHash(syncContext) {
const resp = await syncRequest(syncContext, 'GET', '/api/sync/check');
@@ -210,7 +221,7 @@ async function syncRequest(syncContext, method, uri, body) {
return await rp(options);
}
catch (e) {
- throw new Error(`Request to ${method} ${fullUri} failed, inner exception: ${e.stack}`);
+ throw new Error(`Request to ${method} ${fullUri} failed, error: ${e.message}`);
}
}
diff --git a/src/services/sync_mutex.js b/src/services/sync_mutex.js
index b68937acc..89f9e6ded 100644
--- a/src/services/sync_mutex.js
+++ b/src/services/sync_mutex.js
@@ -10,12 +10,11 @@ async function doExclusively(func) {
const releaseMutex = await instance.acquire();
try {
- await func();
+ return await func();
}
finally {
releaseMutex();
}
-
}
module.exports = {
diff --git a/src/services/sync_setup.js b/src/services/sync_setup.js
index a3b70bd16..a2dfc8099 100644
--- a/src/services/sync_setup.js
+++ b/src/services/sync_setup.js
@@ -1,6 +1,5 @@
"use strict";
-const config = require('./config');
const optionService = require('./options');
module.exports = {
diff --git a/src/services/sync_table.js b/src/services/sync_table.js
index ee5636740..c6ce2b72f 100644
--- a/src/services/sync_table.js
+++ b/src/services/sync_table.js
@@ -74,10 +74,11 @@ async function cleanupSyncRowsForMissingEntities(entityName, entityKey) {
AND sync.entityId NOT IN (SELECT ${entityKey} FROM ${entityName})`);
}
-async function fillSyncRows(entityName, entityKey) {
+async function fillSyncRows(entityName, entityKey, condition = '') {
await cleanupSyncRowsForMissingEntities(entityName, entityKey);
- const entityIds = await sql.getColumn(`SELECT ${entityKey} FROM ${entityName}`);
+ const entityIds = await sql.getColumn(`SELECT ${entityKey} FROM ${entityName}`
+ + (condition ? ` WHERE ${condition}` : ''));
for (const entityId of entityIds) {
const existingRows = await sql.getValue("SELECT COUNT(id) FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
@@ -107,6 +108,7 @@ async function fillAllSyncRows() {
await fillSyncRows("note_images", "noteImageId");
await fillSyncRows("labels", "labelId");
await fillSyncRows("api_tokens", "apiTokenId");
+ await fillSyncRows("options", "name", 'isSynced = 1');
}
module.exports = {
diff --git a/src/services/sync_update.js b/src/services/sync_update.js
index 8c0abd9e1..e00d992b1 100644
--- a/src/services/sync_update.js
+++ b/src/services/sync_update.js
@@ -109,7 +109,7 @@ async function updateNoteReordering(entityId, entity, sourceId) {
async function updateOptions(entity, sourceId) {
const orig = await sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [entity.name]);
- if (!orig.isSynced) {
+ if (orig && !orig.isSynced) {
return;
}
diff --git a/src/views/index.ejs b/src/views/index.ejs
index 4a5669d84..ac2840176 100644
--- a/src/views/index.ejs
+++ b/src/views/index.ejs
@@ -433,11 +433,11 @@
-
Copy document to the server instance
+
Sync document to the server instance
-
This is used when you're setting up server instance of Trilium. After the installation the databa
+
This is used when you want to sync your local document to the server instance configured above. This is a one time action after which the documents are synced automatically and transparently.