From c3d776ae13c6d9422b998d5a2ce61b40ff1b5f6e Mon Sep 17 00:00:00 2001 From: azivner Date: Thu, 2 Nov 2017 20:48:02 -0400 Subject: [PATCH] sync of options --- app.js | 10 ++++----- routes/api/login.js | 4 ++-- routes/api/migration.js | 4 ++-- routes/api/notes.js | 3 ++- routes/api/settings.js | 5 +++-- routes/api/status.js | 3 ++- routes/api/sync.js | 18 ++++++++++++++++ routes/api/tree.js | 11 +++++----- routes/login.js | 6 +++--- services/backup.js | 6 +++--- services/change_password.js | 9 ++++---- services/migration.js | 7 +++--- services/my_scrypt.js | 6 +++--- services/options.js | 41 +++++++++++++++++++++++++++++++++++ services/sql.js | 25 ++++++--------------- services/sync.js | 43 +++++++++++++++++++++++++++++++------ 16 files changed, 142 insertions(+), 59 deletions(-) create mode 100644 services/options.js diff --git a/app.js b/app.js index 2e1194221..3f895fd25 100644 --- a/app.js +++ b/app.js @@ -7,7 +7,7 @@ const helmet = require('helmet'); const session = require('express-session'); const FileStore = require('session-file-store')(session); const os = require('os'); -const sql = require('./services/sql'); +const options = require('./services/options'); const log = require('./services/log'); const utils = require('./services/utils'); @@ -37,12 +37,12 @@ const db = require('sqlite'); const config = require('./services/config'); db.open(dataDir.DOCUMENT_PATH, { Promise }).then(async () => { - if (!await sql.getOption('document_id')) { - await sql.setOption('document_id', utils.randomString(32)); + if (!await options.getOption('document_id')) { + await options.setOption('document_id', utils.randomString(32)); } - if (!await sql.getOption('document_secret')) { - await sql.setOption('document_secret', utils.randomSecureToken(32)); + if (!await options.getOption('document_secret')) { + await options.setOption('document_secret', utils.randomSecureToken(32)); } }); diff --git a/routes/api/login.js b/routes/api/login.js index 5373f80ce..5945115a3 100644 --- a/routes/api/login.js +++ b/routes/api/login.js @@ -2,7 +2,7 @@ const express = require('express'); const router = express.Router(); -const sql = require('../../services/sql'); +const options = require('../../services/options'); const utils = require('../../services/utils'); const migration = require('../../services/migration'); const SOURCE_ID = require('../../services/source_id'); @@ -24,7 +24,7 @@ router.post('', async (req, res, next) => { res.send({ message: 'Non-matching db versions, local is version ' + migration.APP_DB_VERSION }); } - const documentSecret = await sql.getOption('document_secret'); + const documentSecret = await options.getOption('document_secret'); const expectedHash = utils.hmac(documentSecret, timestamp); const givenHash = req.body.hash; diff --git a/routes/api/migration.js b/routes/api/migration.js index 058b6ba64..8894cb907 100644 --- a/routes/api/migration.js +++ b/routes/api/migration.js @@ -3,12 +3,12 @@ const express = require('express'); const router = express.Router(); const auth = require('../../services/auth'); -const sql = require('../../services/sql'); +const options = require('../../services/options'); const migration = require('../../services/migration'); router.get('', auth.checkApiAuthWithoutMigration, async (req, res, next) => { res.send({ - 'db_version': parseInt(await sql.getOption('db_version')), + 'db_version': parseInt(await options.getOption('db_version')), 'app_db_version': migration.APP_DB_VERSION }); }); diff --git a/routes/api/notes.js b/routes/api/notes.js index 9035e53a3..3a3a7b34b 100644 --- a/routes/api/notes.js +++ b/routes/api/notes.js @@ -3,6 +3,7 @@ const express = require('express'); const router = express.Router(); const sql = require('../../services/sql'); +const options = require('../../services/options'); const utils = require('../../services/utils'); const audit_category = require('../../services/audit_category'); const auth = require('../../services/auth'); @@ -38,7 +39,7 @@ router.put('/:noteId', async (req, res, next) => { const now = utils.nowTimestamp(); - const historySnapshotTimeInterval = parseInt(await sql.getOption('history_snapshot_time_interval')); + const historySnapshotTimeInterval = parseInt(await options.getOption('history_snapshot_time_interval')); const historyCutoff = now - historySnapshotTimeInterval; diff --git a/routes/api/settings.js b/routes/api/settings.js index a1cd6d610..05fdf5e54 100644 --- a/routes/api/settings.js +++ b/routes/api/settings.js @@ -3,6 +3,7 @@ const express = require('express'); const router = express.Router(); const sql = require('../../services/sql'); +const options = require('../../services/options'); const audit_category = require('../../services/audit_category'); const auth = require('../../services/auth'); @@ -25,12 +26,12 @@ router.post('/', async (req, res, next) => { const body = req.body; if (ALLOWED_OPTIONS.includes(body['name'])) { - const optionName = await sql.getOption(body['name']); + const optionName = await options.getOption(body['name']); await sql.doInTransaction(async () => { await sql.addAudit(audit_category.SETTINGS, req, null, optionName, body['value'], body['name']); - await sql.setOption(body['name'], body['value']); + await options.setOption(body['name'], body['value']); }); res.send({}); diff --git a/routes/api/status.js b/routes/api/status.js index 7108c32a8..b84983f78 100644 --- a/routes/api/status.js +++ b/routes/api/status.js @@ -3,6 +3,7 @@ const express = require('express'); const router = express.Router(); const sql = require('../../services/sql'); +const options = require('../../services/options'); const auth = require('../../services/auth'); const sync = require('../../services/sync'); const audit_category = require('../../services/audit_category'); @@ -31,7 +32,7 @@ router.post('', auth.checkApiAuth, async (req, res, next) => { let changesToPushCount = 0; if (sync.isSyncSetup) { - const lastSyncedPush = await sql.getOption('last_synced_push'); + const lastSyncedPush = await options.getOption('last_synced_push'); changesToPushCount = await sql.getSingleValue("SELECT COUNT(*) FROM sync WHERE id > ?", [lastSyncedPush]); } diff --git a/routes/api/sync.js b/routes/api/sync.js index cea4e9665..c7665f6b0 100644 --- a/routes/api/sync.js +++ b/routes/api/sync.js @@ -5,6 +5,7 @@ const router = express.Router(); const auth = require('../../services/auth'); const sync = require('../../services/sync'); const sql = require('../../services/sql'); +const options = require('../../services/options'); router.post('/now', auth.checkApiAuth, async (req, res, next) => { const log = await sync.sync(); @@ -43,6 +44,17 @@ router.get('/notes_history/:noteHistoryId', auth.checkApiAuth, async (req, res, res.send(await sql.getSingleResult("SELECT * FROM notes_history WHERE note_history_id = ?", [noteHistoryId])); }); +router.get('/options/:optName', auth.checkApiAuth, async (req, res, next) => { + const optName = req.params.optName; + + if (!options.SYNCED_OPTIONS.includes(optName)) { + res.send("This option can't be synced."); + } + else { + res.send(await sql.getSingleResult("SELECT * FROM options WHERE opt_name = ?", [optName])); + } +}); + router.put('/notes', auth.checkApiAuth, async (req, res, next) => { await sync.updateNote(req.body.entity, req.body.links, req.body.sourceId); @@ -61,4 +73,10 @@ router.put('/notes_history', auth.checkApiAuth, async (req, res, next) => { res.send({}); }); +router.put('/options', auth.checkApiAuth, async (req, res, next) => { + await sync.updateOptions(req.body.entity, req.body.sourceId); + + res.send({}); +}); + module.exports = router; \ No newline at end of file diff --git a/routes/api/tree.js b/routes/api/tree.js index 469bbe88d..664dc9445 100644 --- a/routes/api/tree.js +++ b/routes/api/tree.js @@ -3,6 +3,7 @@ const express = require('express'); const router = express.Router(); const sql = require('../../services/sql'); +const options = require('../../services/options'); const utils = require('../../services/utils'); const auth = require('../../services/auth'); @@ -43,11 +44,11 @@ router.get('/', auth.checkApiAuth, async (req, res, next) => { res.send({ 'notes': root_notes, - 'start_note_id': await sql.getOption('start_node'), - 'password_verification_salt': await sql.getOption('password_verification_salt'), - 'password_derived_key_salt': await sql.getOption('password_derived_key_salt'), - 'encrypted_data_key': await sql.getOption('encrypted_data_key'), - 'encryption_session_timeout': await sql.getOption('encryption_session_timeout'), + 'start_note_id': await options.getOption('start_node'), + 'password_verification_salt': await options.getOption('password_verification_salt'), + 'password_derived_key_salt': await options.getOption('password_derived_key_salt'), + 'encrypted_data_key': await options.getOption('encrypted_data_key'), + 'encryption_session_timeout': await options.getOption('encryption_session_timeout'), 'browser_id': utils.randomString(12), 'tree_load_time': utils.nowTimestamp() }); diff --git a/routes/login.js b/routes/login.js index e1ca605d0..e6c63c2f0 100644 --- a/routes/login.js +++ b/routes/login.js @@ -3,7 +3,7 @@ const express = require('express'); const router = express.Router(); const utils = require('../services/utils'); -const sql = require('../services/sql'); +const options = require('../services/options'); const my_scrypt = require('../services/my_scrypt'); router.get('', (req, res, next) => { @@ -11,7 +11,7 @@ router.get('', (req, res, next) => { }); router.post('', async (req, res, next) => { - const userName = await sql.getOption('username'); + const userName = await options.getOption('username'); const guessedPassword = req.body.password; @@ -36,7 +36,7 @@ router.post('', async (req, res, next) => { async function verifyPassword(guessed_password) { - const hashed_password = utils.fromBase64(await sql.getOption('password_verification_hash')); + const hashed_password = utils.fromBase64(await options.getOption('password_verification_hash')); const guess_hashed = await my_scrypt.getVerificationHash(guessed_password); diff --git a/services/backup.js b/services/backup.js index 3bacfa510..9a30896cc 100644 --- a/services/backup.js +++ b/services/backup.js @@ -1,14 +1,14 @@ "use strict"; const utils = require('./utils'); -const sql = require('./sql'); +const options = require('./options'); const fs = require('fs-extra'); const dataDir = require('./data_dir'); const log = require('./log'); async function regularBackup() { const now = utils.nowTimestamp(); - const last_backup_date = parseInt(await sql.getOption('last_backup_date')); + const last_backup_date = parseInt(await options.getOption('last_backup_date')); if (now - last_backup_date > 43200) { await backupNow(); @@ -28,7 +28,7 @@ async function backupNow() { log.info("Created backup at " + backupFile); - await sql.setOption('last_backup_date', now); + await options.setOption('last_backup_date', now); } async function cleanupOldBackups() { diff --git a/services/change_password.js b/services/change_password.js index 46c21bfd1..2aab4f909 100644 --- a/services/change_password.js +++ b/services/change_password.js @@ -1,6 +1,7 @@ "use strict"; const sql = require('./sql'); +const options = require('./options'); const my_scrypt = require('./my_scrypt'); const utils = require('./utils'); const audit_category = require('./audit_category'); @@ -10,7 +11,7 @@ const aesjs = require('./aes'); async function changePassword(currentPassword, newPassword, req = null) { const current_password_hash = utils.toBase64(await my_scrypt.getVerificationHash(currentPassword)); - if (current_password_hash !== await sql.getOption('password_verification_hash')) { + if (current_password_hash !== await options.getOption('password_verification_hash')) { return { 'success': false, 'message': "Given current password doesn't match hash" @@ -49,16 +50,16 @@ async function changePassword(currentPassword, newPassword, req = null) { return new aesjs.ModeOfOperation.ctr(key, new aesjs.Counter(5)); } - const encryptedDataKey = await sql.getOption('encrypted_data_key'); + const encryptedDataKey = await options.getOption('encrypted_data_key'); const decryptedDataKey = decrypt(encryptedDataKey); const newEncryptedDataKey = encrypt(decryptedDataKey); await sql.doInTransaction(async () => { - await sql.setOption('encrypted_data_key', newEncryptedDataKey); + await options.setOption('encrypted_data_key', newEncryptedDataKey); - await sql.setOption('password_verification_hash', newPasswordVerificationKey); + await options.setOption('password_verification_hash', newPasswordVerificationKey); await sql.addAudit(audit_category.CHANGE_PASSWORD, req); }); diff --git a/services/migration.js b/services/migration.js index 16c7c5b08..1d8a71a84 100644 --- a/services/migration.js +++ b/services/migration.js @@ -1,5 +1,6 @@ const backup = require('./backup'); const sql = require('./sql'); +const options = require('./options'); const fs = require('fs-extra'); const log = require('./log'); @@ -12,7 +13,7 @@ async function migrate() { // backup before attempting migration await backup.backupNow(); - const currentDbVersion = parseInt(await sql.getOption('db_version')); + const currentDbVersion = parseInt(await options.getOption('db_version')); fs.readdirSync(MIGRATIONS_DIR).forEach(file => { const match = file.match(/([0-9]{4})__([a-zA-Z0-9_ ]+)\.sql/); @@ -45,7 +46,7 @@ async function migrate() { await sql.doInTransaction(async () => { await sql.executeScript(migrationSql); - await sql.setOption("db_version", mig.dbVersion); + await options.setOption("db_version", mig.dbVersion); }); log.info("Migration to version " + mig.dbVersion + " has been successful."); @@ -66,7 +67,7 @@ async function migrate() { } async function isDbUpToDate() { - const dbVersion = parseInt(await sql.getOption('db_version')); + const dbVersion = parseInt(await options.getOption('db_version')); return dbVersion >= APP_DB_VERSION; } diff --git a/services/my_scrypt.js b/services/my_scrypt.js index 83799f534..96de4e252 100644 --- a/services/my_scrypt.js +++ b/services/my_scrypt.js @@ -1,16 +1,16 @@ "use strict"; -const sql = require('./sql'); +const options = require('./options'); const scrypt = require('scrypt'); async function getVerificationHash(password) { - const salt = await sql.getOption('password_verification_salt'); + const salt = await options.getOption('password_verification_salt'); return getScryptHash(password, salt); } async function getPasswordDerivedKey(password) { - const salt = await sql.getOption('password_derived_key_salt'); + const salt = await options.getOption('password_derived_key_salt'); return getScryptHash(password, salt); } diff --git a/services/options.js b/services/options.js new file mode 100644 index 000000000..81ca6fda7 --- /dev/null +++ b/services/options.js @@ -0,0 +1,41 @@ +const sql = require('./sql'); +const utils = require('./utils'); + +const SYNCED_OPTIONS = [ 'username', 'password_verification_hash', 'encrypted_data_key', 'encryption_session_timeout', + 'history_snapshot_time_interval' ]; + +async function getOption(optName) { + const row = await sql.getSingleResultOrNull("SELECT opt_value FROM options WHERE opt_name = ?", [optName]); + + if (!row) { + throw new Error("Option " + optName + " doesn't exist"); + } + + return row['opt_value']; +} + +async function setOptionInTransaction(optName, optValue) { + await sql.doInTransaction(async () => setOption(optName, optValue)); +} + +async function setOption(optName, optValue) { + if (SYNCED_OPTIONS.includes(optName)) { + await sql.addOptionsSync(optName); + } + + await setOptionNoSync(optName, optValue); +} + +async function setOptionNoSync(optName, optValue) { + const now = utils.nowTimestamp(); + + await sql.execute("UPDATE options SET opt_value = ?, date_modified = ? WHERE opt_name = ?", [optValue, now, optName]); +} + +module.exports = { + getOption, + setOption, + setOptionNoSync, + setOptionInTransaction, + SYNCED_OPTIONS +}; \ No newline at end of file diff --git a/services/sql.js b/services/sql.js index 854d290f5..db2a4a4f3 100644 --- a/services/sql.js +++ b/services/sql.js @@ -38,22 +38,6 @@ async function rollback() { return await db.run("ROLLBACK"); } -async function getOption(optName) { - const row = await getSingleResultOrNull("SELECT opt_value FROM options WHERE opt_name = ?", [optName]); - - if (!row) { - throw new Error("Option " + optName + " doesn't exist"); - } - - return row['opt_value']; -} - -async function setOption(optName, optValue) { - const now = utils.nowTimestamp(); - - await execute("UPDATE options SET opt_value = ?, date_modified = ? WHERE opt_name = ?", [optValue, now, optName]); -} - async function getSingleResult(query, params = []) { return await wrap(async () => db.get(query, ...params)); } @@ -143,6 +127,10 @@ async function addNoteHistorySync(noteHistoryId, sourceId) { await addEntitySync("notes_history", noteHistoryId, sourceId); } +async function addOptionsSync(optName, sourceId) { + await addEntitySync("options", optName, sourceId); +} + async function addEntitySync(entityName, entityId, sourceId) { await replace("sync", { entity_name: entityName, @@ -195,8 +183,6 @@ module.exports = { getFlattenedResults, execute, executeScript, - getOption, - setOption, addAudit, addSyncAudit, deleteRecentAudits, @@ -204,5 +190,6 @@ module.exports = { doInTransaction, addNoteSync, addNoteTreeSync, - addNoteHistorySync + addNoteHistorySync, + addOptionsSync }; \ No newline at end of file diff --git a/services/sync.js b/services/sync.js index a1cad9129..f571e99ca 100644 --- a/services/sync.js +++ b/services/sync.js @@ -3,6 +3,7 @@ const log = require('./log'); const rp = require('request-promise'); const sql = require('./sql'); +const options = require('./options'); const migration = require('./migration'); const utils = require('./utils'); const config = require('./config'); @@ -16,7 +17,7 @@ const isSyncSetup = !!SYNC_SERVER; let syncInProgress = false; async function pullSync(syncContext, syncLog) { - const lastSyncedPull = parseInt(await sql.getOption('last_synced_pull')); + const lastSyncedPull = parseInt(await options.getOption('last_synced_pull')); let syncRows; @@ -59,11 +60,14 @@ async function pullSync(syncContext, syncLog) { else if (sync.entity_name === 'notes_history') { await updateNoteHistory(resp, syncContext.sourceId, syncLog); } + else if (sync.entity_name === 'options') { + await updateOptions(resp, syncContext.sourceId, syncLog); + } else { logSyncError("Unrecognized entity type " + sync.entity_name, e, syncLog); } - await sql.setOption('last_synced_pull', sync.id); + await options.setOption('last_synced_pull', sync.id); } logSync("Finished pull"); @@ -106,6 +110,9 @@ async function readAndPushEntity(sync, syncLog, syncContext) { else if (sync.entity_name === 'notes_history') { entity = await sql.getSingleResult('SELECT * FROM notes_history WHERE note_history_id = ?', [sync.entity_id]); } + else if (sync.entity_name === 'options') { + entity = await sql.getSingleResult('SELECT * FROM options WHERE opt_name = ?', [sync.entity_id]); + } else { logSyncError("Unrecognized entity type " + sync.entity_name, null, syncLog); } @@ -116,7 +123,7 @@ async function readAndPushEntity(sync, syncLog, syncContext) { } async function pushSync(syncContext, syncLog) { - let lastSyncedPush = parseInt(await sql.getOption('last_synced_push')); + let lastSyncedPush = parseInt(await options.getOption('last_synced_push')); while (true) { const sync = await sql.getSingleResultOrNull('SELECT * FROM sync WHERE id > ? LIMIT 1', [lastSyncedPush]); @@ -129,7 +136,7 @@ async function pushSync(syncContext, syncLog) { break; } - if (sync.sourceId === syncContext.source_id) { + if (sync.source_id === syncContext.sourceId) { logSync("Skipping sync " + sync.entity_name + " " + sync.entity_id + " because it originates from sync target", syncLog); } else { @@ -138,14 +145,14 @@ async function pushSync(syncContext, syncLog) { lastSyncedPush = sync.id; - await sql.setOption('last_synced_push', lastSyncedPush); + await options.setOption('last_synced_push', lastSyncedPush); } } async function login(syncLog) { const timestamp = utils.nowTimestamp(); - const documentSecret = await sql.getOption('document_secret'); + const documentSecret = await options.getOption('document_secret'); const hash = utils.hmac(documentSecret, timestamp); const cookieJar = rp.jar(); @@ -194,6 +201,8 @@ async function sync() { const syncContext = await login(syncLog); + await pushSync(syncContext, syncLog); + await pullSync(syncContext, syncLog); await pushSync(syncContext, syncLog); @@ -297,6 +306,27 @@ async function updateNoteHistory(entity, sourceId, syncLog) { } } +async function updateOptions(entity, sourceId, syncLog) { + if (!options.SYNCED_OPTIONS.includes(entity.opt_name)) { + return; + } + + const orig = await sql.getSingleResultOrNull("select * from options where opt_name = ?", [entity.opt_name]); + + if (orig === null || orig.date_modified < entity.date_modified) { + await sql.doInTransaction(async () => { + await sql.replace('options', entity); + + await sql.addOptionsSync(entity.opt_name, sourceId); + }); + + logSync("Update/sync options " + entity.opt_name, syncLog); + } + else { + logSync("Sync conflict in options for " + entity.opt_name + ", date_modified=" + entity.date_modified, syncLog); + } +} + if (SYNC_SERVER) { log.info("Setting up sync"); @@ -314,5 +344,6 @@ module.exports = { updateNote, updateNoteTree, updateNoteHistory, + updateOptions, isSyncSetup }; \ No newline at end of file