diff --git a/app.js b/app.js index f743838b3..88d16ca36 100644 --- a/app.js +++ b/app.js @@ -9,36 +9,8 @@ const helmet = require('helmet'); const session = require('express-session'); const FileStore = require('session-file-store')(session); const os = require('os'); -const options = require('./services/options'); -const utils = require('./services/utils'); -const sql = require('./services/sql'); -const dataDir = require('./services/data_dir'); const sessionSecret = require('./services/session_secret'); - -const db = require('sqlite'); - -db.open(dataDir.DOCUMENT_PATH, { Promise }).then(async () => { - const tableResults = await sql.getResults("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'"); - - if (tableResults.length !== 1) { - console.log("No connection to initialized DB."); - process.exit(1); - } - - if (!await options.getOption('document_id')) { - await options.setOption('document_id', utils.randomString(32)); - } - - if (!await options.getOption('document_secret')) { - await options.setOption('document_secret', utils.randomSecureToken(32)); - } -}) - .catch(e => { - console.log("Error connecting to DB.", e); - process.exit(1); - }); - const app = express(); // view engine setup diff --git a/migrations/0034__source_ids.sql b/migrations/0034__source_ids.sql new file mode 100644 index 000000000..c48513564 --- /dev/null +++ b/migrations/0034__source_ids.sql @@ -0,0 +1,5 @@ +CREATE TABLE `source_ids` ( + `source_id` TEXT NOT NULL, + `date_created` INTEGER NOT NULL, + PRIMARY KEY(`source_id`) +); \ No newline at end of file diff --git a/routes/api/login.js b/routes/api/login.js index 7cdf43785..dd7df2064 100644 --- a/routes/api/login.js +++ b/routes/api/login.js @@ -5,7 +5,7 @@ const router = express.Router(); const options = require('../../services/options'); const utils = require('../../services/utils'); const migration = require('../../services/migration'); -const SOURCE_ID = require('../../services/source_id'); +const source_id = require('../../services/source_id'); const auth = require('../../services/auth'); const password_encryption = require('../../services/password_encryption'); const protected_session = require('../../services/protected_session'); @@ -40,7 +40,7 @@ router.post('/sync', async (req, res, next) => { req.session.loggedIn = true; res.send({ - sourceId: SOURCE_ID + sourceId: source_id.currentSourceId }); }); diff --git a/routes/api/note_history.js b/routes/api/note_history.js index 0dff4005e..cb0b0ed67 100644 --- a/routes/api/note_history.js +++ b/routes/api/note_history.js @@ -6,6 +6,7 @@ const sql = require('../../services/sql'); const auth = require('../../services/auth'); const data_encryption = require('../../services/data_encryption'); const protected_session = require('../../services/protected_session'); +const sync_table = require('../../services/sync_table'); router.get('/:noteId', auth.checkApiAuth, async (req, res, next) => { const noteId = req.params.noteId; @@ -27,7 +28,7 @@ router.put('', auth.checkApiAuth, async (req, res, next) => { await sql.doInTransaction(async () => { await sql.replace("notes_history", req.body); - await sql.addNoteHistorySync(req.body.note_history_id); + await sync_table.addNoteHistorySync(req.body.note_history_id); }); res.send(); diff --git a/routes/api/notes_move.js b/routes/api/notes_move.js index fa2c15d36..1ce454394 100644 --- a/routes/api/notes_move.js +++ b/routes/api/notes_move.js @@ -6,6 +6,7 @@ const sql = require('../../services/sql'); const utils = require('../../services/utils'); const audit_category = require('../../services/audit_category'); const auth = require('../../services/auth'); +const sync_table = require('../../services/sync_table'); router.put('/:noteId/moveTo/:parentId', auth.checkApiAuth, async (req, res, next) => { let noteId = req.params.noteId; @@ -26,7 +27,7 @@ router.put('/:noteId/moveTo/:parentId', auth.checkApiAuth, async (req, res, next await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?", [parentId, newNotePos, now, noteId]); - await sql.addNoteTreeSync(noteId); + await sync_table.addNoteTreeSync(noteId); await sql.addAudit(audit_category.CHANGE_PARENT, utils.browserId(req), noteId, null, parentId); }); @@ -50,8 +51,8 @@ router.put('/:noteId/moveBefore/:beforeNoteId', async (req, res, next) => { await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?", [beforeNote['note_pid'], beforeNote['note_pos'], now, noteId]); - await sql.addNoteTreeSync(noteId); - await sql.addNoteReorderingSync(beforeNote['note_pid']); + await sync_table.addNoteTreeSync(noteId); + await sync_table.addNoteReorderingSync(beforeNote['note_pid']); await sql.addAudit(audit_category.CHANGE_POSITION, utils.browserId(req), beforeNote['note_pid']); }); } @@ -76,8 +77,8 @@ router.put('/:noteId/moveAfter/:afterNoteId', async (req, res, next) => { await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?", [afterNote['note_pid'], afterNote['note_pos'] + 1, now, noteId]); - await sql.addNoteTreeSync(noteId); - await sql.addNoteReorderingSync(afterNote['note_pid']); + await sync_table.addNoteTreeSync(noteId); + await sync_table.addNoteReorderingSync(afterNote['note_pid']); await sql.addAudit(audit_category.CHANGE_POSITION, utils.browserId(req), afterNote['note_pid']); }); } diff --git a/routes/api/recent_notes.js b/routes/api/recent_notes.js index 8ea6dec7d..04d3d7407 100644 --- a/routes/api/recent_notes.js +++ b/routes/api/recent_notes.js @@ -5,6 +5,7 @@ const router = express.Router(); const sql = require('../../services/sql'); const auth = require('../../services/auth'); const utils = require('../../services/utils'); +const sync_table = require('../../services/sync_table'); router.get('', auth.checkApiAuth, async (req, res, next) => { res.send(await getRecentNotes()); @@ -17,7 +18,7 @@ router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => { is_deleted: 0 }); - await sql.addRecentNoteSync(req.params.noteId); + await sync_table.addRecentNoteSync(req.params.noteId); res.send(await getRecentNotes()); }); @@ -25,7 +26,7 @@ router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => { router.delete('/:noteId', auth.checkApiAuth, async (req, res, next) => { await sql.execute('UPDATE recent_notes SET is_deleted = 1 WHERE note_id = ?', [req.params.noteId]); - await sql.addRecentNoteSync(req.params.noteId); + await sync_table.addRecentNoteSync(req.params.noteId); res.send(await getRecentNotes()); }); diff --git a/routes/api/sync.js b/routes/api/sync.js index 6ab0be33e..1354fdfd8 100644 --- a/routes/api/sync.js +++ b/routes/api/sync.js @@ -14,9 +14,8 @@ router.post('/now', auth.checkApiAuth, async (req, res, next) => { router.get('/changed', auth.checkApiAuth, async (req, res, next) => { const lastSyncId = parseInt(req.query.lastSyncId); - const sourceId = req.query.sourceId; - res.send(await sql.getResults("SELECT * FROM sync WHERE id > ? AND source_id != ?", [lastSyncId, sourceId])); + res.send(await sql.getResults("SELECT * FROM sync WHERE id > ?", [lastSyncId])); }); router.get('/notes/:noteId', auth.checkApiAuth, async (req, res, next) => { diff --git a/services/migration.js b/services/migration.js index a95120254..64d2cd25c 100644 --- a/services/migration.js +++ b/services/migration.js @@ -4,7 +4,7 @@ const options = require('./options'); const fs = require('fs-extra'); const log = require('./log'); -const APP_DB_VERSION = 33; +const APP_DB_VERSION = 34; const MIGRATIONS_DIR = "migrations"; async function migrate() { diff --git a/services/notes.js b/services/notes.js index 04627b60a..6faaddd09 100644 --- a/services/notes.js +++ b/services/notes.js @@ -4,6 +4,7 @@ const utils = require('./utils'); const notes = require('./notes'); const audit_category = require('./audit_category'); const data_encryption = require('./data_encryption'); +const sync_table = require('./sync_table'); async function createNewNote(parentNoteId, note, browserId) { const noteId = utils.newNoteId(); @@ -38,8 +39,8 @@ async function createNewNote(parentNoteId, note, browserId) { await sql.doInTransaction(async () => { await sql.addAudit(audit_category.CREATE_NOTE, browserId, noteId); - await sql.addNoteTreeSync(noteId); - await sql.addNoteSync(noteId); + await sync_table.addNoteTreeSync(noteId); + await sync_table.addNoteSync(noteId); const now = utils.nowTimestamp(); @@ -105,7 +106,7 @@ async function protectNote(note, dataKey, protect) { await sql.execute("UPDATE notes SET note_title = ?, note_text = ?, is_protected = ? WHERE note_id = ?", [note.note_title, note.note_text, note.is_protected, note.note_id]); - await sql.addNoteSync(note.note_id); + await sync_table.addNoteSync(note.note_id); } await protectNoteHistory(note.note_id, dataKey, protect); @@ -129,7 +130,7 @@ async function protectNoteHistory(noteId, dataKey, protect) { await sql.execute("UPDATE notes_history SET note_title = ?, note_text = ?, is_protected = ? WHERE note_history_id = ?", [history.note_title, history.note_text, history.is_protected, history.note_history_id]); - await sql.addNoteHistorySync(history.note_history_id); + await sync_table.addNoteHistorySync(history.note_history_id); } } @@ -166,7 +167,7 @@ async function updateNote(noteId, newNote, ctx) { now ]); - await sql.addNoteHistorySync(newNoteHistoryId); + await sync_table.addNoteHistorySync(newNoteHistoryId); } await protectNoteHistory(noteId, ctx.getDataKey(), newNote.detail.is_protected); @@ -194,8 +195,8 @@ async function updateNote(noteId, newNote, ctx) { await sql.insert("links", link); } - await sql.addNoteTreeSync(noteId); - await sql.addNoteSync(noteId); + await sync_table.addNoteTreeSync(noteId); + await sync_table.addNoteSync(noteId); }); } @@ -232,8 +233,8 @@ async function deleteNote(noteId, browserId) { await sql.execute("update notes_tree set is_deleted = 1, date_modified = ? where note_id = ?", [now, noteId]); await sql.execute("update notes set is_deleted = 1, date_modified = ? where note_id = ?", [now, noteId]); - await sql.addNoteTreeSync(noteId); - await sql.addNoteSync(noteId); + await sync_table.addNoteTreeSync(noteId); + await sync_table.addNoteSync(noteId); await sql.addAudit(audit_category.DELETE_NOTE, browserId, noteId); } diff --git a/services/options.js b/services/options.js index 49004ed4d..51fd1cff5 100644 --- a/services/options.js +++ b/services/options.js @@ -1,5 +1,6 @@ const sql = require('./sql'); const utils = require('./utils'); +const sync_table = require('./sync_table'); const SYNCED_OPTIONS = [ 'username', 'password_verification_hash', 'encrypted_data_key', 'protected_session_timeout', 'history_snapshot_time_interval' ]; @@ -20,7 +21,7 @@ async function setOptionInTransaction(optName, optValue) { async function setOption(optName, optValue) { if (SYNCED_OPTIONS.includes(optName)) { - await sql.addOptionsSync(optName); + await sync_table.addOptionsSync(optName); } await setOptionNoSync(optName, optValue); @@ -32,6 +33,16 @@ async function setOptionNoSync(optName, optValue) { await sql.execute("UPDATE options SET opt_value = ?, date_modified = ? WHERE opt_name = ?", [optValue, now, optName]); } +sql.dbReady.then(async () => { + if (!await getOption('document_id')) { + await setOption('document_id', utils.randomSecureToken(16)); + } + + if (!await getOption('document_secret')) { + await setOption('document_secret', utils.randomSecureToken(16)); + } +}); + module.exports = { getOption, setOption, diff --git a/services/source_id.js b/services/source_id.js index 78efc684c..34410d383 100644 --- a/services/source_id.js +++ b/services/source_id.js @@ -1,8 +1,27 @@ const utils = require('./utils'); const log = require('./log'); +const sql = require('./sql'); -const sourceId = utils.randomString(16); +const currentSourceId = utils.randomString(12); -log.info("Using sourceId=" + sourceId); +log.info("Using sourceId=" + currentSourceId); -module.exports = sourceId; \ No newline at end of file +let allSourceIds = []; + +sql.dbReady.then(async () => { + sql.insert("source_ids", { + source_id: currentSourceId, + date_created: utils.nowTimestamp() + }); + + allSourceIds = await sql.getFlattenedResults("source_id", "SELECT source_id FROM source_ids"); +}); + +function isLocalSourceId(srcId) { + return allSourceIds.includes(srcId); +} + +module.exports = { + currentSourceId, + isLocalSourceId +}; \ No newline at end of file diff --git a/services/sql.js b/services/sql.js index 4c3454cb2..305b84f53 100644 --- a/services/sql.js +++ b/services/sql.js @@ -3,7 +3,7 @@ const db = require('sqlite'); const utils = require('./utils'); const log = require('./log'); -const SOURCE_ID = require('./source_id'); +const dataDir = require('./data_dir'); async function insert(table_name, rec, replace = false) { const keys = Object.keys(rec); @@ -116,39 +116,6 @@ async function deleteRecentAudits(category, browserId, noteId) { [category, browserId, noteId, deleteCutoff]) } -async function addNoteSync(noteId, sourceId) { - await addEntitySync("notes", noteId, sourceId) -} - -async function addNoteTreeSync(noteId, sourceId) { - await addEntitySync("notes_tree", noteId, sourceId) -} - -async function addNoteReorderingSync(noteId, sourceId) { - await addEntitySync("notes_reordering", noteId, sourceId) -} - -async function addNoteHistorySync(noteHistoryId, sourceId) { - await addEntitySync("notes_history", noteHistoryId, sourceId); -} - -async function addOptionsSync(optName, sourceId) { - await addEntitySync("options", optName, sourceId); -} - -async function addRecentNoteSync(noteId, sourceId) { - await addEntitySync("recent_notes", noteId, sourceId); -} - -async function addEntitySync(entityName, entityId, sourceId) { - await replace("sync", { - entity_name: entityName, - entity_id: entityId, - sync_date: utils.nowTimestamp(), - source_id: sourceId || SOURCE_ID - }); -} - async function wrap(func) { const error = new Error(); @@ -182,7 +149,24 @@ async function doInTransaction(func) { } } +const dbReady = db.open(dataDir.DOCUMENT_PATH, { Promise }); + +dbReady + .then(async () => { + const tableResults = await getResults("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'"); + + if (tableResults.length !== 1) { + console.log("No connection to initialized DB."); + process.exit(1); + } + }) + .catch(e => { + console.log("Error connecting to DB.", e); + process.exit(1); + }); + module.exports = { + dbReady, insert, replace, getSingleValue, @@ -196,11 +180,5 @@ module.exports = { addAudit, deleteRecentAudits, remove, - doInTransaction, - addNoteSync, - addNoteTreeSync, - addNoteReorderingSync, - addNoteHistorySync, - addOptionsSync, - addRecentNoteSync + doInTransaction }; \ No newline at end of file diff --git a/services/sync.js b/services/sync.js index a55ff8ca5..4c54bef9e 100644 --- a/services/sync.js +++ b/services/sync.js @@ -7,7 +7,7 @@ const options = require('./options'); const migration = require('./migration'); const utils = require('./utils'); const config = require('./config'); -const SOURCE_ID = require('./source_id'); +const source_id = require('./source_id'); const notes = require('./notes'); const syncUpdate = require('./sync_update'); @@ -100,13 +100,19 @@ async function login() { async function pullSync(syncContext) { const lastSyncedPull = parseInt(await options.getOption('last_synced_pull')); - const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull + "&sourceId=" + SOURCE_ID; + const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull; const syncRows = await syncRequest(syncContext, 'GET', changesUri); log.info("Pulled " + syncRows.length + " changes from " + changesUri); for (const sync of syncRows) { + if (source_id.isLocalSourceId(sync.source_id)) { + log.info("Skipping " + sync.entity_name + " " + sync.entity_id + " because it has local source id."); + + continue; + } + const resp = await syncRequest(syncContext, 'GET', "/api/sync/" + sync.entity_name + "/" + sync.entity_id); if (sync.entity_name === 'notes') { @@ -204,7 +210,7 @@ async function readAndPushEntity(sync, syncContext) { async function sendEntity(syncContext, entity, entityName) { const payload = { - sourceId: SOURCE_ID, + sourceId: source_id.currentSourceId, entity: entity }; diff --git a/services/sync_table.js b/services/sync_table.js new file mode 100644 index 000000000..cfcbd3447 --- /dev/null +++ b/services/sync_table.js @@ -0,0 +1,45 @@ +const sql = require('./sql'); +const source_id = require('./source_id'); +const utils = require('./utils'); + +async function addNoteSync(noteId, sourceId) { + await addEntitySync("notes", noteId, sourceId) +} + +async function addNoteTreeSync(noteId, sourceId) { + await addEntitySync("notes_tree", noteId, sourceId) +} + +async function addNoteReorderingSync(noteId, sourceId) { + await addEntitySync("notes_reordering", noteId, sourceId) +} + +async function addNoteHistorySync(noteHistoryId, sourceId) { + await addEntitySync("notes_history", noteHistoryId, sourceId); +} + +async function addOptionsSync(optName, sourceId) { + await addEntitySync("options", optName, sourceId); +} + +async function addRecentNoteSync(noteId, sourceId) { + await addEntitySync("recent_notes", noteId, sourceId); +} + +async function addEntitySync(entityName, entityId, sourceId) { + await sql.replace("sync", { + entity_name: entityName, + entity_id: entityId, + sync_date: utils.nowTimestamp(), + source_id: sourceId || source_id.currentSourceId + }); +} + +module.exports = { + addNoteSync, + addNoteTreeSync, + addNoteReorderingSync, + addNoteHistorySync, + addOptionsSync, + addRecentNoteSync +}; \ No newline at end of file diff --git a/services/sync_update.js b/services/sync_update.js index bfbe1d8c4..fb849b899 100644 --- a/services/sync_update.js +++ b/services/sync_update.js @@ -5,6 +5,7 @@ const utils = require('./utils'); const audit_category = require('./audit_category'); const eventLog = require('./event_log'); const notes = require('./notes'); +const sync_table = require('./sync_table'); async function updateNote(entity, links, sourceId) { const origNote = await sql.getSingleResult("select * from notes where note_id = ?", [entity.note_id]); @@ -21,7 +22,7 @@ async function updateNote(entity, links, sourceId) { await sql.insert('link', link); } - await sql.addNoteSync(entity.note_id, sourceId); + await sync_table.addNoteSync(entity.note_id, sourceId); await notes.addNoteAudits(origNote, entity, sourceId); await eventLog.addNoteEvent(entity.note_id, "Synced note "); }); @@ -42,7 +43,7 @@ async function updateNoteTree(entity, sourceId) { await sql.replace('notes_tree', entity); - await sql.addNoteTreeSync(entity.note_id, sourceId); + await sync_table.addNoteTreeSync(entity.note_id, sourceId); await sql.addAudit(audit_category.UPDATE_TITLE, sourceId, entity.note_id); }); @@ -61,7 +62,7 @@ async function updateNoteHistory(entity, sourceId) { await sql.doInTransaction(async () => { await sql.replace('notes_history', entity); - await sql.addNoteHistorySync(entity.note_history_id, sourceId); + await sync_table.addNoteHistorySync(entity.note_history_id, sourceId); }); log.info("Update/sync note history " + entity.note_history_id); @@ -77,7 +78,7 @@ async function updateNoteReordering(entity, sourceId) { await sql.execute("UPDATE notes_tree SET note_pos = ? WHERE note_id = ?", [entity.ordering[key], key]); }); - await sql.addNoteReorderingSync(entity.note_pid, sourceId); + await sync_table.addNoteReorderingSync(entity.note_pid, sourceId); await sql.addAudit(audit_category.CHANGE_POSITION, sourceId, entity.note_pid); }); } @@ -93,7 +94,7 @@ async function updateOptions(entity, sourceId) { await sql.doInTransaction(async () => { await sql.replace('options', entity); - await sql.addOptionsSync(entity.opt_name, sourceId); + await sync_table.addOptionsSync(entity.opt_name, sourceId); }); await eventLog.addEvent("Synced option " + entity.opt_name); @@ -110,7 +111,7 @@ async function updateRecentNotes(entity, sourceId) { await sql.doInTransaction(async () => { await sql.replace('recent_notes', entity); - await sql.addRecentNoteSync(entity.note_id, sourceId); + await sync_table.addRecentNoteSync(entity.note_id, sourceId); }); } }