gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-06-05 01:18:44 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

we now remember past source ids so app restart won't change anything in sync operation

Browse Source
This commit is contained in:
azivner 2017-11-16 21:50:00 -05:00
parent 9e7fcdfe15
commit b2d1ca1c28
15 changed files with 144 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app.js
View File

@ -9,36 +9,8 @@ const helmet = require('helmet');
const session = require('express-session');
const FileStore = require('session-file-store')(session);
const os = require('os');
const options = require('./services/options');
const utils = require('./services/utils');
const sql = require('./services/sql');
const dataDir = require('./services/data_dir');
const sessionSecret = require('./services/session_secret');
const db = require('sqlite');
db.open(dataDir.DOCUMENT_PATH, { Promise }).then(async () => {
const tableResults = await sql.getResults("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'");
if (tableResults.length !== 1) {
console.log("No connection to initialized DB.");
process.exit(1);
}
if (!await options.getOption('document_id')) {
await options.setOption('document_id', utils.randomString(32));
}
if (!await options.getOption('document_secret')) {
await options.setOption('document_secret', utils.randomSecureToken(32));
}
})
.catch(e => {
console.log("Error connecting to DB.", e);
process.exit(1);
});
const app = express();
// view engine setup

5
migrations/0034__source_ids.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE `source_ids` (
`source_id` TEXT NOT NULL,
`date_created` INTEGER NOT NULL,
PRIMARY KEY(`source_id`)
);

4
routes/api/login.js
View File

@ -5,7 +5,7 @@ const router = express.Router();
const options = require('../../services/options');
const utils = require('../../services/utils');
const migration = require('../../services/migration');
const SOURCE_ID = require('../../services/source_id');
const source_id = require('../../services/source_id');
const auth = require('../../services/auth');
const password_encryption = require('../../services/password_encryption');
const protected_session = require('../../services/protected_session');
@ -40,7 +40,7 @@ router.post('/sync', async (req, res, next) => {
req.session.loggedIn = true;
res.send({
sourceId: SOURCE_ID
sourceId: source_id.currentSourceId
});
});

3
routes/api/note_history.js
View File

@ -6,6 +6,7 @@ const sql = require('../../services/sql');
const auth = require('../../services/auth');
const data_encryption = require('../../services/data_encryption');
const protected_session = require('../../services/protected_session');
const sync_table = require('../../services/sync_table');
router.get('/:noteId', auth.checkApiAuth, async (req, res, next) => {
const noteId = req.params.noteId;
@ -27,7 +28,7 @@ router.put('', auth.checkApiAuth, async (req, res, next) => {
await sql.doInTransaction(async () => {
await sql.replace("notes_history", req.body);
await sql.addNoteHistorySync(req.body.note_history_id);
await sync_table.addNoteHistorySync(req.body.note_history_id);
});
res.send();

11
routes/api/notes_move.js
View File

@ -6,6 +6,7 @@ const sql = require('../../services/sql');
const utils = require('../../services/utils');
const audit_category = require('../../services/audit_category');
const auth = require('../../services/auth');
const sync_table = require('../../services/sync_table');
router.put('/:noteId/moveTo/:parentId', auth.checkApiAuth, async (req, res, next) => {
let noteId = req.params.noteId;
@ -26,7 +27,7 @@ router.put('/:noteId/moveTo/:parentId', auth.checkApiAuth, async (req, res, next
await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?",
[parentId, newNotePos, now, noteId]);
await sql.addNoteTreeSync(noteId);
await sync_table.addNoteTreeSync(noteId);
await sql.addAudit(audit_category.CHANGE_PARENT, utils.browserId(req), noteId, null, parentId);
});
@ -50,8 +51,8 @@ router.put('/:noteId/moveBefore/:beforeNoteId', async (req, res, next) => {
await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?",
[beforeNote['note_pid'], beforeNote['note_pos'], now, noteId]);
await sql.addNoteTreeSync(noteId);
await sql.addNoteReorderingSync(beforeNote['note_pid']);
await sync_table.addNoteTreeSync(noteId);
await sync_table.addNoteReorderingSync(beforeNote['note_pid']);
await sql.addAudit(audit_category.CHANGE_POSITION, utils.browserId(req), beforeNote['note_pid']);
});
}
@ -76,8 +77,8 @@ router.put('/:noteId/moveAfter/:afterNoteId', async (req, res, next) => {
await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?",
[afterNote['note_pid'], afterNote['note_pos'] + 1, now, noteId]);
await sql.addNoteTreeSync(noteId);
await sql.addNoteReorderingSync(afterNote['note_pid']);
await sync_table.addNoteTreeSync(noteId);
await sync_table.addNoteReorderingSync(afterNote['note_pid']);
await sql.addAudit(audit_category.CHANGE_POSITION, utils.browserId(req), afterNote['note_pid']);
});
}

5
routes/api/recent_notes.js
View File

@ -5,6 +5,7 @@ const router = express.Router();
const sql = require('../../services/sql');
const auth = require('../../services/auth');
const utils = require('../../services/utils');
const sync_table = require('../../services/sync_table');
router.get('', auth.checkApiAuth, async (req, res, next) => {
res.send(await getRecentNotes());
@ -17,7 +18,7 @@ router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => {
is_deleted: 0
});
await sql.addRecentNoteSync(req.params.noteId);
await sync_table.addRecentNoteSync(req.params.noteId);
res.send(await getRecentNotes());
});
@ -25,7 +26,7 @@ router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => {
router.delete('/:noteId', auth.checkApiAuth, async (req, res, next) => {
await sql.execute('UPDATE recent_notes SET is_deleted = 1 WHERE note_id = ?', [req.params.noteId]);
await sql.addRecentNoteSync(req.params.noteId);
await sync_table.addRecentNoteSync(req.params.noteId);
res.send(await getRecentNotes());
});

3
routes/api/sync.js
View File

@ -14,9 +14,8 @@ router.post('/now', auth.checkApiAuth, async (req, res, next) => {
router.get('/changed', auth.checkApiAuth, async (req, res, next) => {
const lastSyncId = parseInt(req.query.lastSyncId);
const sourceId = req.query.sourceId;
res.send(await sql.getResults("SELECT * FROM sync WHERE id > ? AND source_id != ?", [lastSyncId, sourceId]));
res.send(await sql.getResults("SELECT * FROM sync WHERE id > ?", [lastSyncId]));
});
router.get('/notes/:noteId', auth.checkApiAuth, async (req, res, next) => {

2
services/migration.js
View File

@ -4,7 +4,7 @@ const options = require('./options');
const fs = require('fs-extra');
const log = require('./log');
const APP_DB_VERSION = 33;
const APP_DB_VERSION = 34;
const MIGRATIONS_DIR = "migrations";
async function migrate() {

19
services/notes.js
View File

@ -4,6 +4,7 @@ const utils = require('./utils');
const notes = require('./notes');
const audit_category = require('./audit_category');
const data_encryption = require('./data_encryption');
const sync_table = require('./sync_table');
async function createNewNote(parentNoteId, note, browserId) {
const noteId = utils.newNoteId();
@ -38,8 +39,8 @@ async function createNewNote(parentNoteId, note, browserId) {
await sql.doInTransaction(async () => {
await sql.addAudit(audit_category.CREATE_NOTE, browserId, noteId);
await sql.addNoteTreeSync(noteId);
await sql.addNoteSync(noteId);
await sync_table.addNoteTreeSync(noteId);
await sync_table.addNoteSync(noteId);
const now = utils.nowTimestamp();
@ -105,7 +106,7 @@ async function protectNote(note, dataKey, protect) {
await sql.execute("UPDATE notes SET note_title = ?, note_text = ?, is_protected = ? WHERE note_id = ?",
[note.note_title, note.note_text, note.is_protected, note.note_id]);
await sql.addNoteSync(note.note_id);
await sync_table.addNoteSync(note.note_id);
}
await protectNoteHistory(note.note_id, dataKey, protect);
@ -129,7 +130,7 @@ async function protectNoteHistory(noteId, dataKey, protect) {
await sql.execute("UPDATE notes_history SET note_title = ?, note_text = ?, is_protected = ? WHERE note_history_id = ?",
[history.note_title, history.note_text, history.is_protected, history.note_history_id]);
await sql.addNoteHistorySync(history.note_history_id);
await sync_table.addNoteHistorySync(history.note_history_id);
}
}
@ -166,7 +167,7 @@ async function updateNote(noteId, newNote, ctx) {
now
]);
await sql.addNoteHistorySync(newNoteHistoryId);
await sync_table.addNoteHistorySync(newNoteHistoryId);
}
await protectNoteHistory(noteId, ctx.getDataKey(), newNote.detail.is_protected);
@ -194,8 +195,8 @@ async function updateNote(noteId, newNote, ctx) {
await sql.insert("links", link);
}
await sql.addNoteTreeSync(noteId);
await sql.addNoteSync(noteId);
await sync_table.addNoteTreeSync(noteId);
await sync_table.addNoteSync(noteId);
});
}
@ -232,8 +233,8 @@ async function deleteNote(noteId, browserId) {
await sql.execute("update notes_tree set is_deleted = 1, date_modified = ? where note_id = ?", [now, noteId]);
await sql.execute("update notes set is_deleted = 1, date_modified = ? where note_id = ?", [now, noteId]);
await sql.addNoteTreeSync(noteId);
await sql.addNoteSync(noteId);
await sync_table.addNoteTreeSync(noteId);
await sync_table.addNoteSync(noteId);
await sql.addAudit(audit_category.DELETE_NOTE, browserId, noteId);
}

13
services/options.js
View File

@ -1,5 +1,6 @@
const sql = require('./sql');
const utils = require('./utils');
const sync_table = require('./sync_table');
const SYNCED_OPTIONS = [ 'username', 'password_verification_hash', 'encrypted_data_key', 'protected_session_timeout',
'history_snapshot_time_interval' ];
@ -20,7 +21,7 @@ async function setOptionInTransaction(optName, optValue) {
async function setOption(optName, optValue) {
if (SYNCED_OPTIONS.includes(optName)) {
await sql.addOptionsSync(optName);
await sync_table.addOptionsSync(optName);
}
await setOptionNoSync(optName, optValue);
@ -32,6 +33,16 @@ async function setOptionNoSync(optName, optValue) {
await sql.execute("UPDATE options SET opt_value = ?, date_modified = ? WHERE opt_name = ?", [optValue, now, optName]);
}
sql.dbReady.then(async () => {
if (!await getOption('document_id')) {
await setOption('document_id', utils.randomSecureToken(16));
}
if (!await getOption('document_secret')) {
await setOption('document_secret', utils.randomSecureToken(16));
}
});
module.exports = {
getOption,
setOption,

25
services/source_id.js
View File

@ -1,8 +1,27 @@
const utils = require('./utils');
const log = require('./log');
const sql = require('./sql');
const sourceId = utils.randomString(16);
const currentSourceId = utils.randomString(12);
log.info("Using sourceId=" + sourceId);
log.info("Using sourceId=" + currentSourceId);
module.exports = sourceId;
let allSourceIds = [];
sql.dbReady.then(async () => {
sql.insert("source_ids", {
source_id: currentSourceId,
date_created: utils.nowTimestamp()
});
allSourceIds = await sql.getFlattenedResults("source_id", "SELECT source_id FROM source_ids");
});
function isLocalSourceId(srcId) {
return allSourceIds.includes(srcId);
}
module.exports = {
currentSourceId,
isLocalSourceId
};

60
services/sql.js
View File

@ -3,7 +3,7 @@
const db = require('sqlite');
const utils = require('./utils');
const log = require('./log');
const SOURCE_ID = require('./source_id');
const dataDir = require('./data_dir');
async function insert(table_name, rec, replace = false) {
const keys = Object.keys(rec);
@ -116,39 +116,6 @@ async function deleteRecentAudits(category, browserId, noteId) {
[category, browserId, noteId, deleteCutoff])
}
async function addNoteSync(noteId, sourceId) {
await addEntitySync("notes", noteId, sourceId)
}
async function addNoteTreeSync(noteId, sourceId) {
await addEntitySync("notes_tree", noteId, sourceId)
}
async function addNoteReorderingSync(noteId, sourceId) {
await addEntitySync("notes_reordering", noteId, sourceId)
}
async function addNoteHistorySync(noteHistoryId, sourceId) {
await addEntitySync("notes_history", noteHistoryId, sourceId);
}
async function addOptionsSync(optName, sourceId) {
await addEntitySync("options", optName, sourceId);
}
async function addRecentNoteSync(noteId, sourceId) {
await addEntitySync("recent_notes", noteId, sourceId);
}
async function addEntitySync(entityName, entityId, sourceId) {
await replace("sync", {
entity_name: entityName,
entity_id: entityId,
sync_date: utils.nowTimestamp(),
source_id: sourceId || SOURCE_ID
});
}
async function wrap(func) {
const error = new Error();
@ -182,7 +149,24 @@ async function doInTransaction(func) {
}
}
const dbReady = db.open(dataDir.DOCUMENT_PATH, { Promise });
dbReady
.then(async () => {
const tableResults = await getResults("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'");
if (tableResults.length !== 1) {
console.log("No connection to initialized DB.");
process.exit(1);
}
})
.catch(e => {
console.log("Error connecting to DB.", e);
process.exit(1);
});
module.exports = {
dbReady,
insert,
replace,
getSingleValue,
@ -196,11 +180,5 @@ module.exports = {
addAudit,
deleteRecentAudits,
remove,
doInTransaction,
addNoteSync,
addNoteTreeSync,
addNoteReorderingSync,
addNoteHistorySync,
addOptionsSync,
addRecentNoteSync
doInTransaction
};

12
services/sync.js
View File

@ -7,7 +7,7 @@ const options = require('./options');
const migration = require('./migration');
const utils = require('./utils');
const config = require('./config');
const SOURCE_ID = require('./source_id');
const source_id = require('./source_id');
const notes = require('./notes');
const syncUpdate = require('./sync_update');
@ -100,13 +100,19 @@ async function login() {
async function pullSync(syncContext) {
const lastSyncedPull = parseInt(await options.getOption('last_synced_pull'));
const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull + "&sourceId=" + SOURCE_ID;
const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull;
const syncRows = await syncRequest(syncContext, 'GET', changesUri);
log.info("Pulled " + syncRows.length + " changes from " + changesUri);
for (const sync of syncRows) {
if (source_id.isLocalSourceId(sync.source_id)) {
log.info("Skipping " + sync.entity_name + " " + sync.entity_id + " because it has local source id.");
continue;
}
const resp = await syncRequest(syncContext, 'GET', "/api/sync/" + sync.entity_name + "/" + sync.entity_id);
if (sync.entity_name === 'notes') {
@ -204,7 +210,7 @@ async function readAndPushEntity(sync, syncContext) {
async function sendEntity(syncContext, entity, entityName) {
const payload = {
sourceId: SOURCE_ID,
sourceId: source_id.currentSourceId,
entity: entity
};

45
services/sync_table.js Normal file
View File

@ -0,0 +1,45 @@
const sql = require('./sql');
const source_id = require('./source_id');
const utils = require('./utils');
async function addNoteSync(noteId, sourceId) {
await addEntitySync("notes", noteId, sourceId)
}
async function addNoteTreeSync(noteId, sourceId) {
await addEntitySync("notes_tree", noteId, sourceId)
}
async function addNoteReorderingSync(noteId, sourceId) {
await addEntitySync("notes_reordering", noteId, sourceId)
}
async function addNoteHistorySync(noteHistoryId, sourceId) {
await addEntitySync("notes_history", noteHistoryId, sourceId);
}
async function addOptionsSync(optName, sourceId) {
await addEntitySync("options", optName, sourceId);
}
async function addRecentNoteSync(noteId, sourceId) {
await addEntitySync("recent_notes", noteId, sourceId);
}
async function addEntitySync(entityName, entityId, sourceId) {
await sql.replace("sync", {
entity_name: entityName,
entity_id: entityId,
sync_date: utils.nowTimestamp(),
source_id: sourceId || source_id.currentSourceId
});
}
module.exports = {
addNoteSync,
addNoteTreeSync,
addNoteReorderingSync,
addNoteHistorySync,
addOptionsSync,
addRecentNoteSync
};

13
services/sync_update.js
View File

@ -5,6 +5,7 @@ const utils = require('./utils');
const audit_category = require('./audit_category');
const eventLog = require('./event_log');
const notes = require('./notes');
const sync_table = require('./sync_table');
async function updateNote(entity, links, sourceId) {
const origNote = await sql.getSingleResult("select * from notes where note_id = ?", [entity.note_id]);
@ -21,7 +22,7 @@ async function updateNote(entity, links, sourceId) {
await sql.insert('link', link);
}
await sql.addNoteSync(entity.note_id, sourceId);
await sync_table.addNoteSync(entity.note_id, sourceId);
await notes.addNoteAudits(origNote, entity, sourceId);
await eventLog.addNoteEvent(entity.note_id, "Synced note <note>");
});
@ -42,7 +43,7 @@ async function updateNoteTree(entity, sourceId) {
await sql.replace('notes_tree', entity);
await sql.addNoteTreeSync(entity.note_id, sourceId);
await sync_table.addNoteTreeSync(entity.note_id, sourceId);
await sql.addAudit(audit_category.UPDATE_TITLE, sourceId, entity.note_id);
});
@ -61,7 +62,7 @@ async function updateNoteHistory(entity, sourceId) {
await sql.doInTransaction(async () => {
await sql.replace('notes_history', entity);
await sql.addNoteHistorySync(entity.note_history_id, sourceId);
await sync_table.addNoteHistorySync(entity.note_history_id, sourceId);
});
log.info("Update/sync note history " + entity.note_history_id);
@ -77,7 +78,7 @@ async function updateNoteReordering(entity, sourceId) {
await sql.execute("UPDATE notes_tree SET note_pos = ? WHERE note_id = ?", [entity.ordering[key], key]);
});
await sql.addNoteReorderingSync(entity.note_pid, sourceId);
await sync_table.addNoteReorderingSync(entity.note_pid, sourceId);
await sql.addAudit(audit_category.CHANGE_POSITION, sourceId, entity.note_pid);
});
}
@ -93,7 +94,7 @@ async function updateOptions(entity, sourceId) {
await sql.doInTransaction(async () => {
await sql.replace('options', entity);
await sql.addOptionsSync(entity.opt_name, sourceId);
await sync_table.addOptionsSync(entity.opt_name, sourceId);
});
await eventLog.addEvent("Synced option " + entity.opt_name);
@ -110,7 +111,7 @@ async function updateRecentNotes(entity, sourceId) {
await sql.doInTransaction(async () => {
await sql.replace('recent_notes', entity);
await sql.addRecentNoteSync(entity.note_id, sourceId);
await sync_table.addRecentNoteSync(entity.note_id, sourceId);
});
}
}