From cf53cbf1ddd3fabc90653a9474ca5edc43b8baf1 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 31 Oct 2019 21:58:34 +0100 Subject: [PATCH] moving out note revision content into separate table, refactoring, WIP --- .../0150__note_revision_contents.sql | 33 ++++++ src/entities/note.js | 16 +-- src/entities/note_revision.js | 106 ++++++++++++++++-- src/routes/api/recent_changes.js | 4 +- src/services/app_info.js | 4 +- src/services/note_cache.js | 2 +- src/services/protected_session.js | 81 ++----------- src/services/sync_table.js | 54 ++------- src/services/sync_update.js | 4 +- src/services/utils.js | 11 +- 10 files changed, 172 insertions(+), 143 deletions(-) create mode 100644 db/migrations/0150__note_revision_contents.sql diff --git a/db/migrations/0150__note_revision_contents.sql b/db/migrations/0150__note_revision_contents.sql new file mode 100644 index 000000000..e892915f9 --- /dev/null +++ b/db/migrations/0150__note_revision_contents.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS "note_revisions_mig" (`noteRevisionId` TEXT NOT NULL PRIMARY KEY, + `noteId` TEXT NOT NULL, + `title` TEXT, + `content` TEXT, + `isProtected` INT NOT NULL DEFAULT 0, + `utcDateLastEdited` TEXT NOT NULL, + `utcDateCreated` TEXT NOT NULL, + `utcDateModified` TEXT NOT NULL, + `dateLastEdited` TEXT NOT NULL, + `dateCreated` TEXT NOT NULL, + type TEXT DEFAULT '' NOT NULL, + mime TEXT DEFAULT '' NOT NULL, + hash TEXT DEFAULT '' NOT NULL); + +CREATE TABLE IF NOT EXISTS "note_revision_contents" (`noteRevisionId` TEXT NOT NULL PRIMARY KEY, + `content` TEXT, + hash TEXT DEFAULT '' NOT NULL, + `utcDateModified` TEXT NOT NULL); + +INSERT INTO note_revision_contents (noteRevisionId, content, hash, utcDateModified) +SELECT noteRevisionId, content, hash, utcDateModified FROM note_revisions; + +INSERT INTO note_revisions_mig (noteRevisionId, noteId, title, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, type, mime, hash) +SELECT noteRevisionId, noteId, title, isProtected, utcDateModifiedFrom, utcDateModifiedTo, utcDateModifiedTo, dateModifiedFrom, dateModifiedTo, type, mime, hash FROM note_revisions; + +DROP TABLE note_revisions; +ALTER TABLE note_revisions_mig RENAME TO note_revisions; + +CREATE INDEX `IDX_note_revisions_noteId` ON `note_revisions` (`noteId`); +CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCreated`); +CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`); +CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`); +CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`); diff --git a/src/entities/note.js b/src/entities/note.js index b2f7a4b11..edeafc59c 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -14,8 +14,6 @@ const LABEL_DEFINITION = 'label-definition'; const RELATION = 'relation'; const RELATION_DEFINITION = 'relation-definition'; -const STRING_MIME_TYPES = ["application/x-javascript"]; - /** * This represents a Note which is a central object in the Trilium Notes project. * @@ -44,7 +42,7 @@ class Note extends Entity { super(row); this.isProtected = !!this.isProtected; - /* true if content (meaning any kind of potentially encrypted content) is either not encrypted + /* true if content is either not encrypted * or encrypted, but with available protected session (so effectively decrypted) */ this.isContentAvailable = true; @@ -53,7 +51,7 @@ class Note extends Entity { this.isContentAvailable = protectedSessionService.isProtectedSessionAvailable(); if (this.isContentAvailable) { - protectedSessionService.decryptNote(this); + this.title = protectedSessionService.decrypt(this.title); } else { this.title = "[protected]"; @@ -88,7 +86,7 @@ class Note extends Entity { if (this.isProtected) { if (this.isContentAvailable) { - protectedSessionService.decryptNoteContent(this); + this.content = this.content === null ? null : protectedSessionService.decrypt(this.content); } else { this.content = ""; @@ -129,7 +127,7 @@ class Note extends Entity { if (this.isProtected) { if (this.isContentAvailable) { - protectedSessionService.encryptNoteContent(pojo); + pojo.content = protectedSessionService.encrypt(pojo.content); } else { throw new Error(`Cannot update content of noteId=${this.noteId} since we're out of protected session.`); @@ -171,9 +169,7 @@ class Note extends Entity { /** @returns {boolean} true if the note has string content (not binary) */ isStringNote() { - return ["text", "code", "relation-map", "search"].includes(this.type) - || this.mime.startsWith('text/') - || STRING_MIME_TYPES.includes(this.mime); + return utils.isStringNote(this.type, this.mime); } /** @returns {string} JS script environment - either "frontend" or "backend" */ @@ -746,7 +742,7 @@ class Note extends Entity { updatePojo(pojo) { if (pojo.isProtected) { if (this.isContentAvailable) { - protectedSessionService.encryptNote(pojo); + pojo.title = protectedSessionService.encrypt(pojo.title); } else { // updating protected note outside of protected session means we will keep original ciphertexts diff --git a/src/entities/note_revision.js b/src/entities/note_revision.js index d73e16a3e..c748cc694 100644 --- a/src/entities/note_revision.js +++ b/src/entities/note_revision.js @@ -3,6 +3,9 @@ const Entity = require('./entity'); const protectedSessionService = require('../services/protected_session'); const repository = require('../services/repository'); +const utils = require('../services/utils'); +const dateUtils = require('../services/date_utils'); +const syncTableService = require('../services/sync_table'); /** * NoteRevision represents snapshot of note's title and content at some point in the past. It's used for seamless note versioning. @@ -12,7 +15,6 @@ const repository = require('../services/repository'); * @param {string} type * @param {string} mime * @param {string} title - * @param {string} content * @param {string} isProtected * @param {string} dateModifiedFrom * @param {string} dateModifiedTo @@ -24,7 +26,7 @@ const repository = require('../services/repository'); class NoteRevision extends Entity { static get entityName() { return "note_revisions"; } static get primaryKeyName() { return "noteRevisionId"; } - static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "content", "isProtected", "dateModifiedFrom", "dateModifiedTo", "utcDateModifiedFrom", "utcDateModifiedTo"]; } + static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "isProtected", "dateModifiedFrom", "dateModifiedTo", "utcDateModifiedFrom", "utcDateModifiedTo"]; } constructor(row) { super(row); @@ -32,7 +34,12 @@ class NoteRevision extends Entity { this.isProtected = !!this.isProtected; if (this.isProtected) { - protectedSessionService.decryptNoteRevision(this); + if (protectedSessionService.isProtectedSessionAvailable()) { + this.title = protectedSessionService.decrypt(this.title); + } + else { + this.title = "[Protected]"; + } } } @@ -40,12 +47,97 @@ class NoteRevision extends Entity { return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]); } - beforeSaving() { - if (this.isProtected) { - protectedSessionService.encryptNoteRevision(this); + /** @returns {boolean} true if the note has string content (not binary) */ + isStringNote() { + return utils.isStringNote(this.type, this.mime); + } + + /* + * Note revision content has quite special handling - it's not a separate entity, but a lazily loaded + * part of NoteRevision entity with it's own sync. Reason behind this hybrid design is that + * content can be quite large and it's not necessary to load it / fill memory for any note access even + * if we don't need a content, especially for bulk operations like search. + * + * This is the same approach as is used for Note's content. + */ + + /** @returns {Promise<*>} */ + async getContent(silentNotFoundError = false) { + if (this.content === undefined) { + const res = await sql.getRow(`SELECT content, hash FROM note_revision_contents WHERE noteRevisionId = ?`, [this.noteRevisionId]); + + if (!res) { + if (silentNotFoundError) { + return undefined; + } + else { + throw new Error("Cannot find note revision content for noteRevisionId=" + this.noteRevisionId); + } + } + + this.content = res.content; + + if (this.isProtected) { + if (protectedSessionService.isProtectedSessionAvailable()) { + this.content = protectedSessionService.decrypt(this.content); + } + else { + this.content = ""; + } + } + + if (this.isStringNote()) { + this.content = this.content === null + ? "" + : this.content.toString("UTF-8"); + } } - super.beforeSaving(); + return this.content; + } + + /** @returns {Promise} */ + async setContent(content) { + // force updating note itself so that dateChanged is represented correctly even for the content + this.forcedChange = true; + await this.save(); + + this.content = content; + + const pojo = { + noteRevisionId: this.noteRevisionId, + content: content, + utcDateModified: dateUtils.utcNowDateTime(), + hash: utils.hash(this.noteRevisionId + "|" + content) + }; + + if (this.isProtected) { + if (protectedSessionService.isProtectedSessionAvailable()) { + pojo.content = protectedSessionService.encrypt(pojo.content); + } + else { + throw new Error(`Cannot update content of noteRevisionId=${this.noteRevisionId} since we're out of protected session.`); + } + } + + await sql.upsert("note_revision_contents", "noteRevisionId", pojo); + + await syncTableService.addNoteContentSync(this.noteId); + } + + // cannot be static! + updatePojo(pojo) { + if (pojo.isProtected) { + if (protectedSessionService.isProtectedSessionAvailable()) { + pojo.title = protectedSessionService.encrypt(pojo.title); + } + else { + // updating protected note outside of protected session means we will keep original ciphertexts + delete pojo.title; + } + } + + delete pojo.content; } } diff --git a/src/routes/api/recent_changes.js b/src/routes/api/recent_changes.js index 162c0a353..806fde7b5 100644 --- a/src/routes/api/recent_changes.js +++ b/src/routes/api/recent_changes.js @@ -41,8 +41,8 @@ async function getRecentChanges() { for (const change of recentChanges) { if (change.current_isProtected) { if (protectedSessionService.isProtectedSessionAvailable()) { - change.title = protectedSessionService.decryptNoteTitle(change.noteId, change.title); - change.current_title = protectedSessionService.decryptNoteTitle(change.noteId, change.current_title); + change.title = protectedSessionService.decrypt(change.title); + change.current_title = protectedSessionService.decrypt(change.current_title); } else { change.title = change.current_title = "[Protected]"; diff --git a/src/services/app_info.js b/src/services/app_info.js index 4c372f9ce..402425b92 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -4,8 +4,8 @@ const build = require('./build'); const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); -const APP_DB_VERSION = 149; -const SYNC_VERSION = 10; +const APP_DB_VERSION = 150; +const SYNC_VERSION = 11; const CLIPPER_PROTOCOL_VERSION = "1.0"; module.exports = { diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 8f238e7d7..058a39a89 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -54,7 +54,7 @@ async function loadProtectedNotes() { protectedNoteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 1`); for (const noteId in protectedNoteTitles) { - protectedNoteTitles[noteId] = protectedSessionService.decryptNoteTitle(noteId, protectedNoteTitles[noteId]); + protectedNoteTitles[noteId] = protectedSessionService.decrypt(protectedNoteTitles[noteId]); } } diff --git a/src/services/protected_session.js b/src/services/protected_session.js index 182e20a65..35b3a58d7 100644 --- a/src/services/protected_session.js +++ b/src/services/protected_session.js @@ -34,93 +34,28 @@ function isProtectedSessionAvailable() { return !!dataKeyMap[protectedSessionId]; } -function decryptNoteTitle(noteId, encryptedTitle) { - const dataKey = getDataKey(); - - try { - return dataEncryptionService.decryptString(dataKey, encryptedTitle); - } - catch (e) { - e.message = `Cannot decrypt note title for noteId=${noteId}: ` + e.message; - throw e; - } -} - -function decryptNote(note) { - if (!note.isProtected) { - return; - } - - if (note.title) { - note.title = decryptNoteTitle(note.noteId, note.title); - } -} - -function decryptNoteContent(note) { - try { - if (note.content != null) { - note.content = dataEncryptionService.decrypt(getDataKey(), note.content); - } - } - catch (e) { - e.message = `Cannot decrypt content for noteId=${note.noteId}: ` + e.message; - throw e; - } -} - function decryptNotes(notes) { for (const note of notes) { - decryptNote(note); - } -} - -function decryptNoteRevision(hist) { - const dataKey = getDataKey(); - - if (!hist.isProtected) { - return; - } - - try { - if (hist.title) { - hist.title = dataEncryptionService.decryptString(dataKey, hist.title.toString()); - } - - if (hist.content) { - hist.content = dataEncryptionService.decryptString(dataKey, hist.content.toString()); + if (note.isProtected) { + note.title = decrypt(note.title); } } - catch (e) { - throw new Error(`Decryption failed for note ${hist.noteId}, revision ${hist.noteRevisionId}: ` + e.message + " " + e.stack); - } } -function encryptNote(note) { - note.title = dataEncryptionService.encrypt(getDataKey(), note.title); +function encrypt(plainText) { + return dataEncryptionService.encrypt(getDataKey(), plainText); } -function encryptNoteContent(note) { - note.content = dataEncryptionService.encrypt(getDataKey(), note.content); -} - -function encryptNoteRevision(revision) { - const dataKey = getDataKey(); - - revision.title = dataEncryptionService.encrypt(dataKey, revision.title); - revision.content = dataEncryptionService.encrypt(dataKey, revision.content); +function decrypt(cipherText) { + return dataEncryptionService.encrypt(getDataKey(), cipherText); } module.exports = { setDataKey, getDataKey, isProtectedSessionAvailable, - decryptNoteTitle, - decryptNote, - decryptNoteContent, + encrypt, + decrypt, decryptNotes, - decryptNoteRevision, - encryptNote, - encryptNoteContent, - encryptNoteRevision, setProtectedSessionId }; \ No newline at end of file diff --git a/src/services/sync_table.js b/src/services/sync_table.js index b293acabb..bb3ace759 100644 --- a/src/services/sync_table.js +++ b/src/services/sync_table.js @@ -4,42 +4,6 @@ const dateUtils = require('./date_utils'); const log = require('./log'); const cls = require('./cls'); -async function addNoteSync(noteId, sourceId) { - await addEntitySync("notes", noteId, sourceId) -} - -async function addNoteContentSync(noteId, sourceId) { - await addEntitySync("note_contents", noteId, sourceId) -} - -async function addBranchSync(branchId, sourceId) { - await addEntitySync("branches", branchId, sourceId) -} - -async function addNoteReorderingSync(parentNoteId, sourceId) { - await addEntitySync("note_reordering", parentNoteId, sourceId) -} - -async function addNoteRevisionSync(noteRevisionId, sourceId) { - await addEntitySync("note_revisions", noteRevisionId, sourceId); -} - -async function addOptionsSync(name, sourceId) { - await addEntitySync("options", name, sourceId); -} - -async function addRecentNoteSync(noteId, sourceId) { - await addEntitySync("recent_notes", noteId, sourceId); -} - -async function addAttributeSync(attributeId, sourceId) { - await addEntitySync("attributes", attributeId, sourceId); -} - -async function addApiTokenSync(apiTokenId, sourceId) { - await addEntitySync("api_tokens", apiTokenId, sourceId); -} - async function addEntitySync(entityName, entityId, sourceId) { await sql.replace("sync", { entityName: entityName, @@ -107,15 +71,15 @@ async function fillAllSyncRows() { } module.exports = { - addNoteSync, - addNoteContentSync, - addBranchSync, - addNoteReorderingSync, - addNoteRevisionSync, - addOptionsSync, - addRecentNoteSync, - addAttributeSync, - addApiTokenSync, + addNoteSync: async (noteId, sourceId) => await addEntitySync("notes", noteId, sourceId), + addNoteContentSync: async (noteId, sourceId) => await addEntitySync("note_contents", noteId, sourceId), + addBranchSync: async (branchId, sourceId) => await addEntitySync("branches", branchId, sourceId), + addNoteReorderingSync: async (parentNoteId, sourceId) => await addEntitySync("note_reordering", parentNoteId, sourceId), + addNoteRevisionSync: async (noteRevisionId, sourceId) => await addEntitySync("note_revisions", noteRevisionId, sourceId), + addOptionsSync: async (name, sourceId) => await addEntitySync("options", name, sourceId), + addRecentNoteSync: async (noteId, sourceId) => await addEntitySync("recent_notes", noteId, sourceId), + addAttributeSync: async (attributeId, sourceId) => await addEntitySync("attributes", attributeId, sourceId), + addApiTokenSync: async (apiTokenId, sourceId) => await addEntitySync("api_tokens", apiTokenId, sourceId), addEntitySync, fillAllSyncRows }; \ No newline at end of file diff --git a/src/services/sync_update.js b/src/services/sync_update.js index 9b5e42331..1db1c9622 100644 --- a/src/services/sync_update.js +++ b/src/services/sync_update.js @@ -117,9 +117,9 @@ async function updateNoteRevision(entity, sourceId) { async function updateNoteReordering(entityId, entity, sourceId) { await sql.transactional(async () => { - Object.keys(entity).forEach(async key => { + for (const key in entity) { await sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]); - }); + } await syncTableService.addNoteReorderingSync(entityId, sourceId); }); diff --git a/src/services/utils.js b/src/services/utils.js index b5b9e3684..28e74d019 100644 --- a/src/services/utils.js +++ b/src/services/utils.js @@ -154,6 +154,14 @@ function getContentDisposition(filename) { return `file; filename="${sanitizedFilename}"; filename*=UTF-8''${sanitizedFilename}`; } +const STRING_MIME_TYPES = ["application/x-javascript"]; + +function isStringNote(type, mime) { + return ["text", "code", "relation-map", "search"].includes(type) + || mime.startsWith('text/') + || STRING_MIME_TYPES.includes(mime); +} + module.exports = { randomSecureToken, randomString, @@ -177,5 +185,6 @@ module.exports = { escapeRegExp, crash, sanitizeFilenameForHeader, - getContentDisposition + getContentDisposition, + isStringNote }; \ No newline at end of file