moving out note revision content into separate table, refactoring, WIP

db/migrations/0150__note_revision_contents.sql

@@ -0,0 +1,33 @@
+CREATE TABLE IF NOT EXISTS "note_revisions_mig" (`noteRevisionId`	TEXT NOT NULL PRIMARY KEY,
+                                                `noteId`	TEXT NOT NULL,
+                                                `title`	TEXT,
+                                                `content`	TEXT,
+                                                `isProtected`	INT NOT NULL DEFAULT 0,
+                                                `utcDateLastEdited` TEXT NOT NULL,
+                                                `utcDateCreated` TEXT NOT NULL,
+                                                `utcDateModified` TEXT NOT NULL,
+                                                `dateLastEdited` TEXT NOT NULL,
+                                                `dateCreated` TEXT NOT NULL,
+                                                type TEXT DEFAULT '' NOT NULL,
+                                                mime TEXT DEFAULT '' NOT NULL,
+                                                hash TEXT DEFAULT '' NOT NULL);
+
+CREATE TABLE IF NOT EXISTS "note_revision_contents" (`noteRevisionId`	TEXT NOT NULL PRIMARY KEY,
+                                                 `content`	TEXT,
+                                                 hash TEXT DEFAULT '' NOT NULL,
+                                                 `utcDateModified` TEXT NOT NULL);
+
+INSERT INTO note_revision_contents (noteRevisionId, content, hash, utcDateModified)
+SELECT noteRevisionId, content, hash, utcDateModified FROM note_revisions;
+
+INSERT INTO note_revisions_mig (noteRevisionId, noteId, title, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, type, mime, hash)
+SELECT noteRevisionId, noteId, title, isProtected, utcDateModifiedFrom, utcDateModifiedTo, utcDateModifiedTo, dateModifiedFrom, dateModifiedTo, type, mime, hash FROM note_revisions;
+
+DROP TABLE note_revisions;
+ALTER TABLE note_revisions_mig RENAME TO note_revisions;
+
+CREATE INDEX `IDX_note_revisions_noteId` ON `note_revisions` (`noteId`);
+CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCreated`);
+CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`);
+CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`);
+CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`);

src/entities/note.js

@@ -14,8 +14,6 @@ const LABEL_DEFINITION = 'label-definition';
 const RELATION = 'relation';
 const RELATION_DEFINITION = 'relation-definition';
 
-const STRING_MIME_TYPES = ["application/x-javascript"];
-
 /**
  * This represents a Note which is a central object in the Trilium Notes project.
  *
@@ -44,7 +42,7 @@ class Note extends Entity {
         super(row);
 
         this.isProtected = !!this.isProtected;
-        /* true if content (meaning any kind of potentially encrypted content) is either not encrypted
+        /* true if content is either not encrypted
          * or encrypted, but with available protected session (so effectively decrypted) */
         this.isContentAvailable = true;
 
@@ -53,7 +51,7 @@ class Note extends Entity {
             this.isContentAvailable = protectedSessionService.isProtectedSessionAvailable();
 
             if (this.isContentAvailable) {
-                protectedSessionService.decryptNote(this);
+                this.title = protectedSessionService.decrypt(this.title);
             }
             else {
                 this.title = "[protected]";
@@ -88,7 +86,7 @@ class Note extends Entity {
 
             if (this.isProtected) {
                 if (this.isContentAvailable) {
-                    protectedSessionService.decryptNoteContent(this);
+                    this.content = this.content === null ? null : protectedSessionService.decrypt(this.content);
                 }
                 else {
                     this.content = "";
@@ -129,7 +127,7 @@ class Note extends Entity {
 
         if (this.isProtected) {
             if (this.isContentAvailable) {
-                protectedSessionService.encryptNoteContent(pojo);
+                pojo.content = protectedSessionService.encrypt(pojo.content);
             }
             else {
                 throw new Error(`Cannot update content of noteId=${this.noteId} since we're out of protected session.`);
@@ -171,9 +169,7 @@ class Note extends Entity {
 
     /** @returns {boolean} true if the note has string content (not binary) */
     isStringNote() {
-        return ["text", "code", "relation-map", "search"].includes(this.type)
+        return utils.isStringNote(this.type, this.mime);
-            || this.mime.startsWith('text/')
-            || STRING_MIME_TYPES.includes(this.mime);
     }
 
     /** @returns {string} JS script environment - either "frontend" or "backend" */
@@ -746,7 +742,7 @@ class Note extends Entity {
     updatePojo(pojo) {
         if (pojo.isProtected) {
             if (this.isContentAvailable) {
-                protectedSessionService.encryptNote(pojo);
+                pojo.title = protectedSessionService.encrypt(pojo.title);
             }
             else {
                 // updating protected note outside of protected session means we will keep original ciphertexts

src/entities/note_revision.js

@@ -3,6 +3,9 @@
 const Entity = require('./entity');
 const protectedSessionService = require('../services/protected_session');
 const repository = require('../services/repository');
+const utils = require('../services/utils');
+const dateUtils = require('../services/date_utils');
+const syncTableService = require('../services/sync_table');
 
 /**
  * NoteRevision represents snapshot of note's title and content at some point in the past. It's used for seamless note versioning.
@@ -12,7 +15,6 @@ const repository = require('../services/repository');
  * @param {string} type
  * @param {string} mime
  * @param {string} title
- * @param {string} content
  * @param {string} isProtected
  * @param {string} dateModifiedFrom
  * @param {string} dateModifiedTo
@@ -24,7 +26,7 @@ const repository = require('../services/repository');
 class NoteRevision extends Entity {
     static get entityName() { return "note_revisions"; }
     static get primaryKeyName() { return "noteRevisionId"; }
-    static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "content", "isProtected", "dateModifiedFrom", "dateModifiedTo", "utcDateModifiedFrom", "utcDateModifiedTo"]; }
+    static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "isProtected", "dateModifiedFrom", "dateModifiedTo", "utcDateModifiedFrom", "utcDateModifiedTo"]; }
 
     constructor(row) {
         super(row);
@@ -32,7 +34,12 @@ class NoteRevision extends Entity {
         this.isProtected = !!this.isProtected;
 
         if (this.isProtected) {
-            protectedSessionService.decryptNoteRevision(this);
+            if (protectedSessionService.isProtectedSessionAvailable()) {
+                this.title = protectedSessionService.decrypt(this.title);
+            }
+            else {
+                this.title = "[Protected]";
+            }
         }
     }
 
@@ -40,12 +47,97 @@ class NoteRevision extends Entity {
         return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]);
     }
 
-    beforeSaving() {
+    /** @returns {boolean} true if the note has string content (not binary) */
-        if (this.isProtected) {
+    isStringNote() {
-            protectedSessionService.encryptNoteRevision(this);
+        return utils.isStringNote(this.type, this.mime);
+    }
+
+    /*
+     * Note revision content has quite special handling - it's not a separate entity, but a lazily loaded
+     * part of NoteRevision entity with it's own sync. Reason behind this hybrid design is that
+     * content can be quite large and it's not necessary to load it / fill memory for any note access even
+     * if we don't need a content, especially for bulk operations like search.
+     *
+     * This is the same approach as is used for Note's content.
+     */
+
+    /** @returns {Promise<*>} */
+    async getContent(silentNotFoundError = false) {
+        if (this.content === undefined) {
+            const res = await sql.getRow(`SELECT content, hash FROM note_revision_contents WHERE noteRevisionId = ?`, [this.noteRevisionId]);
+
+            if (!res) {
+                if (silentNotFoundError) {
+                    return undefined;
+                }
+                else {
+                    throw new Error("Cannot find note revision content for noteRevisionId=" + this.noteRevisionId);
+                }
+            }
+
+            this.content = res.content;
+
+            if (this.isProtected) {
+                if (protectedSessionService.isProtectedSessionAvailable()) {
+                    this.content = protectedSessionService.decrypt(this.content);
+                }
+                else {
+                    this.content = "";
+                }
+            }
+
+            if (this.isStringNote()) {
+                this.content = this.content === null
+                    ? ""
+                    : this.content.toString("UTF-8");
+            }
         }
 
-        super.beforeSaving();
+        return this.content;
+    }
+
+    /** @returns {Promise} */
+    async setContent(content) {
+        // force updating note itself so that dateChanged is represented correctly even for the content
+        this.forcedChange = true;
+        await this.save();
+
+        this.content = content;
+
+        const pojo = {
+            noteRevisionId: this.noteRevisionId,
+            content: content,
+            utcDateModified: dateUtils.utcNowDateTime(),
+            hash: utils.hash(this.noteRevisionId + "|" + content)
+        };
+
+        if (this.isProtected) {
+            if (protectedSessionService.isProtectedSessionAvailable()) {
+                pojo.content = protectedSessionService.encrypt(pojo.content);
+            }
+            else {
+                throw new Error(`Cannot update content of noteRevisionId=${this.noteRevisionId} since we're out of protected session.`);
+            }
+        }
+
+        await sql.upsert("note_revision_contents", "noteRevisionId", pojo);
+
+        await syncTableService.addNoteContentSync(this.noteId);
+    }
+
+    // cannot be static!
+    updatePojo(pojo) {
+        if (pojo.isProtected) {
+            if (protectedSessionService.isProtectedSessionAvailable()) {
+                pojo.title = protectedSessionService.encrypt(pojo.title);
+            }
+            else {
+                // updating protected note outside of protected session means we will keep original ciphertexts
+                delete pojo.title;
+            }
+        }
+
+        delete pojo.content;
     }
 }
 

src/routes/api/recent_changes.js

@@ -41,8 +41,8 @@ async function getRecentChanges() {
     for (const change of recentChanges) {
         if (change.current_isProtected) {
             if (protectedSessionService.isProtectedSessionAvailable()) {
-                change.title = protectedSessionService.decryptNoteTitle(change.noteId, change.title);
+                change.title = protectedSessionService.decrypt(change.title);
-                change.current_title = protectedSessionService.decryptNoteTitle(change.noteId, change.current_title);
+                change.current_title = protectedSessionService.decrypt(change.current_title);
             }
             else {
                 change.title = change.current_title = "[Protected]";

src/services/app_info.js

@@ -4,8 +4,8 @@ const build = require('./build');
 const packageJson = require('../../package');
 const {TRILIUM_DATA_DIR} = require('./data_dir');
 
-const APP_DB_VERSION = 149;
+const APP_DB_VERSION = 150;
-const SYNC_VERSION = 10;
+const SYNC_VERSION = 11;
 const CLIPPER_PROTOCOL_VERSION = "1.0";
 
 module.exports = {

src/services/note_cache.js

@@ -54,7 +54,7 @@ async function loadProtectedNotes() {
     protectedNoteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 1`);
 
     for (const noteId in protectedNoteTitles) {
-        protectedNoteTitles[noteId] = protectedSessionService.decryptNoteTitle(noteId, protectedNoteTitles[noteId]);
+        protectedNoteTitles[noteId] = protectedSessionService.decrypt(protectedNoteTitles[noteId]);
     }
 }
 

src/services/protected_session.js

@@ -34,93 +34,28 @@ function isProtectedSessionAvailable() {
     return !!dataKeyMap[protectedSessionId];
 }
 
-function decryptNoteTitle(noteId, encryptedTitle) {
-    const dataKey = getDataKey();
-
-    try {
-        return dataEncryptionService.decryptString(dataKey, encryptedTitle);
-    }
-    catch (e) {
-        e.message = `Cannot decrypt note title for noteId=${noteId}: ` + e.message;
-        throw e;
-    }
-}
-
-function decryptNote(note) {
-    if (!note.isProtected) {
-        return;
-    }
-
-    if (note.title) {
-        note.title = decryptNoteTitle(note.noteId, note.title);
-    }
-}
-
-function decryptNoteContent(note) {
-    try {
-        if (note.content != null) {
-            note.content = dataEncryptionService.decrypt(getDataKey(), note.content);
-        }
-    }
-    catch (e) {
-        e.message = `Cannot decrypt content for noteId=${note.noteId}: ` + e.message;
-        throw e;
-    }
-}
-
 function decryptNotes(notes) {
     for (const note of notes) {
-        decryptNote(note);
+        if (note.isProtected) {
-    }
+            note.title = decrypt(note.title);
-}
-
-function decryptNoteRevision(hist) {
-    const dataKey = getDataKey();
-
-    if (!hist.isProtected) {
-        return;
-    }
-
-    try {
-        if (hist.title) {
-            hist.title = dataEncryptionService.decryptString(dataKey, hist.title.toString());
-        }
-
-        if (hist.content) {
-            hist.content = dataEncryptionService.decryptString(dataKey, hist.content.toString());
         }
     }
-    catch (e) {
-        throw new Error(`Decryption failed for note ${hist.noteId}, revision ${hist.noteRevisionId}: ` + e.message + " " + e.stack);
-    }
 }
 
-function encryptNote(note) {
+function encrypt(plainText) {
-    note.title = dataEncryptionService.encrypt(getDataKey(), note.title);
+    return dataEncryptionService.encrypt(getDataKey(), plainText);
 }
 
-function encryptNoteContent(note) {
+function decrypt(cipherText) {
-    note.content = dataEncryptionService.encrypt(getDataKey(), note.content);
+    return dataEncryptionService.encrypt(getDataKey(), cipherText);
-}
-
-function encryptNoteRevision(revision) {
-    const dataKey = getDataKey();
-
-    revision.title = dataEncryptionService.encrypt(dataKey, revision.title);
-    revision.content = dataEncryptionService.encrypt(dataKey, revision.content);
 }
 
 module.exports = {
     setDataKey,
     getDataKey,
     isProtectedSessionAvailable,
-    decryptNoteTitle,
+    encrypt,
-    decryptNote,
+    decrypt,
-    decryptNoteContent,
     decryptNotes,
-    decryptNoteRevision,
-    encryptNote,
-    encryptNoteContent,
-    encryptNoteRevision,
     setProtectedSessionId
 };

src/services/sync_table.js

@@ -4,42 +4,6 @@ const dateUtils = require('./date_utils');
 const log = require('./log');
 const cls = require('./cls');
 
-async function addNoteSync(noteId, sourceId) {
-    await addEntitySync("notes", noteId, sourceId)
-}
-
-async function addNoteContentSync(noteId, sourceId) {
-    await addEntitySync("note_contents", noteId, sourceId)
-}
-
-async function addBranchSync(branchId, sourceId) {
-    await addEntitySync("branches", branchId, sourceId)
-}
-
-async function addNoteReorderingSync(parentNoteId, sourceId) {
-    await addEntitySync("note_reordering", parentNoteId, sourceId)
-}
-
-async function addNoteRevisionSync(noteRevisionId, sourceId) {
-    await addEntitySync("note_revisions", noteRevisionId, sourceId);
-}
-
-async function addOptionsSync(name, sourceId) {
-    await addEntitySync("options", name, sourceId);
-}
-
-async function addRecentNoteSync(noteId, sourceId) {
-    await addEntitySync("recent_notes", noteId, sourceId);
-}
-
-async function addAttributeSync(attributeId, sourceId) {
-    await addEntitySync("attributes", attributeId, sourceId);
-}
-
-async function addApiTokenSync(apiTokenId, sourceId) {
-    await addEntitySync("api_tokens", apiTokenId, sourceId);
-}
-
 async function addEntitySync(entityName, entityId, sourceId) {
     await sql.replace("sync", {
         entityName: entityName,
@@ -107,15 +71,15 @@ async function fillAllSyncRows() {
 }
 
 module.exports = {
-    addNoteSync,
+    addNoteSync: async (noteId, sourceId) => await addEntitySync("notes", noteId, sourceId),
-    addNoteContentSync,
+    addNoteContentSync: async (noteId, sourceId) => await addEntitySync("note_contents", noteId, sourceId),
-    addBranchSync,
+    addBranchSync: async (branchId, sourceId) => await addEntitySync("branches", branchId, sourceId),
-    addNoteReorderingSync,
+    addNoteReorderingSync: async (parentNoteId, sourceId) => await addEntitySync("note_reordering", parentNoteId, sourceId),
-    addNoteRevisionSync,
+    addNoteRevisionSync: async (noteRevisionId, sourceId) => await addEntitySync("note_revisions", noteRevisionId, sourceId),
-    addOptionsSync,
+    addOptionsSync: async (name, sourceId) => await addEntitySync("options", name, sourceId),
-    addRecentNoteSync,
+    addRecentNoteSync: async (noteId, sourceId) => await addEntitySync("recent_notes", noteId, sourceId),
-    addAttributeSync,
+    addAttributeSync: async (attributeId, sourceId) => await addEntitySync("attributes", attributeId, sourceId),
-    addApiTokenSync,
+    addApiTokenSync: async (apiTokenId, sourceId) => await addEntitySync("api_tokens", apiTokenId, sourceId),
     addEntitySync,
     fillAllSyncRows
 };

src/services/sync_update.js

@@ -117,9 +117,9 @@ async function updateNoteRevision(entity, sourceId) {
 
 async function updateNoteReordering(entityId, entity, sourceId) {
     await sql.transactional(async () => {
-        Object.keys(entity).forEach(async key => {
+        for (const key in entity) {
             await sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]);
-        });
+        }
 
         await syncTableService.addNoteReorderingSync(entityId, sourceId);
     });

src/services/utils.js

@@ -154,6 +154,14 @@ function getContentDisposition(filename) {
     return `file; filename="${sanitizedFilename}"; filename*=UTF-8''${sanitizedFilename}`;
 }
 
+const STRING_MIME_TYPES = ["application/x-javascript"];
+
+function isStringNote(type, mime) {
+    return ["text", "code", "relation-map", "search"].includes(type)
+        || mime.startsWith('text/')
+        || STRING_MIME_TYPES.includes(mime);
+}
+
 module.exports = {
     randomSecureToken,
     randomString,
@@ -177,5 +185,6 @@ module.exports = {
     escapeRegExp,
     crash,
     sanitizeFilenameForHeader,
-    getContentDisposition
+    getContentDisposition,
+    isStringNote
 };