From 5a8e216deca95ed3742d50d54aaf09231e68316a Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 15 Mar 2023 22:44:08 +0100 Subject: [PATCH] WIP blob --- db/migrations/0214__content_structure.sql | 10 ++ .../0215__move_content_into_blobs.js | 63 +++++++ db/migrations/0216__drop_content_tables.sql | 2 + ...llaries.sql => 0217__note_ancillaries.sql} | 5 +- src/becca/becca_loader.js | 2 +- src/becca/entities/bnote.js | 84 ++++++---- src/becca/entities/bnote_revision.js | 61 +++---- src/public/app/entities/fnote_complement.js | 2 +- src/public/app/services/froca.js | 12 +- src/public/app/services/froca_updater.js | 6 +- src/public/app/services/load_results.js | 6 +- src/public/app/widgets/toc.js | 2 +- src/services/app_info.js | 2 +- src/services/consistency_checks.js | 154 +++++++++--------- src/services/utils.js | 17 +- src/services/ws.js | 5 +- 16 files changed, 275 insertions(+), 158 deletions(-) create mode 100644 db/migrations/0215__move_content_into_blobs.js create mode 100644 db/migrations/0216__drop_content_tables.sql rename db/migrations/{0215__note_ancillaries.sql => 0217__note_ancillaries.sql} (67%) diff --git a/db/migrations/0214__content_structure.sql b/db/migrations/0214__content_structure.sql index e69de29bb..9f599cec3 100644 --- a/db/migrations/0214__content_structure.sql +++ b/db/migrations/0214__content_structure.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS "blobs" ( + `blobId` TEXT NOT NULL, + `content` TEXT NULL DEFAULT NULL, + `dateModified` TEXT NOT NULL, + `utcDateModified` TEXT NOT NULL, + PRIMARY KEY(`blobId`) +); + +ALTER TABLE notes ADD blobId TEXT DEFAULT NULL; +ALTER TABLE note_revisions ADD blobId TEXT DEFAULT NULL; diff --git a/db/migrations/0215__move_content_into_blobs.js b/db/migrations/0215__move_content_into_blobs.js new file mode 100644 index 000000000..4bd6b6d46 --- /dev/null +++ b/db/migrations/0215__move_content_into_blobs.js @@ -0,0 +1,63 @@ +const sql = require("../../src/services/sql.js"); +module.exports = () => { + const sql = require("../../src/services/sql"); + const utils = require("../../src/services/utils"); + + const existingBlobIds = new Set(); + + for (const noteId of sql.getColumn(`SELECT noteId FROM note_contents`)) { + const row = sql.getRow(`SELECT noteId, content, dateModified, utcDateModified FROM note_contents WHERE noteId = ?`, [noteId]); + const blobId = utils.hashedBlobId(row.content); + + if (!existingBlobIds.has(blobId)) { + existingBlobIds.add(blobId); + + sql.insert('blobs', { + blobId, + content: row.content, + dateModified: row.dateModified, + utcDateModified: row.utcDateModified + }); + + sql.execute("UPDATE entity_changes SET entityName = 'blobs', entityId = ? WHERE entityName = 'note_contents' AND entityId = ?", [blobId, row.noteId]); + } else { + // duplicates + sql.execute("DELETE FROM entity_changes WHERE entityName = 'note_contents' AND entityId = ?", [row.noteId]); + } + + sql.execute('UPDATE notes SET blobId = ? WHERE noteId = ?', [blobId, row.noteId]); + } + + for (const noteRevisionId of sql.getColumn(`SELECT noteRevisionId FROM note_revision_contents`)) { + const row = sql.getRow(`SELECT noteRevisionId, content, utcDateModified FROM note_revision_contents WHERE noteRevisionId = ?`, [noteRevisionId]); + const blobId = utils.hashedBlobId(row.content); + + if (!existingBlobIds.has(blobId)) { + existingBlobIds.add(blobId); + + sql.insert('blobs', { + blobId, + content: row.content, + dateModified: row.utcDateModified, + utcDateModified: row.utcDateModified + }); + + sql.execute("UPDATE entity_changes SET entityName = 'blobs', entityId = ? WHERE entityName = 'note_revision_contents' AND entityId = ?", [blobId, row.noteRevisionId]); + } else { + // duplicates + sql.execute("DELETE FROM entity_changes WHERE entityName = 'note_revision_contents' AND entityId = ?", [row.noteId]); + } + + sql.execute('UPDATE note_revisions SET blobId = ? WHERE noteRevisionId = ?', [blobId, row.noteRevisionId]); + } + + const notesWithoutBlobIds = sql.getColumn("SELECT noteId FROM notes WHERE blobId IS NULL"); + if (notesWithoutBlobIds.length > 0) { + throw new Error("BlobIds were not filled correctly in notes: " + JSON.stringify(notesWithoutBlobIds)); + } + + const noteRevisionsWithoutBlobIds = sql.getColumn("SELECT noteRevisionId FROM note_revisions WHERE blobId IS NULL"); + if (noteRevisionsWithoutBlobIds.length > 0) { + throw new Error("BlobIds were not filled correctly in note revisions: " + JSON.stringify(noteRevisionsWithoutBlobIds)); + } +}; diff --git a/db/migrations/0216__drop_content_tables.sql b/db/migrations/0216__drop_content_tables.sql new file mode 100644 index 000000000..c0db8879d --- /dev/null +++ b/db/migrations/0216__drop_content_tables.sql @@ -0,0 +1,2 @@ +DROP TABLE note_contents; +DROP TABLE note_revision_contents; diff --git a/db/migrations/0215__note_ancillaries.sql b/db/migrations/0217__note_ancillaries.sql similarity index 67% rename from db/migrations/0215__note_ancillaries.sql rename to db/migrations/0217__note_ancillaries.sql index 03cadd8d8..26b9faa5d 100644 --- a/db/migrations/0215__note_ancillaries.sql +++ b/db/migrations/0217__note_ancillaries.sql @@ -6,14 +6,11 @@ CREATE TABLE IF NOT EXISTS "note_ancillaries" mime TEXT not null, isProtected INT not null DEFAULT 0, contentCheckSum TEXT not null, + blobId TEXT not null, utcDateModified TEXT not null, isDeleted INT not null, `deleteId` TEXT DEFAULT NULL); -CREATE TABLE IF NOT EXISTS "note_ancillary_contents" (`noteAncillaryId` TEXT NOT NULL PRIMARY KEY, - `content` TEXT DEFAULT NULL, - `utcDateModified` TEXT NOT NULL); - CREATE INDEX IDX_note_ancillaries_name on note_ancillaries (name); CREATE UNIQUE INDEX IDX_note_ancillaries_noteId_name diff --git a/src/becca/becca_loader.js b/src/becca/becca_loader.js index ad4c90a10..9ac087d44 100644 --- a/src/becca/becca_loader.js +++ b/src/becca/becca_loader.js @@ -30,7 +30,7 @@ function load() { // using raw query and passing arrays to avoid allocating new objects // this is worth it for becca load since it happens every run and blocks the app until finished - for (const row of sql.getRawRows(`SELECT noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) { + for (const row of sql.getRawRows(`SELECT noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) { new BNote().update(row).init(); } diff --git a/src/becca/entities/bnote.js b/src/becca/entities/bnote.js index 2187dc41e..8561c5c33 100644 --- a/src/becca/entities/bnote.js +++ b/src/becca/entities/bnote.js @@ -46,6 +46,7 @@ class BNote extends AbstractBeccaEntity { row.type, row.mime, row.isProtected, + row.blobId, row.dateCreated, row.dateModified, row.utcDateCreated, @@ -53,19 +54,21 @@ class BNote extends AbstractBeccaEntity { ]); } - update([noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified]) { + update([noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified]) { // ------ Database persisted attributes ------ /** @type {string} */ this.noteId = noteId; /** @type {string} */ this.title = title; - /** @type {boolean} */ - this.isProtected = !!isProtected; /** @type {string} */ this.type = type; /** @type {string} */ this.mime = mime; + /** @type {boolean} */ + this.isProtected = !!isProtected; + /** @type {string} */ + this.blobId = blobId; /** @type {string} */ this.dateCreated = dateCreated || dateUtils.localNowDateTime(); /** @type {string} */ @@ -206,14 +209,14 @@ class BNote extends AbstractBeccaEntity { /** @returns {*} */ getContent(silentNotFoundError = false) { - const row = sql.getRow(`SELECT content FROM note_contents WHERE noteId = ?`, [this.noteId]); + const row = sql.getRow(`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]); if (!row) { if (silentNotFoundError) { return undefined; } else { - throw new Error(`Cannot find note content for noteId=${this.noteId}`); + throw new Error(`Cannot find note content for noteId '${this.noteId}', blobId '${this.blobId}'.`); } } @@ -245,8 +248,8 @@ class BNote extends AbstractBeccaEntity { LENGTH(content) AS contentLength, dateModified, utcDateModified - FROM note_contents - WHERE noteId = ?`, [this.noteId]); + FROM blobs + WHERE blobId = ?`, [this.blobId]); } get dateCreatedObj() { @@ -276,6 +279,10 @@ class BNote extends AbstractBeccaEntity { return JSON.parse(content); } + isHot() { + return ['text', 'code', 'relationMap', 'canvas', 'mermaid'].includes(this.type); + } + setContent(content, ignoreMissingProtectedSession = false) { if (content === null || content === undefined) { throw new Error(`Cannot set null content to note '${this.noteId}'`); @@ -288,39 +295,57 @@ class BNote extends AbstractBeccaEntity { content = Buffer.isBuffer(content) ? content : Buffer.from(content); } - const pojo = { - noteId: this.noteId, - content: content, - dateModified: dateUtils.localNowDateTime(), - utcDateModified: dateUtils.utcNowDateTime() - }; - if (this.isProtected) { if (protectedSessionService.isProtectedSessionAvailable()) { - pojo.content = protectedSessionService.encrypt(pojo.content); + content = protectedSessionService.encrypt(content); } else if (!ignoreMissingProtectedSession) { throw new Error(`Cannot update content of noteId '${this.noteId}' since we're out of protected session.`); } } - sql.upsert("note_contents", "noteId", pojo); + let newBlobId; + let blobNeedsInsert; - const hash = utils.hash(`${this.noteId}|${pojo.content.toString()}`); + if (this.isHot()) { + newBlobId = this.blobId || utils.randomBlobId(); + blobNeedsInsert = true; + } else { + newBlobId = utils.hashedBlobId(content); + blobNeedsInsert = !sql.getValue('SELECT 1 FROM blobs WHERE blobId = ?', [newBlobId]); + } - entityChangesService.addEntityChange({ - entityName: 'note_contents', - entityId: this.noteId, - hash: hash, - isErased: false, - utcDateChanged: pojo.utcDateModified, - isSynced: true - }); + if (blobNeedsInsert) { + const pojo = { + blobId: this.blobId, + content: content, + dateModified: dateUtils.localNowDateTime(), + utcDateModified: dateUtils.utcNowDateTime() + }; - eventService.emit(eventService.ENTITY_CHANGED, { - entityName: 'note_contents', - entity: this - }); + sql.upsert("blobs", "blobId", pojo); + + const hash = utils.hash(`${this.blobId}|${pojo.content.toString()}`); + + entityChangesService.addEntityChange({ + entityName: 'blobs', + entityId: this.blobId, + hash: hash, + isErased: false, + utcDateChanged: pojo.utcDateModified, + isSynced: true + }); + + eventService.emit(eventService.ENTITY_CHANGED, { + entityName: 'blobs', + entity: this + }); + } + + if (newBlobId !== this.blobId) { + this.blobId = newBlobId; + this.save(); + } } setJsonContent(content) { @@ -1517,6 +1542,7 @@ class BNote extends AbstractBeccaEntity { isProtected: this.isProtected, type: this.type, mime: this.mime, + blobId: this.blobId, isDeleted: false, dateCreated: this.dateCreated, dateModified: this.dateModified, diff --git a/src/becca/entities/bnote_revision.js b/src/becca/entities/bnote_revision.js index b2153ca09..c5f5c466f 100644 --- a/src/becca/entities/bnote_revision.js +++ b/src/becca/entities/bnote_revision.js @@ -35,6 +35,8 @@ class BNoteRevision extends AbstractBeccaEntity { /** @type {string} */ this.title = row.title; /** @type {string} */ + this.blobId = row.blobId; + /** @type {string} */ this.dateLastEdited = row.dateLastEdited; /** @type {string} */ this.dateCreated = row.dateCreated; @@ -74,14 +76,14 @@ class BNoteRevision extends AbstractBeccaEntity { /** @returns {*} */ getContent(silentNotFoundError = false) { - const res = sql.getRow(`SELECT content FROM note_revision_contents WHERE noteRevisionId = ?`, [this.noteRevisionId]); + const res = sql.getRow(`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]); if (!res) { if (silentNotFoundError) { return undefined; } else { - throw new Error(`Cannot find note revision content for noteRevisionId=${this.noteRevisionId}`); + throw new Error(`Cannot find note revision content for noteRevisionId '${this.noteRevisionId}', blobId '${this.blobId}'`); } } @@ -107,44 +109,42 @@ class BNoteRevision extends AbstractBeccaEntity { } setContent(content) { - const pojo = { - noteRevisionId: this.noteRevisionId, - content: content, - utcDateModified: dateUtils.utcNowDateTime() - }; - if (this.isProtected) { if (protectedSessionService.isProtectedSessionAvailable()) { - pojo.content = protectedSessionService.encrypt(pojo.content); + content = protectedSessionService.encrypt(content); } else { - throw new Error(`Cannot update content of noteRevisionId=${this.noteRevisionId} since we're out of protected session.`); + throw new Error(`Cannot update content of noteRevisionId '${this.noteRevisionId}' since we're out of protected session.`); } } - sql.upsert("note_revision_contents", "noteRevisionId", pojo); + this.blobId = utils.hashedBlobId(content); - const hash = utils.hash(`${this.noteRevisionId}|${pojo.content.toString()}`); + const blobAlreadyExists = !sql.getValue('SELECT 1 FROM blobs WHERE blobId = ?', [this.blobId]); - entityChangesService.addEntityChange({ - entityName: 'note_revision_contents', - entityId: this.noteRevisionId, - hash: hash, - isErased: false, - utcDateChanged: this.getUtcDateChanged(), - isSynced: true - }); - } + if (!blobAlreadyExists) { + const pojo = { + blobId: this.blobId, + content: content, + dateModified: dateUtils.localNowDate(), + utcDateModified: dateUtils.utcNowDateTime() + }; - /** @returns {{contentLength, dateModified, utcDateModified}} */ - getContentMetadata() { - return sql.getRow(` - SELECT - LENGTH(content) AS contentLength, - dateModified, - utcDateModified - FROM note_revision_contents - WHERE noteRevisionId = ?`, [this.noteRevisionId]); + sql.insert("blobs", pojo); + + const hash = utils.hash(`${this.noteRevisionId}|${pojo.content.toString()}`); + + entityChangesService.addEntityChange({ + entityName: 'blobs', + entityId: this.blobId, + hash: hash, + isErased: false, + utcDateChanged: this.getUtcDateChanged(), + isSynced: true + }); + } + + this.save(); // saving this.blobId } beforeSaving() { @@ -161,6 +161,7 @@ class BNoteRevision extends AbstractBeccaEntity { mime: this.mime, isProtected: this.isProtected, title: this.title, + blobId: this.blobId, dateLastEdited: this.dateLastEdited, dateCreated: this.dateCreated, utcDateLastEdited: this.utcDateLastEdited, diff --git a/src/public/app/entities/fnote_complement.js b/src/public/app/entities/fnote_complement.js index 5f4d83abc..4884e757a 100644 --- a/src/public/app/entities/fnote_complement.js +++ b/src/public/app/entities/fnote_complement.js @@ -27,7 +27,7 @@ class FNoteComplement { /** @type {string} */ this.utcDateModified = row.utcDateModified; - // "combined" date modified give larger out of note's and note_content's dateModified + // "combined" date modified give larger out of note's and blob's dateModified /** @type {string} */ this.combinedDateModified = row.combinedDateModified; diff --git a/src/public/app/services/froca.js b/src/public/app/services/froca.js index 8bedf2fa6..d20a55578 100644 --- a/src/public/app/services/froca.js +++ b/src/public/app/services/froca.js @@ -35,7 +35,7 @@ class Froca { this.attributes = {}; /** @type {Object.>} */ - this.noteComplementPromises = {}; + this.blobPromises = {}; this.addResp(resp); } @@ -314,20 +314,20 @@ class Froca { * @returns {Promise} */ async getNoteComplement(noteId) { - if (!this.noteComplementPromises[noteId]) { - this.noteComplementPromises[noteId] = server.get(`notes/${noteId}`) + if (!this.blobPromises[noteId]) { + this.blobPromises[noteId] = server.get(`notes/${noteId}`) .then(row => new FNoteComplement(row)) .catch(e => console.error(`Cannot get note complement for note '${noteId}'`)); // we don't want to keep large payloads forever in memory, so we clean that up quite quickly // this cache is more meant to share the data between different components within one business transaction (e.g. loading of the note into the tab context and all the components) // this is also a workaround for missing invalidation after change - this.noteComplementPromises[noteId].then( - () => setTimeout(() => this.noteComplementPromises[noteId] = null, 1000) + this.blobPromises[noteId].then( + () => setTimeout(() => this.blobPromises[noteId] = null, 1000) ); } - return await this.noteComplementPromises[noteId]; + return await this.blobPromises[noteId]; } } diff --git a/src/public/app/services/froca_updater.js b/src/public/app/services/froca_updater.js index a858f64ea..cb8a7248b 100644 --- a/src/public/app/services/froca_updater.js +++ b/src/public/app/services/froca_updater.js @@ -19,10 +19,10 @@ async function processEntityChanges(entityChanges) { processAttributeChange(loadResults, ec); } else if (ec.entityName === 'note_reordering') { processNoteReordering(loadResults, ec); - } else if (ec.entityName === 'note_contents') { - delete froca.noteComplementPromises[ec.entityId]; + } else if (ec.entityName === 'blobs') { + delete froca.blobPromises[ec.entityId]; - loadResults.addNoteContent(ec.entityId, ec.componentId); + loadResults.addNoteContent(ec.noteIds, ec.componentId); } else if (ec.entityName === 'note_revisions') { loadResults.addNoteRevision(ec.entityId, ec.noteId, ec.componentId); } else if (ec.entityName === 'note_revision_contents') { diff --git a/src/public/app/services/load_results.js b/src/public/app/services/load_results.js index 5069c4a2a..6e8fe4245 100644 --- a/src/public/app/services/load_results.js +++ b/src/public/app/services/load_results.js @@ -94,8 +94,10 @@ export default class LoadResults { return componentIds && componentIds.find(sId => sId !== componentId) !== undefined; } - addNoteContent(noteId, componentId) { - this.contentNoteIdToComponentId.push({noteId, componentId}); + addNoteContent(noteIds, componentId) { + for (const noteId of noteIds) { + this.contentNoteIdToComponentId.push({noteId, componentId}); + } } isNoteContentReloaded(noteId, componentId) { diff --git a/src/public/app/widgets/toc.js b/src/public/app/widgets/toc.js index 5e826cf1f..3efb880ed 100644 --- a/src/public/app/widgets/toc.js +++ b/src/public/app/widgets/toc.js @@ -302,4 +302,4 @@ class CloseTocButton extends OnClickButtonWidget { }) .class("icon-action close-toc"); } -} \ No newline at end of file +} diff --git a/src/services/app_info.js b/src/services/app_info.js index 7a142891c..667b156a2 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -4,7 +4,7 @@ const build = require('./build'); const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); -const APP_DB_VERSION = 215; +const APP_DB_VERSION = 217; const SYNC_VERSION = 30; const CLIPPER_PROTOCOL_VERSION = "1.0"; diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index 6bb6e066d..696a89c49 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -383,86 +383,86 @@ class ConsistencyChecks { } }); - this.findAndFixIssues(` - SELECT notes.noteId, notes.isProtected, notes.type, notes.mime - FROM notes - LEFT JOIN note_contents USING (noteId) - WHERE note_contents.noteId IS NULL`, - ({noteId, isProtected, type, mime}) => { - if (this.autoFix) { - // it might be possible that the note_content is not available only because of the interrupted - // sync, and it will come later. It's therefore important to guarantee that this artifical - // record won't overwrite the real one coming from the sync. - const fakeDate = "2000-01-01 00:00:00Z"; - - // manually creating row since this can also affect deleted notes - sql.upsert("note_contents", "noteId", { - noteId: noteId, - content: getBlankContent(isProtected, type, mime), - utcDateModified: fakeDate, - dateModified: fakeDate - }); - - const hash = utils.hash(utils.randomString(10)); - - entityChangesService.addEntityChange({ - entityName: 'note_contents', - entityId: noteId, - hash: hash, - isErased: false, - utcDateChanged: fakeDate, - isSynced: true - }); - - this.reloadNeeded = true; - - logFix(`Note '${noteId}' content was set to empty string since there was no corresponding row`); - } else { - logError(`Note '${noteId}' content row does not exist`); - } - }); + // this.findAndFixIssues(` + // SELECT notes.noteId, notes.isProtected, notes.type, notes.mime + // FROM notes + // LEFT JOIN note_contents USING (noteId) + // WHERE note_contents.noteId IS NULL`, + // ({noteId, isProtected, type, mime}) => { + // if (this.autoFix) { + // // it might be possible that the note_content is not available only because of the interrupted + // // sync, and it will come later. It's therefore important to guarantee that this artifical + // // record won't overwrite the real one coming from the sync. + // const fakeDate = "2000-01-01 00:00:00Z"; + // + // // manually creating row since this can also affect deleted notes + // sql.upsert("note_contents", "noteId", { + // noteId: noteId, + // content: getBlankContent(isProtected, type, mime), + // utcDateModified: fakeDate, + // dateModified: fakeDate + // }); + // + // const hash = utils.hash(utils.randomString(10)); + // + // entityChangesService.addEntityChange({ + // entityName: 'note_contents', + // entityId: noteId, + // hash: hash, + // isErased: false, + // utcDateChanged: fakeDate, + // isSynced: true + // }); + // + // this.reloadNeeded = true; + // + // logFix(`Note '${noteId}' content was set to empty string since there was no corresponding row`); + // } else { + // logError(`Note '${noteId}' content row does not exist`); + // } + // }); if (sqlInit.getDbSize() < 500000) { // querying for "content IS NULL" is expensive since content is not indexed. See e.g. https://github.com/zadam/trilium/issues/2887 - this.findAndFixIssues(` - SELECT notes.noteId, notes.type, notes.mime - FROM notes - JOIN note_contents USING (noteId) - WHERE isDeleted = 0 - AND isProtected = 0 - AND content IS NULL`, - ({noteId, type, mime}) => { - if (this.autoFix) { - const note = becca.getNote(noteId); - const blankContent = getBlankContent(false, type, mime); - note.setContent(blankContent); - - this.reloadNeeded = true; - - logFix(`Note '${noteId}' content was set to '${blankContent}' since it was null even though it is not deleted`); - } else { - logError(`Note '${noteId}' content is null even though it is not deleted`); - } - }); + // this.findAndFixIssues(` + // SELECT notes.noteId, notes.type, notes.mime + // FROM notes + // JOIN note_contents USING (noteId) + // WHERE isDeleted = 0 + // AND isProtected = 0 + // AND content IS NULL`, + // ({noteId, type, mime}) => { + // if (this.autoFix) { + // const note = becca.getNote(noteId); + // const blankContent = getBlankContent(false, type, mime); + // note.setContent(blankContent); + // + // this.reloadNeeded = true; + // + // logFix(`Note '${noteId}' content was set to '${blankContent}' since it was null even though it is not deleted`); + // } else { + // logError(`Note '${noteId}' content is null even though it is not deleted`); + // } + // }); } - this.findAndFixIssues(` - SELECT note_revisions.noteRevisionId - FROM note_revisions - LEFT JOIN note_revision_contents USING (noteRevisionId) - WHERE note_revision_contents.noteRevisionId IS NULL`, - ({noteRevisionId}) => { - if (this.autoFix) { - noteRevisionService.eraseNoteRevisions([noteRevisionId]); - - this.reloadNeeded = true; - - logFix(`Note revision content '${noteRevisionId}' was set to erased since it did not exist.`); - } else { - logError(`Note revision content '${noteRevisionId}' does not exist`); - } - }); + // this.findAndFixIssues(` + // SELECT note_revisions.noteRevisionId + // FROM note_revisions + // LEFT JOIN note_revision_contents USING (noteRevisionId) + // WHERE note_revision_contents.noteRevisionId IS NULL`, + // ({noteRevisionId}) => { + // if (this.autoFix) { + // noteRevisionService.eraseNoteRevisions([noteRevisionId]); + // + // this.reloadNeeded = true; + // + // logFix(`Note revision content '${noteRevisionId}' was set to erased since it did not exist.`); + // } else { + // logError(`Note revision content '${noteRevisionId}' does not exist`); + // } + // }); this.findAndFixIssues(` SELECT parentNoteId @@ -656,11 +656,11 @@ class ConsistencyChecks { findEntityChangeIssues() { this.runEntityChangeChecks("notes", "noteId"); - this.runEntityChangeChecks("note_contents", "noteId"); + //this.runEntityChangeChecks("note_contents", "noteId"); this.runEntityChangeChecks("note_revisions", "noteRevisionId"); - this.runEntityChangeChecks("note_revision_contents", "noteRevisionId"); + //this.runEntityChangeChecks("note_revision_contents", "noteRevisionId"); this.runEntityChangeChecks("note_ancillaries", "noteAncillaryId"); - this.runEntityChangeChecks("note_ancillary_contents", "noteAncillaryId"); + //this.runEntityChangeChecks("note_ancillary_contents", "noteAncillaryId"); this.runEntityChangeChecks("branches", "branchId"); this.runEntityChangeChecks("attributes", "attributeId"); this.runEntityChangeChecks("etapi_tokens", "etapiTokenId"); diff --git a/src/services/utils.js b/src/services/utils.js index a50f443ef..432d69013 100644 --- a/src/services/utils.js +++ b/src/services/utils.js @@ -25,6 +25,19 @@ function md5(content) { return crypto.createHash('md5').update(content).digest('hex'); } +function hashedBlobId(content) { + // sha512 is faster than sha256 + const base64Hash = crypto.createHash('sha512').update(content).digest('base64'); + + // 20 characters of base64 gives us 120 bit of entropy which is plenty enough + return base64Hash.substr(0, 20); +} + +function randomBlobId(content) { + // underscore prefix to easily differentiate the random as opposed to hashed + return '_' + randomString(19); +} + function toBase64(plainText) { return Buffer.from(plainText).toString('base64'); } @@ -343,5 +356,7 @@ module.exports = { deferred, removeDiacritic, normalize, - filterAttributeName + filterAttributeName, + hashedBlobId, + randomBlobId }; diff --git a/src/services/ws.js b/src/services/ws.js index ac81c367d..8c1f34dd4 100644 --- a/src/services/ws.js +++ b/src/services/ws.js @@ -129,13 +129,14 @@ function fillInAdditionalProperties(entityChange) { entityChange.positions[childBranch.branchId] = childBranch.notePosition; } } - } - else if (entityChange.entityName === 'options') { + } else if (entityChange.entityName === 'options') { entityChange.entity = becca.getOption(entityChange.entityId); if (!entityChange.entity) { entityChange.entity = sql.getRow(`SELECT * FROM options WHERE name = ?`, [entityChange.entityId]); } + } else if (entityChange.entityName === 'blob') { + entityChange.noteIds = sql.getColumn("SELECT noteId FROM notes WHERE blobId = ? AND isDeleted = 0", [entityChange.entityId]); } if (entityChange.entity instanceof AbstractBeccaEntity) {