gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

WIP blob

Browse Source
This commit is contained in:
zadam 2023-03-15 22:44:08 +01:00
parent 1faf8225c7
commit 5a8e216dec
16 changed files with 275 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
db/migrations/0214__content_structure.sql
View File

@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS "blobs" (
`blobId` TEXT NOT NULL,
`content` TEXT NULL DEFAULT NULL,
`dateModified` TEXT NOT NULL,
`utcDateModified` TEXT NOT NULL,
PRIMARY KEY(`blobId`)
);
ALTER TABLE notes ADD blobId TEXT DEFAULT NULL;
ALTER TABLE note_revisions ADD blobId TEXT DEFAULT NULL;

63
db/migrations/0215__move_content_into_blobs.js Normal file
View File

@ -0,0 +1,63 @@
const sql = require("../../src/services/sql.js");
module.exports = () => {
const sql = require("../../src/services/sql");
const utils = require("../../src/services/utils");
const existingBlobIds = new Set();
for (const noteId of sql.getColumn(`SELECT noteId FROM note_contents`)) {
const row = sql.getRow(`SELECT noteId, content, dateModified, utcDateModified FROM note_contents WHERE noteId = ?`, [noteId]);
const blobId = utils.hashedBlobId(row.content);
if (!existingBlobIds.has(blobId)) {
existingBlobIds.add(blobId);
sql.insert('blobs', {
blobId,
content: row.content,
dateModified: row.dateModified,
utcDateModified: row.utcDateModified
});
sql.execute("UPDATE entity_changes SET entityName = 'blobs', entityId = ? WHERE entityName = 'note_contents' AND entityId = ?", [blobId, row.noteId]);
} else {
// duplicates
sql.execute("DELETE FROM entity_changes WHERE entityName = 'note_contents' AND entityId = ?", [row.noteId]);
}
sql.execute('UPDATE notes SET blobId = ? WHERE noteId = ?', [blobId, row.noteId]);
}
for (const noteRevisionId of sql.getColumn(`SELECT noteRevisionId FROM note_revision_contents`)) {
const row = sql.getRow(`SELECT noteRevisionId, content, utcDateModified FROM note_revision_contents WHERE noteRevisionId = ?`, [noteRevisionId]);
const blobId = utils.hashedBlobId(row.content);
if (!existingBlobIds.has(blobId)) {
existingBlobIds.add(blobId);
sql.insert('blobs', {
blobId,
content: row.content,
dateModified: row.utcDateModified,
utcDateModified: row.utcDateModified
});
sql.execute("UPDATE entity_changes SET entityName = 'blobs', entityId = ? WHERE entityName = 'note_revision_contents' AND entityId = ?", [blobId, row.noteRevisionId]);
} else {
// duplicates
sql.execute("DELETE FROM entity_changes WHERE entityName = 'note_revision_contents' AND entityId = ?", [row.noteId]);
}
sql.execute('UPDATE note_revisions SET blobId = ? WHERE noteRevisionId = ?', [blobId, row.noteRevisionId]);
}
const notesWithoutBlobIds = sql.getColumn("SELECT noteId FROM notes WHERE blobId IS NULL");
if (notesWithoutBlobIds.length > 0) {
throw new Error("BlobIds were not filled correctly in notes: " + JSON.stringify(notesWithoutBlobIds));
}
const noteRevisionsWithoutBlobIds = sql.getColumn("SELECT noteRevisionId FROM note_revisions WHERE blobId IS NULL");
if (noteRevisionsWithoutBlobIds.length > 0) {
throw new Error("BlobIds were not filled correctly in note revisions: " + JSON.stringify(noteRevisionsWithoutBlobIds));
}
};

2
db/migrations/0216__drop_content_tables.sql Normal file
View File

@ -0,0 +1,2 @@
DROP TABLE note_contents;
DROP TABLE note_revision_contents;

5
db/migrations/0215__note_ancillaries.sql → db/migrations/0217__note_ancillaries.sql
View File

@ -6,14 +6,11 @@ CREATE TABLE IF NOT EXISTS "note_ancillaries"
mime TEXT not null, mime TEXT not null,
isProtected INT not null DEFAULT 0, isProtected INT not null DEFAULT 0,
contentCheckSum TEXT not null, contentCheckSum TEXT not null,
blobId TEXT not null,
utcDateModified TEXT not null, utcDateModified TEXT not null,
isDeleted INT not null, isDeleted INT not null,
`deleteId` TEXT DEFAULT NULL); `deleteId` TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "note_ancillary_contents" (`noteAncillaryId` TEXT NOT NULL PRIMARY KEY,
`content` TEXT DEFAULT NULL,
`utcDateModified` TEXT NOT NULL);
CREATE INDEX IDX_note_ancillaries_name CREATE INDEX IDX_note_ancillaries_name
on note_ancillaries (name); on note_ancillaries (name);
CREATE UNIQUE INDEX IDX_note_ancillaries_noteId_name CREATE UNIQUE INDEX IDX_note_ancillaries_noteId_name

2
src/becca/becca_loader.js
View File

@ -30,7 +30,7 @@ function load() {
// using raw query and passing arrays to avoid allocating new objects // using raw query and passing arrays to avoid allocating new objects
// this is worth it for becca load since it happens every run and blocks the app until finished // this is worth it for becca load since it happens every run and blocks the app until finished
for (const row of sql.getRawRows(`SELECT noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) { for (const row of sql.getRawRows(`SELECT noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) {
new BNote().update(row).init(); new BNote().update(row).init();
} }

84
src/becca/entities/bnote.js
View File

@ -46,6 +46,7 @@ class BNote extends AbstractBeccaEntity {
row.type, row.type,
row.mime, row.mime,
row.isProtected, row.isProtected,
row.blobId,
row.dateCreated, row.dateCreated,
row.dateModified, row.dateModified,
row.utcDateCreated, row.utcDateCreated,
@ -53,19 +54,21 @@ class BNote extends AbstractBeccaEntity {
]); ]);
} }
update([noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified]) { update([noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified]) {
// ------ Database persisted attributes ------ // ------ Database persisted attributes ------
/** @type {string} */ /** @type {string} */
this.noteId = noteId; this.noteId = noteId;
/** @type {string} */ /** @type {string} */
this.title = title; this.title = title;
/** @type {boolean} */
this.isProtected = !!isProtected;
/** @type {string} */ /** @type {string} */
this.type = type; this.type = type;
/** @type {string} */ /** @type {string} */
this.mime = mime; this.mime = mime;
/** @type {boolean} */
this.isProtected = !!isProtected;
/** @type {string} */
this.blobId = blobId;
/** @type {string} */ /** @type {string} */
this.dateCreated = dateCreated || dateUtils.localNowDateTime(); this.dateCreated = dateCreated || dateUtils.localNowDateTime();
/** @type {string} */ /** @type {string} */
@ -206,14 +209,14 @@ class BNote extends AbstractBeccaEntity {
/** @returns {*} */ /** @returns {*} */
getContent(silentNotFoundError = false) { getContent(silentNotFoundError = false) {
const row = sql.getRow(`SELECT content FROM note_contents WHERE noteId = ?`, [this.noteId]); const row = sql.getRow(`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]);
if (!row) { if (!row) {
if (silentNotFoundError) { if (silentNotFoundError) {
return undefined; return undefined;
} }
else { else {
throw new Error(`Cannot find note content for noteId=${this.noteId}`); throw new Error(`Cannot find note content for noteId '${this.noteId}', blobId '${this.blobId}'.`);
} }
} }
@ -245,8 +248,8 @@ class BNote extends AbstractBeccaEntity {
LENGTH(content) AS contentLength, LENGTH(content) AS contentLength,
dateModified, dateModified,
utcDateModified utcDateModified
FROM note_contents FROM blobs
WHERE noteId = ?`, [this.noteId]); WHERE blobId = ?`, [this.blobId]);
} }
get dateCreatedObj() { get dateCreatedObj() {
@ -276,6 +279,10 @@ class BNote extends AbstractBeccaEntity {
return JSON.parse(content); return JSON.parse(content);
} }
isHot() {
return ['text', 'code', 'relationMap', 'canvas', 'mermaid'].includes(this.type);
}
setContent(content, ignoreMissingProtectedSession = false) { setContent(content, ignoreMissingProtectedSession = false) {
if (content === null || content === undefined) { if (content === null || content === undefined) {
throw new Error(`Cannot set null content to note '${this.noteId}'`); throw new Error(`Cannot set null content to note '${this.noteId}'`);
@ -288,39 +295,57 @@ class BNote extends AbstractBeccaEntity {
content = Buffer.isBuffer(content) ? content : Buffer.from(content); content = Buffer.isBuffer(content) ? content : Buffer.from(content);
} }
const pojo = {
noteId: this.noteId,
content: content,
dateModified: dateUtils.localNowDateTime(),
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) { if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) { if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content); content = protectedSessionService.encrypt(content);
} }
else if (!ignoreMissingProtectedSession) { else if (!ignoreMissingProtectedSession) {
throw new Error(`Cannot update content of noteId '${this.noteId}' since we're out of protected session.`); throw new Error(`Cannot update content of noteId '${this.noteId}' since we're out of protected session.`);
} }
} }
sql.upsert("note_contents", "noteId", pojo); let newBlobId;
let blobNeedsInsert;
const hash = utils.hash(`${this.noteId}|${pojo.content.toString()}`); if (this.isHot()) {
newBlobId = this.blobId || utils.randomBlobId();
blobNeedsInsert = true;
} else {
newBlobId = utils.hashedBlobId(content);
blobNeedsInsert = !sql.getValue('SELECT 1 FROM blobs WHERE blobId = ?', [newBlobId]);
}
entityChangesService.addEntityChange({ if (blobNeedsInsert) {
entityName: 'note_contents', const pojo = {
entityId: this.noteId, blobId: this.blobId,
hash: hash, content: content,
isErased: false, dateModified: dateUtils.localNowDateTime(),
utcDateChanged: pojo.utcDateModified, utcDateModified: dateUtils.utcNowDateTime()
isSynced: true };
});
eventService.emit(eventService.ENTITY_CHANGED, { sql.upsert("blobs", "blobId", pojo);
entityName: 'note_contents',
entity: this const hash = utils.hash(`${this.blobId}|${pojo.content.toString()}`);
});
entityChangesService.addEntityChange({
entityName: 'blobs',
entityId: this.blobId,
hash: hash,
isErased: false,
utcDateChanged: pojo.utcDateModified,
isSynced: true
});
eventService.emit(eventService.ENTITY_CHANGED, {
entityName: 'blobs',
entity: this
});
}
if (newBlobId !== this.blobId) {
this.blobId = newBlobId;
this.save();
}
} }
setJsonContent(content) { setJsonContent(content) {
@ -1517,6 +1542,7 @@ class BNote extends AbstractBeccaEntity {
isProtected: this.isProtected, isProtected: this.isProtected,
type: this.type, type: this.type,
mime: this.mime, mime: this.mime,
blobId: this.blobId,
isDeleted: false, isDeleted: false,
dateCreated: this.dateCreated, dateCreated: this.dateCreated,
dateModified: this.dateModified, dateModified: this.dateModified,

61
src/becca/entities/bnote_revision.js
View File

@ -35,6 +35,8 @@ class BNoteRevision extends AbstractBeccaEntity {
/** @type {string} */ /** @type {string} */
this.title = row.title; this.title = row.title;
/** @type {string} */ /** @type {string} */
this.blobId = row.blobId;
/** @type {string} */
this.dateLastEdited = row.dateLastEdited; this.dateLastEdited = row.dateLastEdited;
/** @type {string} */ /** @type {string} */
this.dateCreated = row.dateCreated; this.dateCreated = row.dateCreated;
@ -74,14 +76,14 @@ class BNoteRevision extends AbstractBeccaEntity {
/** @returns {*} */ /** @returns {*} */
getContent(silentNotFoundError = false) { getContent(silentNotFoundError = false) {
const res = sql.getRow(`SELECT content FROM note_revision_contents WHERE noteRevisionId = ?`, [this.noteRevisionId]); const res = sql.getRow(`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]);
if (!res) { if (!res) {
if (silentNotFoundError) { if (silentNotFoundError) {
return undefined; return undefined;
} }
else { else {
throw new Error(`Cannot find note revision content for noteRevisionId=${this.noteRevisionId}`); throw new Error(`Cannot find note revision content for noteRevisionId '${this.noteRevisionId}', blobId '${this.blobId}'`);
} }
} }
@ -107,44 +109,42 @@ class BNoteRevision extends AbstractBeccaEntity {
} }
setContent(content) { setContent(content) {
const pojo = {
noteRevisionId: this.noteRevisionId,
content: content,
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) { if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) { if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content); content = protectedSessionService.encrypt(content);
} }
else { else {
throw new Error(`Cannot update content of noteRevisionId=${this.noteRevisionId} since we're out of protected session.`); throw new Error(`Cannot update content of noteRevisionId '${this.noteRevisionId}' since we're out of protected session.`);
} }
} }
sql.upsert("note_revision_contents", "noteRevisionId", pojo); this.blobId = utils.hashedBlobId(content);
const hash = utils.hash(`${this.noteRevisionId}|${pojo.content.toString()}`); const blobAlreadyExists = !sql.getValue('SELECT 1 FROM blobs WHERE blobId = ?', [this.blobId]);
entityChangesService.addEntityChange({ if (!blobAlreadyExists) {
entityName: 'note_revision_contents', const pojo = {
entityId: this.noteRevisionId, blobId: this.blobId,
hash: hash, content: content,
isErased: false, dateModified: dateUtils.localNowDate(),
utcDateChanged: this.getUtcDateChanged(), utcDateModified: dateUtils.utcNowDateTime()
isSynced: true };
});
}
/** @returns {{contentLength, dateModified, utcDateModified}} */ sql.insert("blobs", pojo);
getContentMetadata() {
return sql.getRow(` const hash = utils.hash(`${this.noteRevisionId}|${pojo.content.toString()}`);
SELECT
LENGTH(content) AS contentLength, entityChangesService.addEntityChange({
dateModified, entityName: 'blobs',
utcDateModified entityId: this.blobId,
FROM note_revision_contents hash: hash,
WHERE noteRevisionId = ?`, [this.noteRevisionId]); isErased: false,
utcDateChanged: this.getUtcDateChanged(),
isSynced: true
});
}
this.save(); // saving this.blobId
} }
beforeSaving() { beforeSaving() {
@ -161,6 +161,7 @@ class BNoteRevision extends AbstractBeccaEntity {
mime: this.mime, mime: this.mime,
isProtected: this.isProtected, isProtected: this.isProtected,
title: this.title, title: this.title,
blobId: this.blobId,
dateLastEdited: this.dateLastEdited, dateLastEdited: this.dateLastEdited,
dateCreated: this.dateCreated, dateCreated: this.dateCreated,
utcDateLastEdited: this.utcDateLastEdited, utcDateLastEdited: this.utcDateLastEdited,

2
src/public/app/entities/fnote_complement.js
View File

@ -27,7 +27,7 @@ class FNoteComplement {
/** @type {string} */ /** @type {string} */
this.utcDateModified = row.utcDateModified; this.utcDateModified = row.utcDateModified;
// "combined" date modified give larger out of note's and note_content's dateModified // "combined" date modified give larger out of note's and blob's dateModified
/** @type {string} */ /** @type {string} */
this.combinedDateModified = row.combinedDateModified; this.combinedDateModified = row.combinedDateModified;

12
src/public/app/services/froca.js
View File

@ -35,7 +35,7 @@ class Froca {
this.attributes = {}; this.attributes = {};
/** @type {Object.<string, Promise<FNoteComplement>>} */ /** @type {Object.<string, Promise<FNoteComplement>>} */
this.noteComplementPromises = {}; this.blobPromises = {};
this.addResp(resp); this.addResp(resp);
} }
@ -314,20 +314,20 @@ class Froca {
* @returns {Promise<FNoteComplement>} * @returns {Promise<FNoteComplement>}
*/ */
async getNoteComplement(noteId) { async getNoteComplement(noteId) {
if (!this.noteComplementPromises[noteId]) { if (!this.blobPromises[noteId]) {
this.noteComplementPromises[noteId] = server.get(`notes/${noteId}`) this.blobPromises[noteId] = server.get(`notes/${noteId}`)
.then(row => new FNoteComplement(row)) .then(row => new FNoteComplement(row))
.catch(e => console.error(`Cannot get note complement for note '${noteId}'`)); .catch(e => console.error(`Cannot get note complement for note '${noteId}'`));
// we don't want to keep large payloads forever in memory, so we clean that up quite quickly // we don't want to keep large payloads forever in memory, so we clean that up quite quickly
// this cache is more meant to share the data between different components within one business transaction (e.g. loading of the note into the tab context and all the components) // this cache is more meant to share the data between different components within one business transaction (e.g. loading of the note into the tab context and all the components)
// this is also a workaround for missing invalidation after change // this is also a workaround for missing invalidation after change
this.noteComplementPromises[noteId].then( this.blobPromises[noteId].then(
() => setTimeout(() => this.noteComplementPromises[noteId] = null, 1000) () => setTimeout(() => this.blobPromises[noteId] = null, 1000)
); );
} }
return await this.noteComplementPromises[noteId]; return await this.blobPromises[noteId];
} }
} }

6
src/public/app/services/froca_updater.js
View File

@ -19,10 +19,10 @@ async function processEntityChanges(entityChanges) {
processAttributeChange(loadResults, ec); processAttributeChange(loadResults, ec);
} else if (ec.entityName === 'note_reordering') { } else if (ec.entityName === 'note_reordering') {
processNoteReordering(loadResults, ec); processNoteReordering(loadResults, ec);
} else if (ec.entityName === 'note_contents') { } else if (ec.entityName === 'blobs') {
delete froca.noteComplementPromises[ec.entityId]; delete froca.blobPromises[ec.entityId];
loadResults.addNoteContent(ec.entityId, ec.componentId); loadResults.addNoteContent(ec.noteIds, ec.componentId);
} else if (ec.entityName === 'note_revisions') { } else if (ec.entityName === 'note_revisions') {
loadResults.addNoteRevision(ec.entityId, ec.noteId, ec.componentId); loadResults.addNoteRevision(ec.entityId, ec.noteId, ec.componentId);
} else if (ec.entityName === 'note_revision_contents') { } else if (ec.entityName === 'note_revision_contents') {

6
src/public/app/services/load_results.js
View File

@ -94,8 +94,10 @@ export default class LoadResults {
return componentIds && componentIds.find(sId => sId !== componentId) !== undefined; return componentIds && componentIds.find(sId => sId !== componentId) !== undefined;
} }
addNoteContent(noteId, componentId) { addNoteContent(noteIds, componentId) {
this.contentNoteIdToComponentId.push({noteId, componentId}); for (const noteId of noteIds) {
this.contentNoteIdToComponentId.push({noteId, componentId});
}
} }
isNoteContentReloaded(noteId, componentId) { isNoteContentReloaded(noteId, componentId) {

2
src/services/app_info.js
View File

@ -4,7 +4,7 @@ const build = require('./build');
const packageJson = require('../../package'); const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir'); const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 215; const APP_DB_VERSION = 217;
const SYNC_VERSION = 30; const SYNC_VERSION = 30;
const CLIPPER_PROTOCOL_VERSION = "1.0"; const CLIPPER_PROTOCOL_VERSION = "1.0";

154
src/services/consistency_checks.js
View File

@ -383,86 +383,86 @@ class ConsistencyChecks {
} }
}); });
this.findAndFixIssues(` // this.findAndFixIssues(`
SELECT notes.noteId, notes.isProtected, notes.type, notes.mime // SELECT notes.noteId, notes.isProtected, notes.type, notes.mime
FROM notes // FROM notes
LEFT JOIN note_contents USING (noteId) // LEFT JOIN note_contents USING (noteId)
WHERE note_contents.noteId IS NULL`, // WHERE note_contents.noteId IS NULL`,
({noteId, isProtected, type, mime}) => { // ({noteId, isProtected, type, mime}) => {
if (this.autoFix) { // if (this.autoFix) {
// it might be possible that the note_content is not available only because of the interrupted // // it might be possible that the note_content is not available only because of the interrupted
// sync, and it will come later. It's therefore important to guarantee that this artifical // // sync, and it will come later. It's therefore important to guarantee that this artifical
// record won't overwrite the real one coming from the sync. // // record won't overwrite the real one coming from the sync.
const fakeDate = "2000-01-01 00:00:00Z"; // const fakeDate = "2000-01-01 00:00:00Z";
//
// manually creating row since this can also affect deleted notes // // manually creating row since this can also affect deleted notes
sql.upsert("note_contents", "noteId", { // sql.upsert("note_contents", "noteId", {
noteId: noteId, // noteId: noteId,
content: getBlankContent(isProtected, type, mime), // content: getBlankContent(isProtected, type, mime),
utcDateModified: fakeDate, // utcDateModified: fakeDate,
dateModified: fakeDate // dateModified: fakeDate
}); // });
//
const hash = utils.hash(utils.randomString(10)); // const hash = utils.hash(utils.randomString(10));
//
entityChangesService.addEntityChange({ // entityChangesService.addEntityChange({
entityName: 'note_contents', // entityName: 'note_contents',
entityId: noteId, // entityId: noteId,
hash: hash, // hash: hash,
isErased: false, // isErased: false,
utcDateChanged: fakeDate, // utcDateChanged: fakeDate,
isSynced: true // isSynced: true
}); // });
//
this.reloadNeeded = true; // this.reloadNeeded = true;
//
logFix(`Note '${noteId}' content was set to empty string since there was no corresponding row`); // logFix(`Note '${noteId}' content was set to empty string since there was no corresponding row`);
} else { // } else {
logError(`Note '${noteId}' content row does not exist`); // logError(`Note '${noteId}' content row does not exist`);
} // }
}); // });
if (sqlInit.getDbSize() < 500000) { if (sqlInit.getDbSize() < 500000) {
// querying for "content IS NULL" is expensive since content is not indexed. See e.g. https://github.com/zadam/trilium/issues/2887 // querying for "content IS NULL" is expensive since content is not indexed. See e.g. https://github.com/zadam/trilium/issues/2887
this.findAndFixIssues(` // this.findAndFixIssues(`
SELECT notes.noteId, notes.type, notes.mime // SELECT notes.noteId, notes.type, notes.mime
FROM notes // FROM notes
JOIN note_contents USING (noteId) // JOIN note_contents USING (noteId)
WHERE isDeleted = 0 // WHERE isDeleted = 0
AND isProtected = 0 // AND isProtected = 0
AND content IS NULL`, // AND content IS NULL`,
({noteId, type, mime}) => { // ({noteId, type, mime}) => {
if (this.autoFix) { // if (this.autoFix) {
const note = becca.getNote(noteId); // const note = becca.getNote(noteId);
const blankContent = getBlankContent(false, type, mime); // const blankContent = getBlankContent(false, type, mime);
note.setContent(blankContent); // note.setContent(blankContent);
//
this.reloadNeeded = true; // this.reloadNeeded = true;
//
logFix(`Note '${noteId}' content was set to '${blankContent}' since it was null even though it is not deleted`); // logFix(`Note '${noteId}' content was set to '${blankContent}' since it was null even though it is not deleted`);
} else { // } else {
logError(`Note '${noteId}' content is null even though it is not deleted`); // logError(`Note '${noteId}' content is null even though it is not deleted`);
} // }
}); // });
} }
this.findAndFixIssues(` // this.findAndFixIssues(`
SELECT note_revisions.noteRevisionId // SELECT note_revisions.noteRevisionId
FROM note_revisions // FROM note_revisions
LEFT JOIN note_revision_contents USING (noteRevisionId) // LEFT JOIN note_revision_contents USING (noteRevisionId)
WHERE note_revision_contents.noteRevisionId IS NULL`, // WHERE note_revision_contents.noteRevisionId IS NULL`,
({noteRevisionId}) => { // ({noteRevisionId}) => {
if (this.autoFix) { // if (this.autoFix) {
noteRevisionService.eraseNoteRevisions([noteRevisionId]); // noteRevisionService.eraseNoteRevisions([noteRevisionId]);
//
this.reloadNeeded = true; // this.reloadNeeded = true;
//
logFix(`Note revision content '${noteRevisionId}' was set to erased since it did not exist.`); // logFix(`Note revision content '${noteRevisionId}' was set to erased since it did not exist.`);
} else { // } else {
logError(`Note revision content '${noteRevisionId}' does not exist`); // logError(`Note revision content '${noteRevisionId}' does not exist`);
} // }
}); // });
this.findAndFixIssues(` this.findAndFixIssues(`
SELECT parentNoteId SELECT parentNoteId
@ -656,11 +656,11 @@ class ConsistencyChecks {
findEntityChangeIssues() { findEntityChangeIssues() {
this.runEntityChangeChecks("notes", "noteId"); this.runEntityChangeChecks("notes", "noteId");
this.runEntityChangeChecks("note_contents", "noteId"); //this.runEntityChangeChecks("note_contents", "noteId");
this.runEntityChangeChecks("note_revisions", "noteRevisionId"); this.runEntityChangeChecks("note_revisions", "noteRevisionId");
this.runEntityChangeChecks("note_revision_contents", "noteRevisionId"); //this.runEntityChangeChecks("note_revision_contents", "noteRevisionId");
this.runEntityChangeChecks("note_ancillaries", "noteAncillaryId"); this.runEntityChangeChecks("note_ancillaries", "noteAncillaryId");
this.runEntityChangeChecks("note_ancillary_contents", "noteAncillaryId"); //this.runEntityChangeChecks("note_ancillary_contents", "noteAncillaryId");
this.runEntityChangeChecks("branches", "branchId"); this.runEntityChangeChecks("branches", "branchId");
this.runEntityChangeChecks("attributes", "attributeId"); this.runEntityChangeChecks("attributes", "attributeId");
this.runEntityChangeChecks("etapi_tokens", "etapiTokenId"); this.runEntityChangeChecks("etapi_tokens", "etapiTokenId");

17
src/services/utils.js
View File

@ -25,6 +25,19 @@ function md5(content) {
return crypto.createHash('md5').update(content).digest('hex'); return crypto.createHash('md5').update(content).digest('hex');
} }
function hashedBlobId(content) {
// sha512 is faster than sha256
const base64Hash = crypto.createHash('sha512').update(content).digest('base64');
// 20 characters of base64 gives us 120 bit of entropy which is plenty enough
return base64Hash.substr(0, 20);
}
function randomBlobId(content) {
// underscore prefix to easily differentiate the random as opposed to hashed
return '_' + randomString(19);
}
function toBase64(plainText) { function toBase64(plainText) {
return Buffer.from(plainText).toString('base64'); return Buffer.from(plainText).toString('base64');
} }
@ -343,5 +356,7 @@ module.exports = {
deferred, deferred,
removeDiacritic, removeDiacritic,
normalize, normalize,
filterAttributeName filterAttributeName,
hashedBlobId,
randomBlobId
}; };

5
src/services/ws.js
View File

@ -129,13 +129,14 @@ function fillInAdditionalProperties(entityChange) {
entityChange.positions[childBranch.branchId] = childBranch.notePosition; entityChange.positions[childBranch.branchId] = childBranch.notePosition;
} }
} }
} } else if (entityChange.entityName === 'options') {
else if (entityChange.entityName === 'options') {
entityChange.entity = becca.getOption(entityChange.entityId); entityChange.entity = becca.getOption(entityChange.entityId);
if (!entityChange.entity) { if (!entityChange.entity) {
entityChange.entity = sql.getRow(`SELECT * FROM options WHERE name = ?`, [entityChange.entityId]); entityChange.entity = sql.getRow(`SELECT * FROM options WHERE name = ?`, [entityChange.entityId]);
} }
} else if (entityChange.entityName === 'blob') {
entityChange.noteIds = sql.getColumn("SELECT noteId FROM notes WHERE blobId = ? AND isDeleted = 0", [entityChange.entityId]);
} }
if (entityChange.entity instanceof AbstractBeccaEntity) { if (entityChange.entity instanceof AbstractBeccaEntity) {