From 7992f32d34f432a1df84f2e7082fcbf43338d24f Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 13 May 2020 14:42:16 +0200 Subject: [PATCH] note cache refactoring --- package-lock.json | 34 ++-- package.json | 2 +- src/entities/attribute.js | 14 +- src/entities/branch.js | 8 +- src/routes/api/attributes.js | 5 +- src/routes/api/similar_notes.js | 4 +- src/services/note_cache.js | 307 ++++++++++++++++++-------------- 7 files changed, 210 insertions(+), 164 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f65b4c89..554169239 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "trilium", - "version": "0.42.1", + "version": "0.42.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2218,9 +2218,9 @@ } }, "cli-spinners": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz", - "integrity": "sha512-tgU3fKwzYjiLEQgPMD9Jt+JjHVL9kW93FiIMX/l7rivvOD4/LL0Mf7gda3+4U2KJBloybwgj5KEoQgGRioMiKQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.3.0.tgz", + "integrity": "sha512-Xs2Hf2nzrvJMFKimOR7YR0QwZ8fc0u98kdtwN1eNAZzNQgH3vK2pXzff6GJtKh7S5hoJ87ECiAiZFS2fb5Ii2w==", "dev": true }, "cli-table3": { @@ -3802,9 +3802,9 @@ } }, "electron-rebuild": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.10.1.tgz", - "integrity": "sha512-KSqp0Xiu7CCvKL2aEdPp/vNe2Rr11vaO8eM/wq9gQJTY02UjtAJ3l7WLV7Mf8oR+UJReJO8SWOWs/FozqK8ggA==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.11.0.tgz", + "integrity": "sha512-cn6AqZBQBVtaEyj5jZW1/LOezZZ22PA1HvhEP7asvYPJ8PDF4i4UFt9be4i9T7xJKiSiomXvY5Fd+dSq3FXZxA==", "dev": true, "requires": { "colors": "^1.3.3", @@ -3877,9 +3877,9 @@ } }, "yargs": { - "version": "14.2.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.2.tgz", - "integrity": "sha512-/4ld+4VV5RnrynMhPZJ/ZpOCGSCeghMykZ3BhdFBDa9Wy/RH6uEGNWDJog+aUlq+9OM1CFTgtYRW5Is1Po9NOA==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz", + "integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==", "dev": true, "requires": { "cliui": "^5.0.0", @@ -3892,13 +3892,13 @@ "string-width": "^3.0.0", "which-module": "^2.0.0", "y18n": "^4.0.0", - "yargs-parser": "^15.0.0" + "yargs-parser": "^15.0.1" } }, "yargs-parser": { - "version": "15.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.0.tgz", - "integrity": "sha512-xLTUnCMc4JhxrPEPUYD5IBR1mWCK/aT6+RJ/K29JY2y1vD+FhtgKK0AXRWvI262q3QSffAQuTouFIKUuHX89wQ==", + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz", + "integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==", "dev": true, "requires": { "camelcase": "^5.0.0", @@ -9929,9 +9929,9 @@ } }, "rxjs": { - "version": "6.5.4", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz", - "integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==", + "version": "6.5.5", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz", + "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==", "dev": true, "requires": { "tslib": "^1.9.0" diff --git a/package.json b/package.json index eb32e74fe..65fd8a95c 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "electron": "9.0.0-beta.24", "electron-builder": "22.6.0", "electron-packager": "14.2.1", - "electron-rebuild": "1.10.1", + "electron-rebuild": "1.11.0", "jsdoc": "3.6.4", "lorem-ipsum": "2.0.3", "webpack": "5.0.0-beta.16", diff --git a/src/entities/attribute.js b/src/entities/attribute.js index 0800d6252..14b7c8a87 100644 --- a/src/entities/attribute.js +++ b/src/entities/attribute.js @@ -8,13 +8,13 @@ const sql = require('../services/sql'); /** * Attribute is key value pair owned by a note. * - * @property {string} attributeId - * @property {string} noteId - * @property {string} type - * @property {string} name + * @property {string} attributeId - immutable + * @property {string} noteId - immutable + * @property {string} type - immutable + * @property {string} name - immutable * @property {string} value * @property {int} position - * @property {boolean} isInheritable + * @property {boolean} isInheritable - immutable * @property {boolean} isDeleted * @property {string|null} deleteId - ID identifying delete transaction * @property {string} utcDateCreated @@ -108,14 +108,14 @@ class Attribute extends Entity { delete pojo.__note; } - createClone(type, name, value) { + createClone(type, name, value, isInheritable) { return new Attribute({ noteId: this.noteId, type: type, name: name, value: value, position: this.position, - isInheritable: this.isInheritable, + isInheritable: isInheritable, isDeleted: false, utcDateCreated: this.utcDateCreated, utcDateModified: this.utcDateModified diff --git a/src/entities/branch.js b/src/entities/branch.js index 7950526dc..060a275cd 100644 --- a/src/entities/branch.js +++ b/src/entities/branch.js @@ -9,9 +9,9 @@ const sql = require('../services/sql'); * Branch represents note's placement in the tree - it's essentially pair of noteId and parentNoteId. * Each note can have multiple (at least one) branches, meaning it can be placed into multiple places in the tree. * - * @property {string} branchId - primary key - * @property {string} noteId - * @property {string} parentNoteId + * @property {string} branchId - primary key, immutable + * @property {string} noteId - immutable + * @property {string} parentNoteId - immutable * @property {int} notePosition * @property {string} prefix * @property {boolean} isExpanded @@ -77,4 +77,4 @@ class Branch extends Entity { } } -module.exports = Branch; \ No newline at end of file +module.exports = Branch; diff --git a/src/routes/api/attributes.js b/src/routes/api/attributes.js index 131e7a77b..abab2ab9e 100644 --- a/src/routes/api/attributes.js +++ b/src/routes/api/attributes.js @@ -98,10 +98,11 @@ async function updateNoteAttributes(req) { if (attribute.type !== attributeEntity.type || attribute.name !== attributeEntity.name - || (attribute.type === 'relation' && attribute.value !== attributeEntity.value)) { + || (attribute.type === 'relation' && attribute.value !== attributeEntity.value) + || attribute.isInheritable !== attributeEntity.isInheritable) { if (attribute.type !== 'relation' || !!attribute.value.trim()) { - const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value); + const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value, attribute.isInheritable); await newAttribute.save(); } diff --git a/src/routes/api/similar_notes.js b/src/routes/api/similar_notes.js index 3403833f8..4d52e94fe 100644 --- a/src/routes/api/similar_notes.js +++ b/src/routes/api/similar_notes.js @@ -12,7 +12,7 @@ async function getSimilarNotes(req) { return [404, `Note ${noteId} not found.`]; } - const results = await noteCacheService.findSimilarNotes(note.title); + const results = await noteCacheService.findSimilarNotes(noteId); return results .filter(note => note.noteId !== noteId); @@ -20,4 +20,4 @@ async function getSimilarNotes(req) { module.exports = { getSimilarNotes -}; \ No newline at end of file +}; diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 88f934c56..e1b4efee2 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -35,46 +35,75 @@ class Note { this.children = []; /** @param {Attribute[]} */ this.ownedAttributes = []; + + /** @param {Attribute[]|null} */ + this.attributeCache = null; + /** @param {Attribute[]|null} */ + this.templateAttributeCache = null; + /** @param {Attribute[]|null} */ + this.inheritableAttributeCache = null; + + /** @param {string|null} */ + this.fulltextCache = null; } /** @return {Attribute[]} */ get attributes() { - if (!(this.noteId in noteAttributeCache)) { - const attrArrs = [ - this.ownedAttributes - ]; - - for (const templateAttr of this.ownedAttributes.filter(oa => oa.type === 'relation' && oa.name === 'template')) { - const templateNote = notes[templateAttr.value]; - - if (templateNote) { - attrArrs.push(templateNote.attributes); - } - } + if (!this.attributeCache) { + const parentAttributes = this.ownedAttributes.slice(); if (this.noteId !== 'root') { for (const parentNote of this.parents) { - attrArrs.push(parentNote.inheritableAttributes); + parentAttributes.push(...parentNote.inheritableAttributes); } } - noteAttributeCache[this.noteId] = attrArrs.flat(); + const templateAttributes = []; + + for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates + if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') { + const templateNote = notes[ownedAttr.value]; + + if (templateNote) { + templateAttributes.push(...templateNote.attributes); + } + } + } + + this.attributeCache = parentAttributes.concat(templateAttributes); + this.inheritableAttributeCache = []; + this.templateAttributeCache = []; + + for (const attr of this.attributeCache) { + if (attr.isInheritable) { + this.inheritableAttributeCache.push(attr); + } + + if (attr.type === 'relation' && attr.name === 'template') { + this.templateAttributeCache.push(attr); + } + } } - return noteAttributeCache[this.noteId]; - } - - addSubTreeNoteIdsTo(noteIdSet) { - noteIdSet.add(this.noteId); - - for (const child of this.children) { - child.addSubTreeNoteIdsTo(noteIdSet); - } + return this.attributeCache; } /** @return {Attribute[]} */ get inheritableAttributes() { - return this.attributes.filter(attr => attr.isInheritable); + if (!this.inheritableAttributeCache) { + this.attributes; // will refresh also this.inheritableAttributeCache + } + + return this.inheritableAttributeCache; + } + + /** @return {Attribute[]} */ + get templateAttributes() { + if (!this.templateAttributeCache) { + this.attributes; // will refresh also this.templateAttributeCache + } + + return this.templateAttributeCache; } hasAttribute(type, name) { @@ -94,6 +123,63 @@ class Note { resortParents() { this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1); } + + get fulltext() { + if (!this.fulltextCache) { + this.fulltextCache = this.title.toLowerCase(); + + for (const attr of this.attributes) { + // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words + this.fulltextCache += ' ' + attr.name.toLowerCase(); + + if (attr.value) { + this.fulltextCache += ' ' + attr.value.toLowerCase(); + } + } + } + + return this.fulltextCache; + } + + invalidateThisCache() { + this.fulltextCache = null; + + this.attributeCache = null; + this.templateAttributeCache = null; + this.inheritableAttributeCache = null; + } + + invalidateSubtreeCaches() { + this.invalidateThisCache(); + + for (const childNote of this.children) { + childNote.invalidateSubtreeCaches(); + } + + for (const templateAttr of this.templateAttributes) { + const targetNote = templateAttr.targetNote; + + if (targetNote) { + targetNote.invalidateSubtreeCaches(); + } + } + } + + invalidateSubtreeFulltext() { + this.fulltextCache = null; + + for (const childNote of this.children) { + childNote.invalidateSubtreeFulltext(); + } + + for (const templateAttr of this.templateAttributes) { + const targetNote = templateAttr.targetNote; + + if (targetNote) { + targetNote.invalidateSubtreeFulltext(); + } + } + } } class Branch { @@ -137,47 +223,16 @@ class Attribute { /** @param {boolean} */ this.isInheritable = !!row.isInheritable; } -} -/** @type {Object.} */ -let fulltext = {}; - -/** @type {Object.} */ -let attributeMetas = {}; - -class AttributeMeta { - constructor(attribute) { - this.type = attribute.type; - this.name = attribute.name; - this.isInheritable = attribute.isInheritable; - this.attributeIds = new Set(attribute.attributeId); + get isAffectingSubtree() { + return this.isInheritable + || (this.type === 'relation' && this.name === 'template'); } - addAttribute(attribute) { - this.attributeIds.add(attribute.attributeId); - this.isInheritable = this.isInheritable || attribute.isInheritable; - } - - updateAttribute(attribute) { - if (attribute.isDeleted) { - this.attributeIds.delete(attribute.attributeId); + get targetNote() { + if (this.type === 'relation') { + return notes[this.value]; } - else { - this.attributeIds.add(attribute.attributeId); - } - - this.isInheritable = !!this.attributeIds.find(attributeId => attributes[attributeId].isInheritable); - } -} - -function addToAttributeMeta(attribute) { - const key = `${attribute.type}-${attribute.name}`; - - if (!(key in attributeMetas)) { - attributeMetas[key] = new AttributeMeta(attribute); - } - else { - attributeMetas[key].addAttribute(attribute); } } @@ -186,9 +241,6 @@ let loadedPromiseResolve; /** Is resolved after the initial load */ let loadedPromise = new Promise(res => loadedPromiseResolve = res); -// key is 'childNoteId-parentNoteId' as a replacement for branchId which we don't use here -let prefixes = {}; - async function getMappedRows(query, cb) { const map = {}; const results = await sql.getRows(query, []); @@ -202,17 +254,6 @@ async function getMappedRows(query, cb) { return map; } -function updateFulltext(note) { - let ft = note.title.toLowerCase(); - - for (const attr of note.attributes) { - ft += '|' + attr.name.toLowerCase(); - ft += '|' + attr.value.toLowerCase(); - } - - fulltext[note.noteId] = ft; -} - async function load() { notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, row => new Note(row)); @@ -225,8 +266,6 @@ async function load() { for (const attr of Object.values(attributes)) { notes[attr.noteId].ownedAttributes.push(attr); - - addToAttributeMeta(attributes); } for (const branch of Object.values(branches)) { @@ -250,10 +289,6 @@ async function load() { await decryptProtectedNotes(); } - for (const note of Object.values(notes)) { - updateFulltext(note); - } - loaded = true; loadedPromiseResolve(); } @@ -325,38 +360,21 @@ function highlightResults(results, allTokens) { * Returns noteIds which have at least one matching tokens * * @param tokens - * @return {Set} + * @return {String[]} */ function getCandidateNotes(tokens) { - const candidateNoteIds = new Set(); + const candidateNotes = []; - for (const token of tokens) { - for (const noteId in fulltext) { - if (!fulltext[noteId].includes(token)) { - continue; + for (const note of Object.values(notes)) { + for (const token of tokens) { + if (note.fulltext.includes(token)) { + candidateNotes.push(note); + break; } - - candidateNoteIds.add(noteId); - const note = notes[noteId]; - const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable); - - searchingAttrs: - // for matching inheritable attributes, include the whole note subtree to the candidates - for (const attr of inheritableAttrs) { - const lcName = attr.name.toLowerCase(); - const lcValue = attr.value.toLowerCase(); - - for (const token of tokens) { - if (lcName.includes(token) || lcValue.includes(token)) { - note.addSubTreeNoteIdsTo(candidateNoteIds); - - break searchingAttrs; - } - } - } } } - return candidateNoteIds; + + return candidateNotes; } async function findNotes(query) { @@ -370,18 +388,16 @@ async function findNotes(query) { .split(/[ -]/) .filter(token => token !== '/'); // '/' is used as separator - const candidateNoteIds = getCandidateNotes(allTokens); + const candidateNotes = getCandidateNotes(allTokens); // now we have set of noteIds which match at least one token let results = []; const tokens = allTokens.slice(); - for (const noteId of candidateNoteIds) { - const note = notes[noteId]; - + for (const note of candidateNotes) { // autocomplete should be able to find notes by their noteIds as well (only leafs) - if (noteId === query) { + if (note.noteId === query) { search(note, [], [], results); continue; } @@ -415,7 +431,7 @@ async function findNotes(query) { if (foundTokens.length > 0) { const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - search(parentNote, remainingTokens, [noteId], results); + search(parentNote, remainingTokens, [note.noteId], results); } } } @@ -678,11 +694,11 @@ function getNotePath(noteId) { } } -function evaluateSimilarity(text, note, results) { - let coeff = stringSimilarity.compareTwoStrings(text, note.title); +function evaluateSimilarity(sourceNote, candidateNote, results) { + let coeff = stringSimilarity.compareTwoStrings(sourceNote.fulltext, candidateNote.fulltext); if (coeff > 0.4) { - const notePath = getSomePath(note); + const notePath = getSomePath(candidateNote); // this takes care of note hoisting if (!notePath) { @@ -693,7 +709,7 @@ function evaluateSimilarity(text, note, results) { coeff -= 0.2; // archived penalization } - results.push({coeff, notePath, noteId: note.noteId}); + results.push({coeff, notePath, noteId: candidateNote.noteId}); } } @@ -707,16 +723,22 @@ function setImmediatePromise() { }); } -async function findSimilarNotes(title) { +async function findSimilarNotes(noteId) { const results = []; let i = 0; + const origNote = notes[noteId]; + + if (!origNote) { + return []; + } + for (const note of Object.values(notes)) { if (note.isProtected && !note.isDecrypted) { continue; } - evaluateSimilarity(title, note, results); + evaluateSimilarity(origNote, note, results); i++; @@ -744,9 +766,12 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED delete notes[noteId]; } else if (noteId in notes) { + const note = notes[noteId]; + // we can assume we have protected session since we managed to update - notes[noteId].title = entity.title; - notes[noteId].isDecrypted = true; + note.title = entity.title; + note.isDecrypted = true; + note.fulltextCache = null; } else { notes[noteId] = new Note(entity); @@ -760,6 +785,10 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED if (childNote) { childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); + + if (childNote.parents.length > 0) { + childNote.invalidateSubtreeCaches(); + } } const parentNote = notes[parentNoteId]; @@ -787,30 +816,46 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED } else if (entityName === 'attributes') { const {attributeId, noteId} = entity; + const note = notes[noteId]; + const attr = attributes[attributeId]; if (entity.isDeleted) { - const note = notes[noteId]; - - if (note) { + if (note && attr) { note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); + + if (attr.isAffectingSubtree) { + note.invalidateSubtreeCaches(); + } } - delete attributes[entity.attributeId]; + delete attributes[attributeId]; } else if (attributeId in attributes) { const attr = attributes[attributeId]; - // attr name cannot change + // attr name and isInheritable are immutable attr.value = entity.value; - attr.isInheritable = entity.isInheritable; + + if (attr.isAffectingSubtree) { + note.invalidateSubtreeFulltext(); + } + else { + note.fulltextCache = null; + } } else { - attributes[attributeId] = new Attribute(entity); - - const note = notes[noteId]; + const attr = new Attribute(entity); + attributes[attributeId] = attr; if (note) { - note.ownedAttributes.push(attributes[attributeId]); + note.ownedAttributes.push(attr); + + if (attr.isAffectingSubtree) { + note.invalidateSubtreeCaches(); + } + else { + this.invalidateThisCache(); + } } } }