gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

note cache refactoring

Browse Source
This commit is contained in:
zadam 2020-05-13 14:42:16 +02:00
parent b07accfd9d
commit 7992f32d34
7 changed files with 210 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
package-lock.json generated
View File

@ -1,6 +1,6 @@
{ {
"name": "trilium", "name": "trilium",
"version": "0.42.1", "version": "0.42.2",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {
@ -2218,9 +2218,9 @@
} }
}, },
"cli-spinners": { "cli-spinners": {
"version": "2.2.0", "version": "2.3.0",
"resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.3.0.tgz",
"integrity": "sha512-tgU3fKwzYjiLEQgPMD9Jt+JjHVL9kW93FiIMX/l7rivvOD4/LL0Mf7gda3+4U2KJBloybwgj5KEoQgGRioMiKQ==", "integrity": "sha512-Xs2Hf2nzrvJMFKimOR7YR0QwZ8fc0u98kdtwN1eNAZzNQgH3vK2pXzff6GJtKh7S5hoJ87ECiAiZFS2fb5Ii2w==",
"dev": true "dev": true
}, },
"cli-table3": { "cli-table3": {
@ -3802,9 +3802,9 @@
} }
}, },
"electron-rebuild": { "electron-rebuild": {
"version": "1.10.1", "version": "1.11.0",
"resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.10.1.tgz", "resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.11.0.tgz",
"integrity": "sha512-KSqp0Xiu7CCvKL2aEdPp/vNe2Rr11vaO8eM/wq9gQJTY02UjtAJ3l7WLV7Mf8oR+UJReJO8SWOWs/FozqK8ggA==", "integrity": "sha512-cn6AqZBQBVtaEyj5jZW1/LOezZZ22PA1HvhEP7asvYPJ8PDF4i4UFt9be4i9T7xJKiSiomXvY5Fd+dSq3FXZxA==",
"dev": true, "dev": true,
"requires": { "requires": {
"colors": "^1.3.3", "colors": "^1.3.3",
@ -3877,9 +3877,9 @@
} }
}, },
"yargs": { "yargs": {
"version": "14.2.2", "version": "14.2.3",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.2.tgz", "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz",
"integrity": "sha512-/4ld+4VV5RnrynMhPZJ/ZpOCGSCeghMykZ3BhdFBDa9Wy/RH6uEGNWDJog+aUlq+9OM1CFTgtYRW5Is1Po9NOA==", "integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==",
"dev": true, "dev": true,
"requires": { "requires": {
"cliui": "^5.0.0", "cliui": "^5.0.0",
@ -3892,13 +3892,13 @@
"string-width": "^3.0.0", "string-width": "^3.0.0",
"which-module": "^2.0.0", "which-module": "^2.0.0",
"y18n": "^4.0.0", "y18n": "^4.0.0",
"yargs-parser": "^15.0.0" "yargs-parser": "^15.0.1"
} }
}, },
"yargs-parser": { "yargs-parser": {
"version": "15.0.0", "version": "15.0.1",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.0.tgz", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz",
"integrity": "sha512-xLTUnCMc4JhxrPEPUYD5IBR1mWCK/aT6+RJ/K29JY2y1vD+FhtgKK0AXRWvI262q3QSffAQuTouFIKUuHX89wQ==", "integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==",
"dev": true, "dev": true,
"requires": { "requires": {
"camelcase": "^5.0.0", "camelcase": "^5.0.0",
@ -9929,9 +9929,9 @@
} }
}, },
"rxjs": { "rxjs": {
"version": "6.5.4", "version": "6.5.5",
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz",
"integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==", "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==",
"dev": true, "dev": true,
"requires": { "requires": {
"tslib": "^1.9.0" "tslib": "^1.9.0"

2
package.json
View File

@ -81,7 +81,7 @@
"electron": "9.0.0-beta.24", "electron": "9.0.0-beta.24",
"electron-builder": "22.6.0", "electron-builder": "22.6.0",
"electron-packager": "14.2.1", "electron-packager": "14.2.1",
"electron-rebuild": "1.10.1", "electron-rebuild": "1.11.0",
"jsdoc": "3.6.4", "jsdoc": "3.6.4",
"lorem-ipsum": "2.0.3", "lorem-ipsum": "2.0.3",
"webpack": "5.0.0-beta.16", "webpack": "5.0.0-beta.16",

14
src/entities/attribute.js
View File

@ -8,13 +8,13 @@ const sql = require('../services/sql');
/** /**
* Attribute is key value pair owned by a note. * Attribute is key value pair owned by a note.
* *
* @property {string} attributeId * @property {string} attributeId - immutable
* @property {string} noteId * @property {string} noteId - immutable
* @property {string} type * @property {string} type - immutable
* @property {string} name * @property {string} name - immutable
* @property {string} value * @property {string} value
* @property {int} position * @property {int} position
* @property {boolean} isInheritable * @property {boolean} isInheritable - immutable
* @property {boolean} isDeleted * @property {boolean} isDeleted
* @property {string|null} deleteId - ID identifying delete transaction * @property {string|null} deleteId - ID identifying delete transaction
* @property {string} utcDateCreated * @property {string} utcDateCreated
@ -108,14 +108,14 @@ class Attribute extends Entity {
delete pojo.__note; delete pojo.__note;
} }
createClone(type, name, value) { createClone(type, name, value, isInheritable) {
return new Attribute({ return new Attribute({
noteId: this.noteId, noteId: this.noteId,
type: type, type: type,
name: name, name: name,
value: value, value: value,
position: this.position, position: this.position,
isInheritable: this.isInheritable, isInheritable: isInheritable,
isDeleted: false, isDeleted: false,
utcDateCreated: this.utcDateCreated, utcDateCreated: this.utcDateCreated,
utcDateModified: this.utcDateModified utcDateModified: this.utcDateModified

6
src/entities/branch.js
View File

@ -9,9 +9,9 @@ const sql = require('../services/sql');
* Branch represents note's placement in the tree - it's essentially pair of noteId and parentNoteId. * Branch represents note's placement in the tree - it's essentially pair of noteId and parentNoteId.
* Each note can have multiple (at least one) branches, meaning it can be placed into multiple places in the tree. * Each note can have multiple (at least one) branches, meaning it can be placed into multiple places in the tree.
* *
* @property {string} branchId - primary key * @property {string} branchId - primary key, immutable
* @property {string} noteId * @property {string} noteId - immutable
* @property {string} parentNoteId * @property {string} parentNoteId - immutable
* @property {int} notePosition * @property {int} notePosition
* @property {string} prefix * @property {string} prefix
* @property {boolean} isExpanded * @property {boolean} isExpanded

5
src/routes/api/attributes.js
View File

@ -98,10 +98,11 @@ async function updateNoteAttributes(req) {
if (attribute.type !== attributeEntity.type if (attribute.type !== attributeEntity.type
|| attribute.name !== attributeEntity.name || attribute.name !== attributeEntity.name
|| (attribute.type === 'relation' && attribute.value !== attributeEntity.value)) { || (attribute.type === 'relation' && attribute.value !== attributeEntity.value)
|| attribute.isInheritable !== attributeEntity.isInheritable) {
if (attribute.type !== 'relation' || !!attribute.value.trim()) { if (attribute.type !== 'relation' || !!attribute.value.trim()) {
const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value); const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value, attribute.isInheritable);
await newAttribute.save(); await newAttribute.save();
} }

2
src/routes/api/similar_notes.js
View File

@ -12,7 +12,7 @@ async function getSimilarNotes(req) {
return [404, `Note ${noteId} not found.`]; return [404, `Note ${noteId} not found.`];
} }
const results = await noteCacheService.findSimilarNotes(note.title); const results = await noteCacheService.findSimilarNotes(noteId);
return results return results
.filter(note => note.noteId !== noteId); .filter(note => note.noteId !== noteId);

307
src/services/note_cache.js
View File

@ -35,46 +35,75 @@ class Note {
this.children = []; this.children = [];
/** @param {Attribute[]} */ /** @param {Attribute[]} */
this.ownedAttributes = []; this.ownedAttributes = [];
/** @param {Attribute[]|null} */
this.attributeCache = null;
/** @param {Attribute[]|null} */
this.templateAttributeCache = null;
/** @param {Attribute[]|null} */
this.inheritableAttributeCache = null;
/** @param {string|null} */
this.fulltextCache = null;
} }
/** @return {Attribute[]} */ /** @return {Attribute[]} */
get attributes() { get attributes() {
if (!(this.noteId in noteAttributeCache)) { if (!this.attributeCache) {
const attrArrs = [ const parentAttributes = this.ownedAttributes.slice();
this.ownedAttributes
];
for (const templateAttr of this.ownedAttributes.filter(oa => oa.type === 'relation' && oa.name === 'template')) {
const templateNote = notes[templateAttr.value];
if (templateNote) {
attrArrs.push(templateNote.attributes);
}
}
if (this.noteId !== 'root') { if (this.noteId !== 'root') {
for (const parentNote of this.parents) { for (const parentNote of this.parents) {
attrArrs.push(parentNote.inheritableAttributes); parentAttributes.push(...parentNote.inheritableAttributes);
} }
} }
noteAttributeCache[this.noteId] = attrArrs.flat(); const templateAttributes = [];
for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates
if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') {
const templateNote = notes[ownedAttr.value];
if (templateNote) {
templateAttributes.push(...templateNote.attributes);
}
}
}
this.attributeCache = parentAttributes.concat(templateAttributes);
this.inheritableAttributeCache = [];
this.templateAttributeCache = [];
for (const attr of this.attributeCache) {
if (attr.isInheritable) {
this.inheritableAttributeCache.push(attr);
}
if (attr.type === 'relation' && attr.name === 'template') {
this.templateAttributeCache.push(attr);
}
}
} }
return noteAttributeCache[this.noteId]; return this.attributeCache;
}
addSubTreeNoteIdsTo(noteIdSet) {
noteIdSet.add(this.noteId);
for (const child of this.children) {
child.addSubTreeNoteIdsTo(noteIdSet);
}
} }
/** @return {Attribute[]} */ /** @return {Attribute[]} */
get inheritableAttributes() { get inheritableAttributes() {
return this.attributes.filter(attr => attr.isInheritable); if (!this.inheritableAttributeCache) {
this.attributes; // will refresh also this.inheritableAttributeCache
}
return this.inheritableAttributeCache;
}
/** @return {Attribute[]} */
get templateAttributes() {
if (!this.templateAttributeCache) {
this.attributes; // will refresh also this.templateAttributeCache
}
return this.templateAttributeCache;
} }
hasAttribute(type, name) { hasAttribute(type, name) {
@ -94,6 +123,63 @@ class Note {
resortParents() { resortParents() {
this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1); this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1);
} }
get fulltext() {
if (!this.fulltextCache) {
this.fulltextCache = this.title.toLowerCase();
for (const attr of this.attributes) {
// it's best to use space as separator since spaces are filtered from the search string by the tokenization into words
this.fulltextCache += ' ' + attr.name.toLowerCase();
if (attr.value) {
this.fulltextCache += ' ' + attr.value.toLowerCase();
}
}
}
return this.fulltextCache;
}
invalidateThisCache() {
this.fulltextCache = null;
this.attributeCache = null;
this.templateAttributeCache = null;
this.inheritableAttributeCache = null;
}
invalidateSubtreeCaches() {
this.invalidateThisCache();
for (const childNote of this.children) {
childNote.invalidateSubtreeCaches();
}
for (const templateAttr of this.templateAttributes) {
const targetNote = templateAttr.targetNote;
if (targetNote) {
targetNote.invalidateSubtreeCaches();
}
}
}
invalidateSubtreeFulltext() {
this.fulltextCache = null;
for (const childNote of this.children) {
childNote.invalidateSubtreeFulltext();
}
for (const templateAttr of this.templateAttributes) {
const targetNote = templateAttr.targetNote;
if (targetNote) {
targetNote.invalidateSubtreeFulltext();
}
}
}
} }
class Branch { class Branch {
@ -137,47 +223,16 @@ class Attribute {
/** @param {boolean} */ /** @param {boolean} */
this.isInheritable = !!row.isInheritable; this.isInheritable = !!row.isInheritable;
} }
}
/** @type {Object.<String, String>} */ get isAffectingSubtree() {
let fulltext = {}; return this.isInheritable
|| (this.type === 'relation' && this.name === 'template');
/** @type {Object.<String, AttributeMeta>} */
let attributeMetas = {};
class AttributeMeta {
constructor(attribute) {
this.type = attribute.type;
this.name = attribute.name;
this.isInheritable = attribute.isInheritable;
this.attributeIds = new Set(attribute.attributeId);
} }
addAttribute(attribute) { get targetNote() {
this.attributeIds.add(attribute.attributeId); if (this.type === 'relation') {
this.isInheritable = this.isInheritable || attribute.isInheritable; return notes[this.value];
}
updateAttribute(attribute) {
if (attribute.isDeleted) {
this.attributeIds.delete(attribute.attributeId);
} }
else {
this.attributeIds.add(attribute.attributeId);
}
this.isInheritable = !!this.attributeIds.find(attributeId => attributes[attributeId].isInheritable);
}
}
function addToAttributeMeta(attribute) {
const key = `${attribute.type}-${attribute.name}`;
if (!(key in attributeMetas)) {
attributeMetas[key] = new AttributeMeta(attribute);
}
else {
attributeMetas[key].addAttribute(attribute);
} }
} }
@ -186,9 +241,6 @@ let loadedPromiseResolve;
/** Is resolved after the initial load */ /** Is resolved after the initial load */
let loadedPromise = new Promise(res => loadedPromiseResolve = res); let loadedPromise = new Promise(res => loadedPromiseResolve = res);
// key is 'childNoteId-parentNoteId' as a replacement for branchId which we don't use here
let prefixes = {};
async function getMappedRows(query, cb) { async function getMappedRows(query, cb) {
const map = {}; const map = {};
const results = await sql.getRows(query, []); const results = await sql.getRows(query, []);
@ -202,17 +254,6 @@ async function getMappedRows(query, cb) {
return map; return map;
} }
function updateFulltext(note) {
let ft = note.title.toLowerCase();
for (const attr of note.attributes) {
ft += '|' + attr.name.toLowerCase();
ft += '|' + attr.value.toLowerCase();
}
fulltext[note.noteId] = ft;
}
async function load() { async function load() {
notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`,
row => new Note(row)); row => new Note(row));
@ -225,8 +266,6 @@ async function load() {
for (const attr of Object.values(attributes)) { for (const attr of Object.values(attributes)) {
notes[attr.noteId].ownedAttributes.push(attr); notes[attr.noteId].ownedAttributes.push(attr);
addToAttributeMeta(attributes);
} }
for (const branch of Object.values(branches)) { for (const branch of Object.values(branches)) {
@ -250,10 +289,6 @@ async function load() {
await decryptProtectedNotes(); await decryptProtectedNotes();
} }
for (const note of Object.values(notes)) {
updateFulltext(note);
}
loaded = true; loaded = true;
loadedPromiseResolve(); loadedPromiseResolve();
} }
@ -325,38 +360,21 @@ function highlightResults(results, allTokens) {
* Returns noteIds which have at least one matching tokens * Returns noteIds which have at least one matching tokens
* *
* @param tokens * @param tokens
* @return {Set<String>} * @return {String[]}
*/ */
function getCandidateNotes(tokens) { function getCandidateNotes(tokens) {
const candidateNoteIds = new Set(); const candidateNotes = [];
for (const token of tokens) { for (const note of Object.values(notes)) {
for (const noteId in fulltext) { for (const token of tokens) {
if (!fulltext[noteId].includes(token)) { if (note.fulltext.includes(token)) {
continue; candidateNotes.push(note);
break;
} }
candidateNoteIds.add(noteId);
const note = notes[noteId];
const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable);
searchingAttrs:
// for matching inheritable attributes, include the whole note subtree to the candidates
for (const attr of inheritableAttrs) {
const lcName = attr.name.toLowerCase();
const lcValue = attr.value.toLowerCase();
for (const token of tokens) {
if (lcName.includes(token) || lcValue.includes(token)) {
note.addSubTreeNoteIdsTo(candidateNoteIds);
break searchingAttrs;
}
}
}
} }
} }
return candidateNoteIds;
return candidateNotes;
} }
async function findNotes(query) { async function findNotes(query) {
@ -370,18 +388,16 @@ async function findNotes(query) {
.split(/[ -]/) .split(/[ -]/)
.filter(token => token !== '/'); // '/' is used as separator .filter(token => token !== '/'); // '/' is used as separator
const candidateNoteIds = getCandidateNotes(allTokens); const candidateNotes = getCandidateNotes(allTokens);
// now we have set of noteIds which match at least one token // now we have set of noteIds which match at least one token
let results = []; let results = [];
const tokens = allTokens.slice(); const tokens = allTokens.slice();
for (const noteId of candidateNoteIds) { for (const note of candidateNotes) {
const note = notes[noteId];
// autocomplete should be able to find notes by their noteIds as well (only leafs) // autocomplete should be able to find notes by their noteIds as well (only leafs)
if (noteId === query) { if (note.noteId === query) {
search(note, [], [], results); search(note, [], [], results);
continue; continue;
} }
@ -415,7 +431,7 @@ async function findNotes(query) {
if (foundTokens.length > 0) { if (foundTokens.length > 0) {
const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); const remainingTokens = tokens.filter(token => !foundTokens.includes(token));
search(parentNote, remainingTokens, [noteId], results); search(parentNote, remainingTokens, [note.noteId], results);
} }
} }
} }
@ -678,11 +694,11 @@ function getNotePath(noteId) {
} }
} }
function evaluateSimilarity(text, note, results) { function evaluateSimilarity(sourceNote, candidateNote, results) {
let coeff = stringSimilarity.compareTwoStrings(text, note.title); let coeff = stringSimilarity.compareTwoStrings(sourceNote.fulltext, candidateNote.fulltext);
if (coeff > 0.4) { if (coeff > 0.4) {
const notePath = getSomePath(note); const notePath = getSomePath(candidateNote);
// this takes care of note hoisting // this takes care of note hoisting
if (!notePath) { if (!notePath) {
@ -693,7 +709,7 @@ function evaluateSimilarity(text, note, results) {
coeff -= 0.2; // archived penalization coeff -= 0.2; // archived penalization
} }
results.push({coeff, notePath, noteId: note.noteId}); results.push({coeff, notePath, noteId: candidateNote.noteId});
} }
} }
@ -707,16 +723,22 @@ function setImmediatePromise() {
}); });
} }
async function findSimilarNotes(title) { async function findSimilarNotes(noteId) {
const results = []; const results = [];
let i = 0; let i = 0;
const origNote = notes[noteId];
if (!origNote) {
return [];
}
for (const note of Object.values(notes)) { for (const note of Object.values(notes)) {
if (note.isProtected && !note.isDecrypted) { if (note.isProtected && !note.isDecrypted) {
continue; continue;
} }
evaluateSimilarity(title, note, results); evaluateSimilarity(origNote, note, results);
i++; i++;
@ -744,9 +766,12 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
delete notes[noteId]; delete notes[noteId];
} }
else if (noteId in notes) { else if (noteId in notes) {
const note = notes[noteId];
// we can assume we have protected session since we managed to update // we can assume we have protected session since we managed to update
notes[noteId].title = entity.title; note.title = entity.title;
notes[noteId].isDecrypted = true; note.isDecrypted = true;
note.fulltextCache = null;
} }
else { else {
notes[noteId] = new Note(entity); notes[noteId] = new Note(entity);
@ -760,6 +785,10 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
if (childNote) { if (childNote) {
childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId);
if (childNote.parents.length > 0) {
childNote.invalidateSubtreeCaches();
}
} }
const parentNote = notes[parentNoteId]; const parentNote = notes[parentNoteId];
@ -787,30 +816,46 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
} }
else if (entityName === 'attributes') { else if (entityName === 'attributes') {
const {attributeId, noteId} = entity; const {attributeId, noteId} = entity;
const note = notes[noteId];
const attr = attributes[attributeId];
if (entity.isDeleted) { if (entity.isDeleted) {
const note = notes[noteId]; if (note && attr) {
if (note) {
note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId);
if (attr.isAffectingSubtree) {
note.invalidateSubtreeCaches();
}
} }
delete attributes[entity.attributeId]; delete attributes[attributeId];
} }
else if (attributeId in attributes) { else if (attributeId in attributes) {
const attr = attributes[attributeId]; const attr = attributes[attributeId];
// attr name cannot change // attr name and isInheritable are immutable
attr.value = entity.value; attr.value = entity.value;
attr.isInheritable = entity.isInheritable;
if (attr.isAffectingSubtree) {
note.invalidateSubtreeFulltext();
}
else {
note.fulltextCache = null;
}
} }
else { else {
attributes[attributeId] = new Attribute(entity); const attr = new Attribute(entity);
attributes[attributeId] = attr;
const note = notes[noteId];
if (note) { if (note) {
note.ownedAttributes.push(attributes[attributeId]); note.ownedAttributes.push(attr);
if (attr.isAffectingSubtree) {
note.invalidateSubtreeCaches();
}
else {
this.invalidateThisCache();
}
} }
} }
} }