From 99aa481acea6634892b467e228c40cf2bb3f87ee Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 20 May 2020 00:03:33 +0200 Subject: [PATCH] refactoring for testing parser --- spec/parens.spec.js | 2 +- spec/parser.spec.js | 10 ++ src/public/app/services/utils.js | 4 +- src/services/note_cache/entities/note.js | 1 - src/services/note_cache/note_cache.js | 163 +---------------- src/services/note_cache/note_cache_loader.js | 169 ++++++++++++++++++ src/services/note_cache/note_cache_service.js | 2 + .../{exists.js => attribute_exists.js} | 4 +- .../{equals.js => field_comparison.js} | 4 +- .../search/expressions/note_cache_fulltext.js | 4 +- .../expressions/note_content_fulltext.js | 2 + src/services/search/parser.js | 12 +- src/services/sql_init.js | 2 +- 13 files changed, 204 insertions(+), 175 deletions(-) create mode 100644 spec/parser.spec.js create mode 100644 src/services/note_cache/note_cache_loader.js rename src/services/search/expressions/{exists.js => attribute_exists.js} (93%) rename src/services/search/expressions/{equals.js => field_comparison.js} (94%) diff --git a/spec/parens.spec.js b/spec/parens.spec.js index 8c7db6d56..c55896e44 100644 --- a/spec/parens.spec.js +++ b/spec/parens.spec.js @@ -1,7 +1,7 @@ const parens = require('../src/services/search/parens'); describe("Parens handler", () => { - it("handles parens", () => {console.log(parens(["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"])) + it("handles parens", () => { expect(parens(["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"])) .toEqual([ [ diff --git a/spec/parser.spec.js b/spec/parser.spec.js new file mode 100644 index 000000000..7d9ad2a55 --- /dev/null +++ b/spec/parser.spec.js @@ -0,0 +1,10 @@ +const parser = require('../src/services/search/parser'); + +describe("Parser", () => { + it("fulltext parser without content", () => { + const exps = parser(["hello", "hi"], [], false); + + expect(exps.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(exps.tokens).toEqual(["hello", "hi"]); + }); +}); diff --git a/src/public/app/services/utils.js b/src/public/app/services/utils.js index 0fa8e9642..504f5ae45 100644 --- a/src/public/app/services/utils.js +++ b/src/public/app/services/utils.js @@ -187,7 +187,7 @@ function setCookie(name, value) { } function setSessionCookie(name, value) { - document.cookie = name + "=" + (value || "") + ";"; + document.cookie = name + "=" + (value || "") + "; SameSite=Strict"; } function getCookie(name) { @@ -356,4 +356,4 @@ export default { copySelectionToClipboard, isCKEditorInitialized, dynamicRequire -}; \ No newline at end of file +}; diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index adc120d15..8d3a7abc2 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -1,6 +1,5 @@ "use strict"; -const noteCache = require('../note_cache'); const protectedSessionService = require('../../protected_session'); class Note { diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js index 8aaa688bc..12942ee7f 100644 --- a/src/services/note_cache/note_cache.js +++ b/src/services/note_cache/note_cache.js @@ -3,9 +3,6 @@ const Note = require('./entities/note'); const Branch = require('./entities/branch'); const Attribute = require('./entities/attribute'); -const sql = require('../sql.js'); -const sqlInit = require('../sql_init.js'); -const eventService = require('../events.js'); class NoteCache { constructor() { @@ -21,7 +18,8 @@ class NoteCache { this.attributeIndex = null; this.loaded = false; - this.loadedPromise = this.load(); + this.loadedResolve = null; + this.loadedPromise = new Promise(res => {this.loadedResolve = res;}); } /** @return {Attribute[]} */ @@ -29,36 +27,6 @@ class NoteCache { return this.attributeIndex[`${type}-${name}`] || []; } - async load() { - await sqlInit.dbReady; - - this.notes = await this.getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, - row => new Note(this, row)); - - this.branches = await this.getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, - row => new Branch(this, row)); - - this.attributeIndex = []; - - this.attributes = await this.getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, - row => new Attribute(this, row)); - - this.loaded = true; - } - - async getMappedRows(query, cb) { - const map = {}; - const results = await sql.getRows(query, []); - - for (const row of results) { - const keys = Object.keys(row); - - map[row[keys[0]]] = cb(row); - } - - return map; - } - decryptProtectedNotes() { for (const note of Object.values(this.notes)) { note.decrypt(); @@ -72,131 +40,4 @@ class NoteCache { const noteCache = new NoteCache(); -eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], async ({entityName, entity}) => { - // note that entity can also be just POJO without methods if coming from sync - - if (!noteCache.loaded) { - return; - } - - if (entityName === 'notes') { - const {noteId} = entity; - - if (entity.isDeleted) { - delete noteCache.notes[noteId]; - } - else if (noteId in noteCache.notes) { - const note = noteCache.notes[noteId]; - - // we can assume we have protected session since we managed to update - note.title = entity.title; - note.isProtected = entity.isProtected; - note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; - note.flatTextCache = null; - - note.decrypt(); - } - else { - const note = new Note(entity); - noteCache.notes[noteId] = note; - - note.decrypt(); - } - } - else if (entityName === 'branches') { - const {branchId, noteId, parentNoteId} = entity; - const childNote = noteCache.notes[noteId]; - - if (entity.isDeleted) { - if (childNote) { - childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); - childNote.parentBranches = childNote.parentBranches.filter(branch => branch.branchId !== branchId); - - if (childNote.parents.length > 0) { - childNote.invalidateSubtreeCaches(); - } - } - - const parentNote = noteCache.notes[parentNoteId]; - - if (parentNote) { - parentNote.children = parentNote.children.filter(child => child.noteId !== noteId); - } - - delete noteCache.childParentToBranch[`${noteId}-${parentNoteId}`]; - delete noteCache.branches[branchId]; - } - else if (branchId in noteCache.branches) { - // only relevant thing which can change in a branch is prefix - noteCache.branches[branchId].prefix = entity.prefix; - - if (childNote) { - childNote.flatTextCache = null; - } - } - else { - noteCache.branches[branchId] = new Branch(entity); - - if (childNote) { - childNote.resortParents(); - } - } - } - else if (entityName === 'attributes') { - const {attributeId, noteId} = entity; - const note = noteCache.notes[noteId]; - const attr = noteCache.attributes[attributeId]; - - if (entity.isDeleted) { - if (note && attr) { - // first invalidate and only then remove the attribute (otherwise invalidation wouldn't be complete) - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeCaches(); - } - - note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); - - const targetNote = attr.targetNote; - - if (targetNote) { - targetNote.targetRelations = targetNote.targetRelations.filter(rel => rel.attributeId !== attributeId); - } - } - - delete noteCache.attributes[attributeId]; - delete noteCache.attributeIndex[`${attr.type}-${attr.name}`]; - } - else if (attributeId in noteCache.attributes) { - const attr = noteCache.attributes[attributeId]; - - // attr name and isInheritable are immutable - attr.value = entity.value; - - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeFlatText(); - } - else { - note.flatTextCache = null; - } - } - else { - const attr = new Attribute(entity); - noteCache.attributes[attributeId] = attr; - - if (note) { - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeCaches(); - } - else { - this.invalidateThisCache(); - } - } - } - } -}); - -eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { - noteCache.loadedPromise.then(() => noteCache.decryptProtectedNotes()); -}); - module.exports = noteCache; diff --git a/src/services/note_cache/note_cache_loader.js b/src/services/note_cache/note_cache_loader.js new file mode 100644 index 000000000..33d387c3e --- /dev/null +++ b/src/services/note_cache/note_cache_loader.js @@ -0,0 +1,169 @@ +"use strict"; + +const sql = require('../sql.js'); +const sqlInit = require('../sql_init.js'); +const eventService = require('../events.js'); +const noteCache = require('./note_cache'); +const Note = require('./entities/note'); +const Branch = require('./entities/branch'); +const Attribute = require('./entities/attribute'); + +async function load() { + await sqlInit.dbReady; + + noteCache.notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, + row => new Note(noteCache, row)); + + noteCache.branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, + row => new Branch(noteCache, row)); + + noteCache.attributeIndex = []; + + noteCache.attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, + row => new Attribute(noteCache, row)); + + noteCache.loaded = true; + noteCache.loadedResolve(); +} + +async function getMappedRows(query, cb) { + const map = {}; + const results = await sql.getRows(query, []); + + for (const row of results) { + const keys = Object.keys(row); + + map[row[keys[0]]] = cb(row); + } + + return map; +} + +eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], async ({entityName, entity}) => { + // note that entity can also be just POJO without methods if coming from sync + + if (!noteCache.loaded) { + return; + } + + if (entityName === 'notes') { + const {noteId} = entity; + + if (entity.isDeleted) { + delete noteCache.notes[noteId]; + } + else if (noteId in noteCache.notes) { + const note = noteCache.notes[noteId]; + + // we can assume we have protected session since we managed to update + note.title = entity.title; + note.isProtected = entity.isProtected; + note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; + note.flatTextCache = null; + + note.decrypt(); + } + else { + const note = new Note(entity); + noteCache.notes[noteId] = note; + + note.decrypt(); + } + } + else if (entityName === 'branches') { + const {branchId, noteId, parentNoteId} = entity; + const childNote = noteCache.notes[noteId]; + + if (entity.isDeleted) { + if (childNote) { + childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); + childNote.parentBranches = childNote.parentBranches.filter(branch => branch.branchId !== branchId); + + if (childNote.parents.length > 0) { + childNote.invalidateSubtreeCaches(); + } + } + + const parentNote = noteCache.notes[parentNoteId]; + + if (parentNote) { + parentNote.children = parentNote.children.filter(child => child.noteId !== noteId); + } + + delete noteCache.childParentToBranch[`${noteId}-${parentNoteId}`]; + delete noteCache.branches[branchId]; + } + else if (branchId in noteCache.branches) { + // only relevant thing which can change in a branch is prefix + noteCache.branches[branchId].prefix = entity.prefix; + + if (childNote) { + childNote.flatTextCache = null; + } + } + else { + noteCache.branches[branchId] = new Branch(entity); + + if (childNote) { + childNote.resortParents(); + } + } + } + else if (entityName === 'attributes') { + const {attributeId, noteId} = entity; + const note = noteCache.notes[noteId]; + const attr = noteCache.attributes[attributeId]; + + if (entity.isDeleted) { + if (note && attr) { + // first invalidate and only then remove the attribute (otherwise invalidation wouldn't be complete) + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeCaches(); + } + + note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); + + const targetNote = attr.targetNote; + + if (targetNote) { + targetNote.targetRelations = targetNote.targetRelations.filter(rel => rel.attributeId !== attributeId); + } + } + + delete noteCache.attributes[attributeId]; + delete noteCache.attributeIndex[`${attr.type}-${attr.name}`]; + } + else if (attributeId in noteCache.attributes) { + const attr = noteCache.attributes[attributeId]; + + // attr name and isInheritable are immutable + attr.value = entity.value; + + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeFlatText(); + } + else { + note.flatTextCache = null; + } + } + else { + const attr = new Attribute(entity); + noteCache.attributes[attributeId] = attr; + + if (note) { + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeCaches(); + } + else { + note.invalidateThisCache(); + } + } + } + } +}); + +eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { + noteCache.loadedPromise.then(() => noteCache.decryptProtectedNotes()); +}); + +module.exports = load; diff --git a/src/services/note_cache/note_cache_service.js b/src/services/note_cache/note_cache_service.js index 8007c9cb3..35b078be1 100644 --- a/src/services/note_cache/note_cache_service.js +++ b/src/services/note_cache/note_cache_service.js @@ -4,6 +4,8 @@ const noteCache = require('./note_cache'); const hoistedNoteService = require('../hoisted_note'); const stringSimilarity = require('string-similarity'); +require('./note_cache_loader')(); + function isNotePathArchived(notePath) { const noteId = notePath[notePath.length - 1]; const note = noteCache.notes[noteId]; diff --git a/src/services/search/expressions/exists.js b/src/services/search/expressions/attribute_exists.js similarity index 93% rename from src/services/search/expressions/exists.js rename to src/services/search/expressions/attribute_exists.js index 25c3a9245..e80f8a9a8 100644 --- a/src/services/search/expressions/exists.js +++ b/src/services/search/expressions/attribute_exists.js @@ -3,7 +3,7 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class ExistsExp { +class AttributeExistsExp { constructor(attributeType, attributeName) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -31,4 +31,4 @@ class ExistsExp { } } -module.exports = ExistsExp; +module.exports = AttributeExistsExp; diff --git a/src/services/search/expressions/equals.js b/src/services/search/expressions/field_comparison.js similarity index 94% rename from src/services/search/expressions/equals.js rename to src/services/search/expressions/field_comparison.js index ecae22241..8c95f2169 100644 --- a/src/services/search/expressions/equals.js +++ b/src/services/search/expressions/field_comparison.js @@ -3,7 +3,7 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class EqualsExp { +class FieldComparisonExp { constructor(attributeType, attributeName, operator, attributeValue) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -33,4 +33,4 @@ class EqualsExp { } } -module.exports = EqualsExp; +module.exports = FieldComparisonExp; diff --git a/src/services/search/expressions/note_cache_fulltext.js b/src/services/search/expressions/note_cache_fulltext.js index f09df1e88..c3cbc7b64 100644 --- a/src/services/search/expressions/note_cache_fulltext.js +++ b/src/services/search/expressions/note_cache_fulltext.js @@ -2,7 +2,6 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -const noteCacheService = require('../../note_cache/note_cache_service'); class NoteCacheFulltextExp { constructor(tokens) { @@ -10,6 +9,9 @@ class NoteCacheFulltextExp { } execute(noteSet, searchContext) { + // has deps on SQL which breaks unit test so needs to be dynamically required + const noteCacheService = require('../../note_cache/note_cache_service'); + const resultNoteSet = new NoteSet(); const candidateNotes = this.getCandidateNotes(noteSet); diff --git a/src/services/search/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js index 4d6900802..d7591c7ef 100644 --- a/src/services/search/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -12,6 +12,8 @@ class NoteContentFulltextExp { const resultNoteSet = new NoteSet(); const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); + const sql = require('../../sql'); + const noteIds = await sql.getColumn(` SELECT notes.noteId FROM notes diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 8ad021f89..1c692eaaf 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -1,8 +1,8 @@ const AndExp = require('./expressions/and'); const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); -const ExistsExp = require('./expressions/exists'); -const EqualsExp = require('./expressions/equals'); +const AttributeExistsExp = require('./expressions/attribute_exists'); +const FieldComparisonExp = require('./expressions/field_comparison'); const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); @@ -44,12 +44,12 @@ function getExpressions(tokens) { const type = token.startsWith('#') ? 'label' : 'relation'; if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { - expressions.push(new EqualsExp(type, token.substr(1), tokens[i + 1], tokens[i + 2])); + expressions.push(new FieldComparisonExp(type, token.substr(1), tokens[i + 1], tokens[i + 2])); i += 2; } else { - expressions.push(new ExistsExp(type, token.substr(1))); + expressions.push(new AttributeExistsExp(type, token.substr(1))); } } else if (['and', 'or'].includes(token.toLowerCase())) { @@ -71,6 +71,8 @@ function getExpressions(tokens) { op = 'and'; } } + + return expressions; } function parse(fulltextTokens, expressionTokens, includingNoteContent) { @@ -79,3 +81,5 @@ function parse(fulltextTokens, expressionTokens, includingNoteContent) { ...getExpressions(expressionTokens) ]); } + +module.exports = parse; diff --git a/src/services/sql_init.js b/src/services/sql_init.js index eaf0aaf23..6a50492e3 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -197,4 +197,4 @@ module.exports = { createInitialDatabase, createDatabaseForSync, dbInitialized -}; \ No newline at end of file +};