From 5098dda37694bc7cc181a9fb078309c96a79265a Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 11 Dec 2020 22:06:12 +0100 Subject: [PATCH] load initial tree from note cache --- .../0173__move_hash_to_entity_changes.sql | 18 +- src/routes/api/tree.js | 165 ++++++++++-------- src/services/note_cache/entities/attribute.js | 2 + src/services/note_cache/entities/branch.js | 4 + src/services/note_cache/note_cache.js | 4 - src/services/note_cache/note_cache_loader.js | 4 +- src/services/tree.js | 36 ---- 7 files changed, 108 insertions(+), 125 deletions(-) diff --git a/db/migrations/0173__move_hash_to_entity_changes.sql b/db/migrations/0173__move_hash_to_entity_changes.sql index 559166f73..11c4f1065 100644 --- a/db/migrations/0173__move_hash_to_entity_changes.sql +++ b/db/migrations/0173__move_hash_to_entity_changes.sql @@ -9,15 +9,15 @@ CREATE TABLE IF NOT EXISTS "mig_entity_changes" ( INSERT INTO mig_entity_changes (entityName, entityId, hash, sourceId, isSynced) SELECT entityName, entityId, '', sourceId, isSynced FROM entity_changes; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM api_tokens WHERE apiTokenId = entityId) WHERE entityName = 'api_tokens'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM attributes WHERE attributeId = entityId) WHERE entityName = 'attributes'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM branches WHERE branchId = entityId) WHERE entityName = 'branches'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM notes WHERE noteId = entityId) WHERE entityName = 'notes'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM note_contents WHERE noteId = entityId) WHERE entityName = 'note_contents'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM note_revisions WHERE noteRevisionId = entityId) WHERE entityName = 'note_revisions'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM note_revision_contents WHERE noteRevisionId = entityId) WHERE entityName = 'note_revision_contents'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM options WHERE name = entityId) WHERE entityName = 'options'; -UPDATE mig_entity_changes SET hash = (SELECT hash FROM recent_notes WHERE noteId = entityId) WHERE entityName = 'recent_notes'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM api_tokens WHERE apiTokenId = entityId), '') WHERE entityName = 'api_tokens'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM attributes WHERE attributeId = entityId), '') WHERE entityName = 'attributes'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM branches WHERE branchId = entityId), '') WHERE entityName = 'branches'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM notes WHERE noteId = entityId), '') WHERE entityName = 'notes'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM note_contents WHERE noteId = entityId), '') WHERE entityName = 'note_contents'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM note_revisions WHERE noteRevisionId = entityId), '') WHERE entityName = 'note_revisions'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM note_revision_contents WHERE noteRevisionId = entityId), '') WHERE entityName = 'note_revision_contents'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM options WHERE name = entityId), '') WHERE entityName = 'options'; +UPDATE mig_entity_changes SET hash = COALESCE((SELECT hash FROM recent_notes WHERE noteId = entityId), '') WHERE entityName = 'recent_notes'; DROP TABLE entity_changes; ALTER TABLE mig_entity_changes RENAME TO entity_changes; diff --git a/src/routes/api/tree.js b/src/routes/api/tree.js index 2a36b40b0..066705bf4 100644 --- a/src/routes/api/tree.js +++ b/src/routes/api/tree.js @@ -1,30 +1,61 @@ "use strict"; -const sql = require('../../services/sql'); -const optionService = require('../../services/options'); -const treeService = require('../../services/tree'); +const noteCache = require('../../services/note_cache/note_cache'); function getNotesAndBranchesAndAttributes(noteIds) { - const notes = treeService.getNotesIncludingAscendants(noteIds); + noteIds = new Set(noteIds); + const collectedNoteIds = new Set(); + const collectedAttributeIds = new Set(); + const collectedBranchIds = new Set(); - noteIds = new Set(notes.map(note => note.noteId)); + function collectEntityIds(note) { + if (collectedNoteIds.has(note.noteId)) { + return; + } - sql.fillNoteIdList(noteIds); + collectedNoteIds.add(note.noteId); - // joining child note to filter out not completely synchronised notes which would then cause errors later - // cannot do that with parent because of root note's 'none' parent - const branches = sql.getRows(` - SELECT - branches.branchId, - branches.noteId, - branches.parentNoteId, - branches.notePosition, - branches.prefix, - branches.isExpanded - FROM param_list - JOIN branches ON param_list.paramId = branches.parentNoteId - JOIN notes AS child ON child.noteId = branches.noteId - WHERE branches.isDeleted = 0`); + for (const branch of note.parentBranches) { + collectedBranchIds.add(branch.branchId); + + collectEntityIds(branch.parentNote); + } + + for (const attr of note.ownedAttributes) { + collectedAttributeIds.add(attr.attributeId); + + if (attr.type === 'relation' && attr.name === 'template') { + collectEntityIds(attr.targetNote); + } + } + } + + for (const noteId of noteIds) { + const note = noteCache.notes[noteId]; + + if (!note) { + continue; + } + + collectEntityIds(note); + } + + const notes = []; + + for (const noteId of collectedNoteIds) { + const note = noteCache.notes[noteId]; + + notes.push({ + noteId: note.noteId, + title: note.title, + isProtected: note.isProtected, + type: note.type, + mime: note.mime, + isDeleted: note.isDeleted + }); + } + + const branches = []; if (noteIds.has('root')) { branches.push({ @@ -37,39 +68,35 @@ function getNotesAndBranchesAndAttributes(noteIds) { }); } - const attributes = sql.getRows(` - SELECT - attributes.attributeId, - attributes.noteId, - attributes.type, - attributes.name, - attributes.value, - attributes.position, - attributes.isInheritable - FROM param_list - JOIN attributes ON attributes.noteId = param_list.paramId - OR (attributes.type = 'relation' AND attributes.value = param_list.paramId) - WHERE attributes.isDeleted = 0`); + for (const branchId of collectedBranchIds) { + const branch = noteCache.branches[branchId]; - // we don't really care about the direction of the relation - const missingTemplateNoteIds = attributes - .filter(attr => attr.type === 'relation' - && attr.name === 'template' - && !noteIds.has(attr.value)) - .map(attr => attr.value); - - if (missingTemplateNoteIds.length > 0) { - const templateData = getNotesAndBranchesAndAttributes(missingTemplateNoteIds); - - // there are going to be duplicates with simple concatenation, however: - // 1) shouldn't matter for the frontend which will update the entity twice - // 2) there shouldn't be many duplicates. There isn't that many templates - addArrays(notes, templateData.notes); - addArrays(branches, templateData.branches); - addArrays(attributes, templateData.attributes); + branches.push({ + branchId: branch.branchId, + noteId: branch.noteId, + parentNoteId: branch.parentNoteId, + notePosition: branch.notePosition, + prefix: branch.prefix, + isExpanded: branch.isExpanded + }); + } + + const attributes = []; + + for (const attributeId of collectedAttributeIds) { + const attribute = noteCache.attributes[attributeId]; + + attributes.push({ + attributeId: attribute.attributeId, + noteId: attribute.noteId, + type: attribute.type, + name: attribute.name, + value: attribute.value, + position: attribute.position, + isInheritable: attribute.isInheritable + }); } - // sorting in memory is faster branches.sort((a, b) => a.notePosition - b.notePosition < 0 ? -1 : 1); attributes.sort((a, b) => a.position - b.position < 0 ? -1 : 1); @@ -80,35 +107,25 @@ function getNotesAndBranchesAndAttributes(noteIds) { }; } -// should be fast based on https://stackoverflow.com/a/64826145/944162 -// in this case it is assumed that target is potentially much larger than elementsToAdd -function addArrays(target, elementsToAdd) { - while (elementsToAdd.length) { - target.push(elementsToAdd.shift()); - } - - return target; -} - function getTree(req) { const subTreeNoteId = req.query.subTreeNoteId || 'root'; + const collectedNoteIds = new Set(['root']); - // FIXME: this query does not return ascendants of template notes - const noteIds = sql.getColumn(` - WITH RECURSIVE - treeWithDescendants(noteId, isExpanded) AS ( - SELECT noteId, isExpanded FROM branches WHERE parentNoteId = ? AND isDeleted = 0 - UNION - SELECT branches.noteId, branches.isExpanded FROM branches - JOIN treeWithDescendants ON branches.parentNoteId = treeWithDescendants.noteId - WHERE treeWithDescendants.isExpanded = 1 - AND branches.isDeleted = 0 - ) - SELECT noteId FROM treeWithDescendants`, [subTreeNoteId]); + function collect(parentNote) { + for (const childNote of parentNote.children || []) { + collectedNoteIds.add(childNote.noteId); - noteIds.push(subTreeNoteId); + const childBranch = noteCache.getBranch(childNote.noteId, parentNote.noteId); - return getNotesAndBranchesAndAttributes(noteIds); + if (childBranch.isExpanded) { + collect(childBranch); + } + } + } + + collect(noteCache.notes[subTreeNoteId]); + + return getNotesAndBranchesAndAttributes(collectedNoteIds); } function load(req) { diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index 55c449177..1dc2c6106 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -14,6 +14,8 @@ class Attribute { this.type = row.type; /** @param {string} */ this.name = row.name.toLowerCase(); + /** @param {int} */ + this.position = row.position; if (typeof row.value !== 'string') { row.value = JSON.stringify(row.value); diff --git a/src/services/note_cache/entities/branch.js b/src/services/note_cache/entities/branch.js index 0e19fdaf0..002dc8f94 100644 --- a/src/services/note_cache/entities/branch.js +++ b/src/services/note_cache/entities/branch.js @@ -14,6 +14,10 @@ class Branch { this.parentNoteId = row.parentNoteId; /** @param {string} */ this.prefix = row.prefix; + /** @param {int} */ + this.notePosition = row.notePosition; + /** @param {boolean} */ + this.isExpanded = row.isExpanded; if (this.branchId === 'root') { return; diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js index b77d6ec5c..812050afd 100644 --- a/src/services/note_cache/note_cache.js +++ b/src/services/note_cache/note_cache.js @@ -1,9 +1,5 @@ "use strict"; -const Note = require('./entities/note'); -const Branch = require('./entities/branch'); -const Attribute = require('./entities/attribute'); - class NoteCache { constructor() { this.reset(); diff --git a/src/services/note_cache/note_cache_loader.js b/src/services/note_cache/note_cache_loader.js index eca5b6d83..dc57b2927 100644 --- a/src/services/note_cache/note_cache_loader.js +++ b/src/services/note_cache/note_cache_loader.js @@ -20,11 +20,11 @@ function load() { new Note(noteCache, row); } - for (const row of sql.iterateRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, [])) { + for (const row of sql.iterateRows(`SELECT branchId, noteId, parentNoteId, prefix, notePosition, isExpanded FROM branches WHERE isDeleted = 0`, [])) { new Branch(noteCache, row); } - for (const row of sql.iterateRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, [])) { + for (const row of sql.iterateRows(`SELECT attributeId, noteId, type, name, value, isInheritable, position FROM attributes WHERE isDeleted = 0`, [])) { new Attribute(noteCache, row); } diff --git a/src/services/tree.js b/src/services/tree.js index 43e6bdc81..582517c36 100644 --- a/src/services/tree.js +++ b/src/services/tree.js @@ -6,41 +6,6 @@ const Branch = require('../entities/branch'); const entityChangesService = require('./entity_changes.js'); const protectedSessionService = require('./protected_session'); -function getNotesIncludingAscendants(noteIds) { - noteIds = Array.from(new Set(noteIds)); - - sql.fillNoteIdList(noteIds); - - // we return also deleted notes which have been specifically asked for - - const notes = sql.getRows(` - WITH RECURSIVE - treeWithAscendants AS ( - SELECT paramId AS noteId FROM param_list - UNION - SELECT branches.parentNoteId FROM branches - JOIN treeWithAscendants ON branches.noteId = treeWithAscendants.noteId - WHERE branches.isDeleted = 0 - ) - SELECT - noteId, - title, - isProtected, - type, - mime, - isDeleted - FROM notes - JOIN treeWithAscendants USING(noteId)`); - - protectedSessionService.decryptNotes(notes); - - notes.forEach(note => { - note.isProtected = !!note.isProtected - }); - - return notes; -} - function getNotes(noteIds) { // we return also deleted notes which have been specifically asked for const notes = sql.getManyRows(` @@ -225,7 +190,6 @@ function setNoteToParent(noteId, prefix, parentNoteId) { module.exports = { getNotes, - getNotesIncludingAscendants, validateParentChild, sortNotesAlphabetically, setNoteToParent