improvements to notemap in relation to search

src/becca/entities/note.js

@@ -10,7 +10,7 @@ const AbstractEntity = require("./abstract_entity");
 const NoteRevision = require("./note_revision");
 const TaskContext = require("../../services/task_context");
 const dayjs = require("dayjs");
-const utc = require('dayjs/plugin/utc')
+const utc = require('dayjs/plugin/utc');
 dayjs.extend(utc)
 
 const LABEL = 'label';
@@ -839,30 +839,76 @@ class Note extends AbstractEntity {
         return Array.from(set);
     }
 
-    /** @returns {Note[]} */
+    /**
-    getSubtreeNotes(includeArchived = true) {
+     * @returns {{notes: Note[], relationships: {parentNoteId, childNoteId}[]}}
+     */
+    getSubtree({includeArchived = true, resolveSearch = false} = {}) {
         const noteSet = new Set();
+        const relationships = []; // list of tuples parentNoteId -> childNoteId
+
+        function resolveSearchNote(searchNote) {
+            try {
+                const searchService = require("../../services/search/services/search");
+                const becca = searchNote.becca;
+                const {searchResultNoteIds} = searchService.searchFromNote(searchNote);
+
+                for (const resultNoteId of searchResultNoteIds) {
+                    const resultNote = becca.notes[resultNoteId];
+
+                    if (resultNote) {
+                        addSubtreeNotesInner(resultNote, searchNote);
+                    }
+                }
+            }
+            catch (e) {
+                log.error(`Could not resolve search note ${searchNote?.noteId}: ${e.message}`);
+            }
+        }
+
+        function addSubtreeNotesInner(note, parentNote = null) {
+            if (parentNote) {
+                // this needs to happen first before noteSet check to include all clone relationships
+                relationships.push({
+                    parentNoteId: parentNote.noteId,
+                    childNoteId: note.noteId
+                });
+            }
+
+            if (noteSet.has(note)) {
+                return;
+            }
 
-        function addSubtreeNotesInner(note) {
             if (!includeArchived && note.isArchived) {
                 return;
             }
 
             noteSet.add(note);
 
+            if (note.type === 'search') {
+                if (resolveSearch) {
+                    resolveSearchNote(note);
+                }
+            }
+            else {
                 for (const childNote of note.children) {
-                addSubtreeNotesInner(childNote);
+                    addSubtreeNotesInner(childNote, note);
+                }
             }
         }
 
         addSubtreeNotesInner(this);
 
-        return Array.from(noteSet);
+        return {
+            notes: Array.from(noteSet),
+            relationships
+        };
     }
 
     /** @returns {String[]} */
-    getSubtreeNoteIds(includeArchived = true) {
+    getSubtreeNoteIds({includeArchived = true, resolveSearch = false} = {}) {
-        return this.getSubtreeNotes(includeArchived).map(note => note.noteId);
+        return this.getSubtree({includeArchived, resolveSearch})
+            .notes
+            .map(note => note.noteId);
     }
 
     getDescendantNoteIds() {

src/public/app/widgets/note_map.js

@@ -86,8 +86,6 @@ export default class NoteMapWidget extends NoteContextAwareWidget {
 
         this.mapType = this.note.getLabelValue("mapType") === "tree" ? "tree" : "link";
 
-        this.setDimensions();
-
         await libraryLoader.requireLibrary(libraryLoader.FORCE_GRAPH);
 
         this.graph = ForceGraph()(this.$container[0])
@@ -121,7 +119,7 @@ export default class NoteMapWidget extends NoteContextAwareWidget {
                 .linkCanvasObjectMode(() => "after");
         }
 
-        let mapRootNoteId = this.getMapRootNoteId();
+        const mapRootNoteId = this.getMapRootNoteId();
         const data = await this.loadNotesAndRelations(mapRootNoteId);
 
         const nodeLinkRatio = data.nodes.length / data.links.length;
@@ -264,7 +262,7 @@ export default class NoteMapWidget extends NoteContextAwareWidget {
         return {
             nodes: this.nodes,
             links: links.map(link => ({
-                id: link.id,
+                id: `${link.sourceNoteId}-${link.targetNoteId}`,
                 source: link.sourceNoteId,
                 target: link.targetNoteId,
                 name: link.names.join(", ")
@@ -333,6 +331,8 @@ export default class NoteMapWidget extends NoteContextAwareWidget {
 
         if (this.widgetMode === 'ribbon') {
             setTimeout(() => {
+                this.setDimensions();
+
                 const subGraphNoteIds = this.getSubGraphConnectedToCurrentNote(data);
 
                 this.graph.zoomToFit(400, 50, node => subGraphNoteIds.has(node.id));
@@ -345,6 +345,8 @@ export default class NoteMapWidget extends NoteContextAwareWidget {
         else if (this.widgetMode === 'type') {
             if (data.nodes.length > 1) {
                 setTimeout(() => {
+                    this.setDimensions();
+
                     this.graph.zoomToFit(400, 10);
 
                     if (data.nodes.length < 30) {
@@ -398,7 +400,12 @@ export default class NoteMapWidget extends NoteContextAwareWidget {
     }
 
     entitiesReloadedEvent({loadResults}) {
-        if (loadResults.getAttributes(this.componentId).find(attr => attr.name === 'mapType' && attributeService.isAffecting(attr, this.note))) {
+        if (loadResults.getAttributes(this.componentId).find(
+            attr =>
+                attr.type === 'label'
+                && ['mapType', 'mapRootNoteId'].includes(attr.name)
+                && attributeService.isAffecting(attr, this.note)
+        )) {
             this.refresh();
         }
     }

src/routes/api/note_map.js

@@ -83,12 +83,13 @@ function getNeighbors(note, depth) {
 
 function getLinkMap(req) {
     const mapRootNote = becca.getNote(req.params.noteId);
-    // if the map root itself has ignore (journal typically) then there wouldn't be anything to display so
+    // if the map root itself has exclude attribute (journal typically) then there wouldn't be anything to display, so
     // we'll just ignore it
     const ignoreExcludeFromNoteMap = mapRootNote.hasLabel('excludeFromNoteMap');
+    const subtree = mapRootNote.getSubtree({includeArchived: false, resolveSearch: true});
 
     const noteIds = new Set(
-        mapRootNote.getSubtreeNotes(false)
+        subtree.notes
             .filter(note => ignoreExcludeFromNoteMap || !note.hasLabel('excludeFromNoteMap'))
             .map(note => note.noteId)
     );
@@ -142,8 +143,9 @@ function getTreeMap(req) {
     // if the map root itself has ignore (journal typically) then there wouldn't be anything to display so
     // we'll just ignore it
     const ignoreExcludeFromNoteMap = mapRootNote.hasLabel('excludeFromNoteMap');
+    const subtree = mapRootNote.getSubtree({includeArchived: false, resolveSearch: true});
 
-    const notes = mapRootNote.getSubtreeNotes(false)
+    const notes = subtree.notes
         .filter(note => ignoreExcludeFromNoteMap || !note.hasLabel('excludeFromNoteMap'))
         .filter(note => {
             if (note.type !== 'image' || note.getChildNotes().length > 0) {
@@ -170,25 +172,40 @@ function getTreeMap(req) {
 
     const links = [];
 
-    for (const branch of Object.values(becca.branches)) {
+    for (const {parentNoteId, childNoteId} of subtree.relationships) {
-        if (!noteIds.has(branch.parentNoteId) || !noteIds.has(branch.noteId)) {
+        if (!noteIds.has(parentNoteId) || !noteIds.has(childNoteId)) {
             continue;
         }
 
         links.push({
-            id: branch.branchId,
+            sourceNoteId: parentNoteId,
-            sourceNoteId: branch.parentNoteId,
+            targetNoteId: childNoteId
-            targetNoteId: branch.noteId
         });
     }
 
+    const noteIdToDescendantCountMap = buildDescendantCountMap();
+
+    updateDescendantCountMapForSearch(noteIdToDescendantCountMap, subtree.relationships);
+
     return {
         notes: notes,
-        noteIdToDescendantCountMap: buildDescendantCountMap(),
+        noteIdToDescendantCountMap: noteIdToDescendantCountMap,
         links: links
     };
 }
 
+function updateDescendantCountMapForSearch(noteIdToDescendantCountMap, relationships) {
+    for (const {parentNoteId, childNoteId} of relationships) {
+        const parentNote = becca.notes[parentNoteId];
+        if (!parentNote || parentNote.type !== 'search') {
+            continue;
+        }
+
+        noteIdToDescendantCountMap[parentNote.noteId] = noteIdToDescendantCountMap[parentNoteId] || 0;
+        noteIdToDescendantCountMap[parentNote.noteId] += noteIdToDescendantCountMap[childNoteId] || 1;
+    }
+}
+
 function removeImages(document) {
     const images = document.getElementsByTagName('img');
     while (images.length > 0) {

src/routes/api/search.js

@@ -2,49 +2,11 @@
 
 const becca = require('../../becca/becca');
 const SearchContext = require('../../services/search/search_context');
-const log = require('../../services/log');
-const scriptService = require('../../services/script');
 const searchService = require('../../services/search/services/search');
 const bulkActionService = require("../../services/bulk_actions");
 const cls = require("../../services/cls");
 const {formatAttrForSearch} = require("../../services/attribute_formatter");
 
-function searchFromNoteInt(note) {
-    let searchResultNoteIds, highlightedTokens;
-
-    const searchScript = note.getRelationValue('searchScript');
-    const searchString = note.getLabelValue('searchString');
-
-    if (searchScript) {
-        searchResultNoteIds = searchFromRelation(note, 'searchScript');
-        highlightedTokens = [];
-    } else {
-        const searchContext = new SearchContext({
-            fastSearch: note.hasLabel('fastSearch'),
-            ancestorNoteId: note.getRelationValue('ancestor'),
-            ancestorDepth: note.getLabelValue('ancestorDepth'),
-            includeArchivedNotes: note.hasLabel('includeArchivedNotes'),
-            orderBy: note.getLabelValue('orderBy'),
-            orderDirection: note.getLabelValue('orderDirection'),
-            limit: note.getLabelValue('limit'),
-            debug: note.hasLabel('debug'),
-            fuzzyAttributeSearch: false
-        });
-
-        searchResultNoteIds = searchService.findResultsWithQuery(searchString, searchContext)
-            .map(sr => sr.noteId);
-
-        highlightedTokens = searchContext.highlightedTokens;
-    }
-
-    // we won't return search note's own noteId
-    // also don't allow root since that would force infinite cycle
-    return {
-        searchResultNoteIds: searchResultNoteIds.filter(resultNoteId => !['root', note.noteId].includes(resultNoteId)),
-        highlightedTokens
-    };
-}
-
 function searchFromNote(req) {
     const note = becca.getNote(req.params.noteId);
 
@@ -61,7 +23,7 @@ function searchFromNote(req) {
         return [400, `Note ${req.params.noteId} is not a search note.`]
     }
 
-    return searchFromNoteInt(note);
+    return searchService.searchFromNote(note);
 }
 
 function searchAndExecute(req) {
@@ -80,48 +42,11 @@ function searchAndExecute(req) {
         return [400, `Note ${req.params.noteId} is not a search note.`]
     }
 
-    const {searchResultNoteIds} = searchFromNoteInt(note);
+    const {searchResultNoteIds} = searchService.searchFromNote(note);
 
     bulkActionService.executeActions(note, searchResultNoteIds);
 }
 
-function searchFromRelation(note, relationName) {
-    const scriptNote = note.getRelationTarget(relationName);
-
-    if (!scriptNote) {
-        log.info(`Search note's relation ${relationName} has not been found.`);
-
-        return [];
-    }
-
-    if (!scriptNote.isJavaScript() || scriptNote.getScriptEnv() !== 'backend') {
-        log.info(`Note ${scriptNote.noteId} is not executable.`);
-
-        return [];
-    }
-
-    if (!note.isContentAvailable()) {
-        log.info(`Note ${scriptNote.noteId} is not available outside of protected session.`);
-
-        return [];
-    }
-
-    const result = scriptService.executeNote(scriptNote, { originEntity: note });
-
-    if (!Array.isArray(result)) {
-        log.info(`Result from ${scriptNote.noteId} is not an array.`);
-
-        return [];
-    }
-
-    if (result.length === 0) {
-        return [];
-    }
-
-    // we expect either array of noteIds (strings) or notes, in that case we extract noteIds ourselves
-    return typeof result[0] === 'string' ? result : result.map(item => item.noteId);
-}
-
 function quickSearch(req) {
     const {searchString} = req.params;
 

src/routes/api/stats.js

@@ -29,7 +29,7 @@ function getSubtreeSize(req) {
         return [404, `Note ${noteId} was not found.`];
     }
 
-    const subTreeNoteIds = note.getSubtreeNotes().map(note => note.noteId);
+    const subTreeNoteIds = note.getSubtreeNoteIds();
 
     sql.fillParamList(subTreeNoteIds);
 

src/services/search/expressions/ancestor.js

@@ -23,7 +23,9 @@ class AncestorExp extends Expression {
             return new NoteSet([]);
         }
 
-        const subTreeNoteSet = new NoteSet(ancestorNote.getSubtreeNotes()).intersection(inputNoteSet);
+        const subtree = ancestorNote.getSubtree();
+
+        const subTreeNoteSet = new NoteSet(subtree.notes).intersection(inputNoteSet);
 
         if (!this.ancestorDepthComparator) {
             return subTreeNoteSet;

src/services/search/expressions/descendant_of.js

@@ -18,7 +18,7 @@ class DescendantOfExp extends Expression {
         const subTreeNoteSet = new NoteSet();
 
         for (const note of subResNoteSet.notes) {
-            subTreeNoteSet.addAll(note.getSubtreeNotes());
+            subTreeNoteSet.addAll(note.getSubtree().notes);
         }
 
         return inputNoteSet.intersection(subTreeNoteSet);

src/services/search/services/search.js

@@ -10,6 +10,80 @@ const becca = require('../../../becca/becca');
 const beccaService = require('../../../becca/becca_service');
 const utils = require('../../utils');
 const log = require('../../log');
+const scriptService = require("../../script.js");
+
+function searchFromNote(note) {
+    let searchResultNoteIds, highlightedTokens;
+
+    const searchScript = note.getRelationValue('searchScript');
+    const searchString = note.getLabelValue('searchString');
+
+    if (searchScript) {
+        searchResultNoteIds = searchFromRelation(note, 'searchScript');
+        highlightedTokens = [];
+    } else {
+        const searchContext = new SearchContext({
+            fastSearch: note.hasLabel('fastSearch'),
+            ancestorNoteId: note.getRelationValue('ancestor'),
+            ancestorDepth: note.getLabelValue('ancestorDepth'),
+            includeArchivedNotes: note.hasLabel('includeArchivedNotes'),
+            orderBy: note.getLabelValue('orderBy'),
+            orderDirection: note.getLabelValue('orderDirection'),
+            limit: note.getLabelValue('limit'),
+            debug: note.hasLabel('debug'),
+            fuzzyAttributeSearch: false
+        });
+
+        searchResultNoteIds = findResultsWithQuery(searchString, searchContext)
+            .map(sr => sr.noteId);
+
+        highlightedTokens = searchContext.highlightedTokens;
+    }
+
+    // we won't return search note's own noteId
+    // also don't allow root since that would force infinite cycle
+    return {
+        searchResultNoteIds: searchResultNoteIds.filter(resultNoteId => !['root', note.noteId].includes(resultNoteId)),
+        highlightedTokens
+    };
+}
+
+function searchFromRelation(note, relationName) {
+    const scriptNote = note.getRelationTarget(relationName);
+
+    if (!scriptNote) {
+        log.info(`Search note's relation ${relationName} has not been found.`);
+
+        return [];
+    }
+
+    if (!scriptNote.isJavaScript() || scriptNote.getScriptEnv() !== 'backend') {
+        log.info(`Note ${scriptNote.noteId} is not executable.`);
+
+        return [];
+    }
+
+    if (!note.isContentAvailable()) {
+        log.info(`Note ${scriptNote.noteId} is not available outside of protected session.`);
+
+        return [];
+    }
+
+    const result = scriptService.executeNote(scriptNote, { originEntity: note });
+
+    if (!Array.isArray(result)) {
+        log.info(`Result from ${scriptNote.noteId} is not an array.`);
+
+        return [];
+    }
+
+    if (result.length === 0) {
+        return [];
+    }
+
+    // we expect either array of noteIds (strings) or notes, in that case we extract noteIds ourselves
+    return typeof result[0] === 'string' ? result : result.map(item => item.noteId);
+}
 
 function loadNeededInfoFromDatabase() {
     const sql = require('../../sql');
@@ -288,6 +362,7 @@ function formatAttribute(attr) {
 }
 
 module.exports = {
+    searchFromNote,
     searchNotesForAutocomplete,
     findResultsWithQuery,
     findFirstNoteWithQuery,