From 872e81fe1fd958d08b658ad239ddc635b4b32d74 Mon Sep 17 00:00:00 2001
From: zadam <adam.zivner@gmail.com>
Date: Fri, 22 Jan 2021 22:20:17 +0100
Subject: [PATCH] search now supports searching / ordering by note size

---
 src/public/app/services/tree_cache.js         |  3 -
 src/public/app/widgets/search_definition.js   |  3 +
 src/routes/api/search.js                      |  4 +-
 src/services/note_cache/entities/note.js      |  9 +++
 .../search/expressions/property_comparison.js | 11 +++-
 src/services/search/search_context.js         |  3 +
 src/services/search/services/parse.js         | 10 ++--
 src/services/search/services/search.js        | 57 +++++++++++++++++--
 src/services/search/value_extractor.js        | 13 ++++-
 9 files changed, 93 insertions(+), 20 deletions(-)

diff --git a/src/public/app/services/tree_cache.js b/src/public/app/services/tree_cache.js
index 97a217be1..6e2e67dd3 100644
--- a/src/public/app/services/tree_cache.js
+++ b/src/public/app/services/tree_cache.js
@@ -172,9 +172,6 @@ class TreeCache {
                     throw new Error(`Search note ${note.noteId} failed: ${searchResultNoteIds}`);
                 }
 
-                // force to load all the notes at once instead of one by one
-                await this.getNotes(searchResultNoteIds);
-
                 // reset all the virtual branches from old search results
                 if (note.noteId in treeCache.notes) {
                     treeCache.notes[note.noteId].children = [];
diff --git a/src/public/app/widgets/search_definition.js b/src/public/app/widgets/search_definition.js
index e67ee9b52..3b4467cb8 100644
--- a/src/public/app/widgets/search_definition.js
+++ b/src/public/app/widgets/search_definition.js
@@ -164,6 +164,9 @@ const TPL = `
                             <option value="title">Title</option>
                             <option value="dateCreated">Date created</option>
                             <option value="dateModified">Date of last modification</option>
+                            <option value="contentSize">Note content size</option>
+                            <option value="noteSize">Note content size including revisions</option>
+                            <option value="revisionCount">Number of revisions</option>
                         </select>
                         
                         <select name="orderDirection" class="form-control w-auto d-inline">
diff --git a/src/routes/api/search.js b/src/routes/api/search.js
index f1580e60f..e4e62af6a 100644
--- a/src/routes/api/search.js
+++ b/src/routes/api/search.js
@@ -15,7 +15,7 @@ async function search(note) {
 
         if (searchScript) {
             searchResultNoteIds = await searchFromRelation(note, 'searchScript');
-        } else if (searchString) {
+        } else {
             const searchContext = new SearchContext({
                 fastSearch: note.hasLabel('fastSearch'),
                 ancestorNoteId: note.getRelationValue('ancestor'),
@@ -27,8 +27,6 @@ async function search(note) {
 
             searchResultNoteIds = searchService.findNotesWithQuery(searchString, searchContext)
                 .map(sr => sr.noteId);
-        } else {
-            searchResultNoteIds = [];
         }
 
         // we won't return search note's own noteId
diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js
index 29e06dafe..74b7b00ca 100644
--- a/src/services/note_cache/entities/note.js
+++ b/src/services/note_cache/entities/note.js
@@ -30,6 +30,15 @@ class Note {
 
         /** @param {Note[]|null} */
         this.ancestorCache = null;
+
+        // following attributes are filled during searching from database
+
+        /** @param {int} size of the content in bytes */
+        this.contentSize = null;
+        /** @param {int} size of the content and note revision contents in bytes */
+        this.noteSize = null;
+        /** @param {int} number of note revisions for this note */
+        this.revisionCount = null;
     }
 
     update(row) {
diff --git a/src/services/search/expressions/property_comparison.js b/src/services/search/expressions/property_comparison.js
index 809247724..722480626 100644
--- a/src/services/search/expressions/property_comparison.js
+++ b/src/services/search/expressions/property_comparison.js
@@ -22,7 +22,10 @@ const PROP_MAPPING = {
     "childrencount": "childrenCount",
     "attributecount": "attributeCount",
     "labelcount": "labelCount",
-    "relationcount": "relationCount"
+    "relationcount": "relationCount",
+    "contentsize": "contentSize",
+    "notesize": "noteSize",
+    "revisioncount": "revisionCount"
 };
 
 class PropertyComparisonExp extends Expression {
@@ -30,11 +33,15 @@ class PropertyComparisonExp extends Expression {
         return name in PROP_MAPPING;
     }
 
-    constructor(propertyName, comparator) {
+    constructor(searchContext, propertyName, comparator) {
         super();
 
         this.propertyName = PROP_MAPPING[propertyName];
         this.comparator = comparator;
+
+        if (['contentsize', 'notesize', 'revisioncount'].includes(this.propertyName)) {
+            searchContext.dbLoadNeeded = true;
+        }
     }
 
     execute(inputNoteSet, executionContext) {
diff --git a/src/services/search/search_context.js b/src/services/search/search_context.js
index afc6d33ad..d64c4edb3 100644
--- a/src/services/search/search_context.js
+++ b/src/services/search/search_context.js
@@ -10,6 +10,9 @@ class SearchContext {
         this.fuzzyAttributeSearch = !!params.fuzzyAttributeSearch;
         this.highlightedTokens = [];
         this.originalQuery = "";
+        // if true, note cache does not have (up-to-date) information needed to process the query
+        // and some extra data needs to be loaded before executing
+        this.dbLoadNeeded = false;
         this.error = null;
     }
 
diff --git a/src/services/search/services/parse.js b/src/services/search/services/parse.js
index 11bdd8935..a8aa2a32f 100644
--- a/src/services/search/services/parse.js
+++ b/src/services/search/services/parse.js
@@ -192,7 +192,7 @@ function getExpression(tokens, searchContext, level = 0) {
             i += 2;
 
             return new OrExp([
-                new PropertyComparisonExp('title', buildComparator('*=*', tokens[i].token)),
+                new PropertyComparisonExp(searchContext, 'title', buildComparator('*=*', tokens[i].token)),
                 new NoteContentProtectedFulltextExp('*=*', [tokens[i].token]),
                 new NoteContentUnprotectedFulltextExp('*=*', [tokens[i].token])
             ]);
@@ -213,7 +213,7 @@ function getExpression(tokens, searchContext, level = 0) {
                 return;
             }
 
-            return new PropertyComparisonExp(propertyName, comparator);
+            return new PropertyComparisonExp(searchContext, propertyName, comparator);
         }
 
         searchContext.addError(`Unrecognized note property "${tokens[i].token}" in ${context(i)}`);
@@ -307,7 +307,7 @@ function getExpression(tokens, searchContext, level = 0) {
                     i++;
                 }
 
-                const valueExtractor = new ValueExtractor(propertyPath);
+                const valueExtractor = new ValueExtractor(searchContext, propertyPath);
 
                 if (valueExtractor.validate()) {
                     searchContext.addError(valueExtractor.validate());
@@ -409,7 +409,7 @@ function getExpression(tokens, searchContext, level = 0) {
 
 function parse({fulltextTokens, expressionTokens, searchContext}) {
     let exp = AndExp.of([
-        searchContext.includeArchivedNotes ? null : new PropertyComparisonExp("isarchived", buildComparator("=", "false")),
+        searchContext.includeArchivedNotes ? null : new PropertyComparisonExp(searchContext, "isarchived", buildComparator("=", "false")),
         searchContext.ancestorNoteId ? new AncestorExp(searchContext.ancestorNoteId) : null,
         getFulltext(fulltextTokens, searchContext),
         getExpression(expressionTokens, searchContext)
@@ -419,7 +419,7 @@ function parse({fulltextTokens, expressionTokens, searchContext}) {
         const filterExp = exp;
 
         exp = new OrderByAndLimitExp([{
-            valueExtractor: new ValueExtractor(['note', searchContext.orderBy]),
+            valueExtractor: new ValueExtractor(searchContext, ['note', searchContext.orderBy]),
             direction: searchContext.orderDirection
         }], 0);
 
diff --git a/src/services/search/services/search.js b/src/services/search/services/search.js
index 80db63ea0..b955d0365 100644
--- a/src/services/search/services/search.js
+++ b/src/services/search/services/search.js
@@ -10,6 +10,54 @@ const noteCache = require('../../note_cache/note_cache.js');
 const noteCacheService = require('../../note_cache/note_cache_service.js');
 const utils = require('../../utils.js');
 const cls = require('../../cls.js');
+const log = require('../../log.js');
+
+function loadNeededInfoFromDatabase() {
+    const sql = require('../../sql.js');
+
+    for (const noteId in noteCache.notes) {
+        noteCache.notes[noteId].contentSize = 0;
+        noteCache.notes[noteId].noteSize = 0;
+        noteCache.notes[noteId].revisionCount = 0;
+    }
+
+    const noteContentLengths = sql.getRows(`
+        SELECT 
+            noteId, 
+            LENGTH(content) AS length 
+        FROM notes
+             JOIN note_contents USING(noteId) 
+        WHERE notes.isDeleted = 0`);
+
+    for (const {noteId, length} of noteContentLengths) {
+        if (!(noteId in noteCache.notes)) {
+            log.error(`Note ${noteId} not found in note cache.`);
+            continue;
+        }
+
+        noteCache.notes[noteId].contentSize = length;
+        noteCache.notes[noteId].noteSize = length;
+    }
+
+    const noteRevisionContentLengths = sql.getRows(`
+        SELECT 
+            noteId, 
+            LENGTH(content) AS length 
+        FROM notes
+             JOIN note_revisions USING(noteId) 
+             JOIN note_revision_contents USING(noteRevisionId) 
+        WHERE notes.isDeleted = 0`);
+
+    for (const {noteId, length} of noteRevisionContentLengths) {
+        if (!(noteId in noteCache.notes)) {
+            log.error(`Note ${noteId} not found in note cache.`);
+            continue;
+        }
+
+        noteCache.notes[noteId].noteSize += length;
+        noteCache.notes[noteId].revisionCount++;
+    }
+}
 
 /**
  * @param {Expression} expression
@@ -22,6 +70,10 @@ function findNotesWithExpression(expression, searchContext) {
         ? hoistedNote.subtreeNotes
         : Object.values(noteCache.notes);
 
+    if (searchContext.dbLoadNeeded) {
+        loadNeededInfoFromDatabase();
+    }
+
     // in the process of loading data sometimes we create "skeleton" note instances which are expected to be filled later
     // in case of inconsistent data this might not work and search will then crash on these
     allNotes = allNotes.filter(note => note.type !== undefined);
@@ -84,10 +136,7 @@ function parseQueryToExpression(query, searchContext) {
  * @return {SearchResult[]}
  */
 function findNotesWithQuery(query, searchContext) {
-    if (!query.trim().length) {
-        return [];
-    }
-
+    query = query || "";
     searchContext.originalQuery = query;
 
     const expression = parseQueryToExpression(query, searchContext);
diff --git a/src/services/search/value_extractor.js b/src/services/search/value_extractor.js
index 7c4c168f8..62c77e294 100644
--- a/src/services/search/value_extractor.js
+++ b/src/services/search/value_extractor.js
@@ -19,18 +19,25 @@ const PROP_MAPPING = {
     "childrencount": "childrenCount",
     "attributecount": "attributeCount",
     "labelcount": "labelCount",
-    "relationcount": "relationCount"
+    "relationcount": "relationCount",
+    "contentsize": "contentSize",
+    "notesize": "noteSize",
+    "revisioncount": "revisionCount"
 };
 
 class ValueExtractor {
-    constructor(propertyPath) {
+    constructor(searchContext, propertyPath) {
         this.propertyPath = propertyPath.map(pathEl => pathEl.toLowerCase());
 
         if (this.propertyPath[0].startsWith('#')) {
             this.propertyPath = ['note', 'labels', this.propertyPath[0].substr(1), ...this.propertyPath.slice( 1, this.propertyPath.length)];
         }
         else if (this.propertyPath[0].startsWith('~')) {
-            this.propertyPath = ['note', 'relations', this.propertyPath[0].substr(1), ...this.propertyPath.slice( 1, this.propertyPath.length)];
+            this.propertyPath = ['note', 'relations', this.propertyPath[0].substr(1), ...this.propertyPath.slice(1, this.propertyPath.length)];
+        }
+
+        if (['contentsize', 'notesize', 'revisioncount'].includes(this.propertyPath[this.propertyPath.length - 1])) {
+            searchContext.dbLoadNeeded = true;
         }
     }