filter reimplementation, WIP

src/routes/api/search.js

@@ -8,54 +8,13 @@ const parseFilters = require('../../services/parse_filters');
 const buildSearchQuery = require('../../services/build_search_query');
 
 async function searchNotes(req) {
-    const {labelFilters, searchText} = parseFilters(req.params.searchString);
+    const filters = parseFilters(req.params.searchString);
 
-    let labelFiltersNoteIds = null;
+    const {query, params} = buildSearchQuery(filters);
 
-    if (labelFilters.length > 0) {
+    const labelFiltersNoteIds = await sql.getColumn(query, params);
-        const {query, params} = buildSearchQuery(labelFilters, searchText);
 
-        labelFiltersNoteIds = await sql.getColumn(query, params);
+    return labelFiltersNoteIds.map(noteCacheService.getNotePath).filter(res => !!res);
-    }
-
-    let searchTextResults = null;
-
-    if (searchText.trim().length > 0) {
-        searchTextResults = await noteCacheService.findNotes(searchText);
-
-        let fullTextNoteIds = await getFullTextResults(searchText);
-
-        for (const noteId of fullTextNoteIds) {
-            if (!searchTextResults.some(item => item.noteId === noteId)) {
-                const result = noteCacheService.getNotePath(noteId);
-
-                if (result) {
-                    searchTextResults.push(result);
-                }
-            }
-        }
-    }
-
-    let results;
-
-    if (labelFiltersNoteIds && searchTextResults) {
-        results = searchTextResults.filter(item => labelFiltersNoteIds.includes(item.noteId));
-    }
-    else if (labelFiltersNoteIds) {
-        results = labelFiltersNoteIds.map(noteCacheService.getNotePath).filter(res => !!res);
-    }
-    else {
-        results = searchTextResults;
-    }
-
-    return results;
-}
-
-async function getFullTextResults(searchText) {
-    const safeSearchText = utils.sanitizeSql(searchText);
-
-    return await sql.getColumn(`SELECT noteId FROM note_fulltext 
-                                       WHERE note_fulltext MATCH '${safeSearchText}'`);
 }
 
 async function saveSearchToNote(req) {

src/services/build_search_query.js

@@ -1,16 +1,7 @@
-function isVirtualAttribute(filter) {
+const VIRTUAL_ATTRIBUTES = ["dateCreated", "dateCreated", "dateModified", "utcDateCreated", "utcDateModified", "isProtected", "title", "content", "type", "mime", "text"];
-    return (
-        filter.name === "utcDateModified"
-        || filter.name === "utcDateCreated"
-        || filter.name === "isProtected"
-    );
-}
 
 function getValueForFilter(filter, i) {
-    return (isVirtualAttribute(filter)
+    return VIRTUAL_ATTRIBUTES.includes(filter.name) ? `notes.${filter.name}` :`attribute${i}.value`;
-        ? `substr(notes.${filter.name}, 0, ${filter.value.length + 1})`
-        :`attribute${i}.value`
-    );
 }
 
 module.exports = function(attributeFilters) {
@@ -22,7 +13,7 @@ module.exports = function(attributeFilters) {
     let i = 1;
 
     for (const filter of attributeFilters) {
-        const virtual = isVirtualAttribute(filter);
+        const virtual = VIRTUAL_ATTRIBUTES.includes(filter.name);
 
         if (!virtual) {
             joins.push(`LEFT JOIN attributes AS attribute${i} `
@@ -31,6 +22,16 @@ module.exports = function(attributeFilters) {
             );
             joinParams.push(filter.name);
         }
+        else if (filter.name === 'content') {
+            // FIXME: this will fail if there's more instances of content
+            joins.push(`JOIN note_contents ON note_contents.noteId = notes.noteId`);
+
+            filter.name = 'note_contents.content';
+        }
+        else if (filter.name === 'text') {
+            // FIXME: this will fail if there's more instances of content
+            joins.push(`JOIN note_fulltext ON note_fulltext.noteId = notes.noteId`);
+        }
 
         where += " " + filter.relation + " ";
 
@@ -44,9 +45,31 @@ module.exports = function(attributeFilters) {
             where += `${test} IS NULL`;
         }
         else if (filter.operator === '=' || filter.operator === '!=') {
+            if (filter.name === 'text') {
+                const safeSearchText = utils.sanitizeSql(filter.value);
+
+                where += (filter.operator === '!=' ? 'NOT ' : '') + `MATCH '${safeSearchText}'`;
+            }
+            else {
                 where += `${getValueForFilter(filter, i)} ${filter.operator} ?`;
                 whereParams.push(filter.value);
             }
+        }
+        else if (filter.operator === '*=' || filter.operator === '!*=') {
+            where += `${getValueForFilter(filter, i)}`
+                    + (filter.operator.includes('!') ? ' NOT' : '')
+                    + ` LIKE '%` + filter.value + "'"; // FIXME: escaping
+        }
+        else if (filter.operator === '=*' || filter.operator === '!=*') {
+            where += `${getValueForFilter(filter, i)}`
+                    + (filter.operator.includes('!') ? ' NOT' : '')
+                    + ` LIKE '` + filter.value + "%'"; // FIXME: escaping
+        }
+        else if (filter.operator === '*=*' || filter.operator === '!*=*') {
+            where += `${getValueForFilter(filter, i)}`
+                    + (filter.operator.includes('!') ? ' NOT' : '')
+                    + ` LIKE '%` + filter.value + "%'"; // FIXME: escaping
+        }
         else if ([">", ">=", "<", "<="].includes(filter.operator)) {
             let floatParam;
             const value = getValueForFilter(filter, i);
@@ -85,5 +108,8 @@ module.exports = function(attributeFilters) {
 
     const params = joinParams.concat(whereParams).concat(searchParams);
 
+    console.log(query);
+    console.log(params);
+
     return { query, params };
 };

src/services/parse_filters.js

@@ -1,41 +1,61 @@
 const dayjs = require("dayjs");
 
-const labelRegex = /(\b(and|or)\s+)?@(!?)([\w_-]+|"[^"]+")((=|!=|<|<=|>|>=)([\w_-]+|"[^"]+"))?/i;
+const filterRegex = /(\b(AND|OR)\s+)?@(!?)([\w_-]+|"[^"]+")((=|!=|<|<=|>|>=|\*=|=\*)([\w_-]+|"[^"]+"))?/ig;
-const smartValueRegex = /^(TODAY|NOW)(([+\-])(\d+)([HDMY])){0,1}$/i;
+const smartValueRegex = /^(NOW|TODAY|WEEK|MONTH|YEAR) *([+\-] *\d+)?$/i;
 
 function calculateSmartValue(v) {
-    const normalizedV = v.toUpperCase() + "+0D"; // defaults of sorts
+    const match = smartValueRegex.exec(v);
-    const [ , keyword, sign, snum, unit] = /(TODAY|NOW)([+\-])(\d+)([HDMY])/.exec(normalizedV);
+    if (match === null) {
-    const num = parseInt(snum);
+        return;
-
-    if (keyword !== "TODAY" && keyword !== "NOW") {
-        return v;
     }
 
-    const fullUnit = {
+    const keyword = match[1].toUpperCase();
-        TODAY: { D: "days", M: "months", Y: "years" },
+    const num = match[2] ? parseInt(match[2].replace(" ", "")) : 0; // can contain spaces between sign and digits
-        NOW: { D: "days", M: "minutes", H: "hours" }
-    }[keyword][unit];
 
-    const format = keyword === "TODAY" ? "YYYY-MM-DD" : "YYYY-MM-DDTHH:mm";
+    let format, date;
-    const date = (sign === "+" ? dayjs().add(num, fullUnit) : dayjs().subtract(num, fullUnit));
+
+    if (keyword === 'NOW') {
+        date = dayjs().add(num, 'second');
+        format = "YYYY-MM-DD HH:mm:ss";
+    }
+    else if (keyword === 'TODAY') {
+        date = dayjs().add(num, 'day');
+        format = "YYYY-MM-DD";
+    }
+    else if (keyword === 'WEEK') {
+        // FIXME
+        //date = dayjs().add(num, 'day');
+        format = "YYYY-MM-DD";
+    }
+    else if (keyword === 'MONTH') {
+        date = dayjs().add(num, 'month');
+        format = "YYYY-MM";
+    }
+    else if (keyword === 'YEAR') {
+        date = dayjs().add(num, 'year');
+        format = "YYYY";
+    }
+    else {
+        throw new Error("Unrecognized keyword: " + keyword);
+    }
 
     return date.format(format);
 }
 
 module.exports = function (searchText) {
-    const labelFilters = [];
+    const filters = [];
-    let match = labelRegex.exec(searchText);
 
     function trimQuotes(str) { return str.startsWith('"') ? str.substr(1, str.length - 2) : str; }
 
-    while (match != null) {
+    let match;
+
+    while (match = filterRegex.exec(searchText)) {
         const relation = match[2] !== undefined ? match[2].toLowerCase() : 'and';
         const operator = match[3] === '!' ? 'not-exists' : 'exists';
 
         const value = match[7] !== undefined ? trimQuotes(match[7]) : null;
 
-        labelFilters.push({
+        filters.push({
             relation: relation,
             name: trimQuotes(match[4]),
             operator: match[6] !== undefined ? match[6] : operator,
@@ -45,12 +65,9 @@ module.exports = function(searchText) {
                     : value
             )
         });
-
-        // remove labels from further fulltext search
-        searchText = searchText.split(match[0]).join('');
-
-        match = labelRegex.exec(searchText);
     }
 
-    return { labelFilters: labelFilters, searchText };
+    console.log(filters);
+
+    return filters;
 };