From 2326eb85f10d36d0b74061e9b308443d43b3aed4 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 16 Mar 2019 16:52:58 +0100 Subject: [PATCH] filter reimplementation, WIP --- src/routes/api/search.js | 49 ++-------------------- src/services/build_search_query.js | 54 +++++++++++++++++------- src/services/parse_filters.js | 67 +++++++++++++++++++----------- 3 files changed, 86 insertions(+), 84 deletions(-) diff --git a/src/routes/api/search.js b/src/routes/api/search.js index 40d5e959f..e86eb1505 100644 --- a/src/routes/api/search.js +++ b/src/routes/api/search.js @@ -8,54 +8,13 @@ const parseFilters = require('../../services/parse_filters'); const buildSearchQuery = require('../../services/build_search_query'); async function searchNotes(req) { - const {labelFilters, searchText} = parseFilters(req.params.searchString); + const filters = parseFilters(req.params.searchString); - let labelFiltersNoteIds = null; + const {query, params} = buildSearchQuery(filters); - if (labelFilters.length > 0) { - const {query, params} = buildSearchQuery(labelFilters, searchText); + const labelFiltersNoteIds = await sql.getColumn(query, params); - labelFiltersNoteIds = await sql.getColumn(query, params); - } - - let searchTextResults = null; - - if (searchText.trim().length > 0) { - searchTextResults = await noteCacheService.findNotes(searchText); - - let fullTextNoteIds = await getFullTextResults(searchText); - - for (const noteId of fullTextNoteIds) { - if (!searchTextResults.some(item => item.noteId === noteId)) { - const result = noteCacheService.getNotePath(noteId); - - if (result) { - searchTextResults.push(result); - } - } - } - } - - let results; - - if (labelFiltersNoteIds && searchTextResults) { - results = searchTextResults.filter(item => labelFiltersNoteIds.includes(item.noteId)); - } - else if (labelFiltersNoteIds) { - results = labelFiltersNoteIds.map(noteCacheService.getNotePath).filter(res => !!res); - } - else { - results = searchTextResults; - } - - return results; -} - -async function getFullTextResults(searchText) { - const safeSearchText = utils.sanitizeSql(searchText); - - return await sql.getColumn(`SELECT noteId FROM note_fulltext - WHERE note_fulltext MATCH '${safeSearchText}'`); + return labelFiltersNoteIds.map(noteCacheService.getNotePath).filter(res => !!res); } async function saveSearchToNote(req) { diff --git a/src/services/build_search_query.js b/src/services/build_search_query.js index 10585ce49..9de569f49 100644 --- a/src/services/build_search_query.js +++ b/src/services/build_search_query.js @@ -1,16 +1,7 @@ -function isVirtualAttribute(filter) { - return ( - filter.name === "utcDateModified" - || filter.name === "utcDateCreated" - || filter.name === "isProtected" - ); -} +const VIRTUAL_ATTRIBUTES = ["dateCreated", "dateCreated", "dateModified", "utcDateCreated", "utcDateModified", "isProtected", "title", "content", "type", "mime", "text"]; function getValueForFilter(filter, i) { - return (isVirtualAttribute(filter) - ? `substr(notes.${filter.name}, 0, ${filter.value.length + 1})` - :`attribute${i}.value` - ); + return VIRTUAL_ATTRIBUTES.includes(filter.name) ? `notes.${filter.name}` :`attribute${i}.value`; } module.exports = function(attributeFilters) { @@ -22,7 +13,7 @@ module.exports = function(attributeFilters) { let i = 1; for (const filter of attributeFilters) { - const virtual = isVirtualAttribute(filter); + const virtual = VIRTUAL_ATTRIBUTES.includes(filter.name); if (!virtual) { joins.push(`LEFT JOIN attributes AS attribute${i} ` @@ -31,6 +22,16 @@ module.exports = function(attributeFilters) { ); joinParams.push(filter.name); } + else if (filter.name === 'content') { + // FIXME: this will fail if there's more instances of content + joins.push(`JOIN note_contents ON note_contents.noteId = notes.noteId`); + + filter.name = 'note_contents.content'; + } + else if (filter.name === 'text') { + // FIXME: this will fail if there's more instances of content + joins.push(`JOIN note_fulltext ON note_fulltext.noteId = notes.noteId`); + } where += " " + filter.relation + " "; @@ -44,8 +45,30 @@ module.exports = function(attributeFilters) { where += `${test} IS NULL`; } else if (filter.operator === '=' || filter.operator === '!=') { - where += `${getValueForFilter(filter, i)} ${filter.operator} ?`; - whereParams.push(filter.value); + if (filter.name === 'text') { + const safeSearchText = utils.sanitizeSql(filter.value); + + where += (filter.operator === '!=' ? 'NOT ' : '') + `MATCH '${safeSearchText}'`; + } + else { + where += `${getValueForFilter(filter, i)} ${filter.operator} ?`; + whereParams.push(filter.value); + } + } + else if (filter.operator === '*=' || filter.operator === '!*=') { + where += `${getValueForFilter(filter, i)}` + + (filter.operator.includes('!') ? ' NOT' : '') + + ` LIKE '%` + filter.value + "'"; // FIXME: escaping + } + else if (filter.operator === '=*' || filter.operator === '!=*') { + where += `${getValueForFilter(filter, i)}` + + (filter.operator.includes('!') ? ' NOT' : '') + + ` LIKE '` + filter.value + "%'"; // FIXME: escaping + } + else if (filter.operator === '*=*' || filter.operator === '!*=*') { + where += `${getValueForFilter(filter, i)}` + + (filter.operator.includes('!') ? ' NOT' : '') + + ` LIKE '%` + filter.value + "%'"; // FIXME: escaping } else if ([">", ">=", "<", "<="].includes(filter.operator)) { let floatParam; @@ -85,5 +108,8 @@ module.exports = function(attributeFilters) { const params = joinParams.concat(whereParams).concat(searchParams); + console.log(query); + console.log(params); + return { query, params }; }; diff --git a/src/services/parse_filters.js b/src/services/parse_filters.js index 612bf3afe..8596b2cfd 100644 --- a/src/services/parse_filters.js +++ b/src/services/parse_filters.js @@ -1,41 +1,61 @@ const dayjs = require("dayjs"); -const labelRegex = /(\b(and|or)\s+)?@(!?)([\w_-]+|"[^"]+")((=|!=|<|<=|>|>=)([\w_-]+|"[^"]+"))?/i; -const smartValueRegex = /^(TODAY|NOW)(([+\-])(\d+)([HDMY])){0,1}$/i; +const filterRegex = /(\b(AND|OR)\s+)?@(!?)([\w_-]+|"[^"]+")((=|!=|<|<=|>|>=|\*=|=\*)([\w_-]+|"[^"]+"))?/ig; +const smartValueRegex = /^(NOW|TODAY|WEEK|MONTH|YEAR) *([+\-] *\d+)?$/i; function calculateSmartValue(v) { - const normalizedV = v.toUpperCase() + "+0D"; // defaults of sorts - const [ , keyword, sign, snum, unit] = /(TODAY|NOW)([+\-])(\d+)([HDMY])/.exec(normalizedV); - const num = parseInt(snum); - - if (keyword !== "TODAY" && keyword !== "NOW") { - return v; + const match = smartValueRegex.exec(v); + if (match === null) { + return; } - const fullUnit = { - TODAY: { D: "days", M: "months", Y: "years" }, - NOW: { D: "days", M: "minutes", H: "hours" } - }[keyword][unit]; + const keyword = match[1].toUpperCase(); + const num = match[2] ? parseInt(match[2].replace(" ", "")) : 0; // can contain spaces between sign and digits - const format = keyword === "TODAY" ? "YYYY-MM-DD" : "YYYY-MM-DDTHH:mm"; - const date = (sign === "+" ? dayjs().add(num, fullUnit) : dayjs().subtract(num, fullUnit)); + let format, date; + + if (keyword === 'NOW') { + date = dayjs().add(num, 'second'); + format = "YYYY-MM-DD HH:mm:ss"; + } + else if (keyword === 'TODAY') { + date = dayjs().add(num, 'day'); + format = "YYYY-MM-DD"; + } + else if (keyword === 'WEEK') { + // FIXME + //date = dayjs().add(num, 'day'); + format = "YYYY-MM-DD"; + } + else if (keyword === 'MONTH') { + date = dayjs().add(num, 'month'); + format = "YYYY-MM"; + } + else if (keyword === 'YEAR') { + date = dayjs().add(num, 'year'); + format = "YYYY"; + } + else { + throw new Error("Unrecognized keyword: " + keyword); + } return date.format(format); } -module.exports = function(searchText) { - const labelFilters = []; - let match = labelRegex.exec(searchText); +module.exports = function (searchText) { + const filters = []; function trimQuotes(str) { return str.startsWith('"') ? str.substr(1, str.length - 2) : str; } - while (match != null) { + let match; + + while (match = filterRegex.exec(searchText)) { const relation = match[2] !== undefined ? match[2].toLowerCase() : 'and'; const operator = match[3] === '!' ? 'not-exists' : 'exists'; const value = match[7] !== undefined ? trimQuotes(match[7]) : null; - labelFilters.push({ + filters.push({ relation: relation, name: trimQuotes(match[4]), operator: match[6] !== undefined ? match[6] : operator, @@ -45,12 +65,9 @@ module.exports = function(searchText) { : value ) }); - - // remove labels from further fulltext search - searchText = searchText.split(match[0]).join(''); - - match = labelRegex.exec(searchText); } - return { labelFilters: labelFilters, searchText }; + console.log(filters); + + return filters; };