From b26100479d15ba2a93019440de10d37bca305a26 Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 20 May 2020 23:20:39 +0200 Subject: [PATCH] parser tests added --- spec/parser.spec.js | 99 ++++++++++++++++++- src/services/note_cache/entities/attribute.js | 4 +- src/services/search/comparator_builder.js | 66 +++++++++++++ src/services/search/expressions/and.js | 13 +-- .../search/expressions/field_comparison.js | 7 +- src/services/search/expressions/or.js | 11 +++ src/services/search/parser.js | 52 ++++++---- 7 files changed, 220 insertions(+), 32 deletions(-) create mode 100644 src/services/search/comparator_builder.js diff --git a/spec/parser.spec.js b/spec/parser.spec.js index 7d9ad2a55..1fd939031 100644 --- a/spec/parser.spec.js +++ b/spec/parser.spec.js @@ -2,9 +2,102 @@ const parser = require('../src/services/search/parser'); describe("Parser", () => { it("fulltext parser without content", () => { - const exps = parser(["hello", "hi"], [], false); + const rootExp = parser(["hello", "hi"], [], false); - expect(exps.constructor.name).toEqual("NoteCacheFulltextExp"); - expect(exps.tokens).toEqual(["hello", "hi"]); + expect(rootExp.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(rootExp.tokens).toEqual(["hello", "hi"]); + }); + + it("fulltext parser with content", () => { + const rootExp = parser(["hello", "hi"], [], true); + + expect(rootExp.constructor.name).toEqual("OrExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(firstSub.tokens).toEqual(["hello", "hi"]); + + expect(secondSub.constructor.name).toEqual("NoteContentFulltextExp"); + expect(secondSub.tokens).toEqual(["hello", "hi"]); + }); + + it("simple label comparison", () => { + const rootExp = parser([], ["#mylabel", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("FieldComparisonExp"); + expect(rootExp.attributeType).toEqual("label"); + expect(rootExp.attributeName).toEqual("mylabel"); + expect(rootExp.comparator).toBeTruthy(); + }); + + it("simple label AND", () => { + const rootExp = parser([], ["#first", "=", "text", "AND", "#second", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("AndExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("second"); + }); + + it("simple label AND without explicit AND", () => { + const rootExp = parser([], ["#first", "=", "text", "#second", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("AndExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("second"); + }); + + it("simple label OR", () => { + const rootExp = parser([], ["#first", "=", "text", "OR", "#second", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("OrExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("second"); + }); + + it("fulltext and simple label", () => { + const rootExp = parser(["hello"], ["#mylabel", "=", "text"], false); + + expect(rootExp.constructor.name).toEqual("AndExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(firstSub.tokens).toEqual(["hello"]); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("mylabel"); + }); + + it("label sub-expression", () => { + const rootExp = parser([], ["#first", "=", "text", "OR", ["#second", "=", "text", "AND", "#third", "=", "text"]], false); + + expect(rootExp.constructor.name).toEqual("OrExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("AndExp"); + const [firstSubSub, secondSubSub] = secondSub.subExpressions; + + expect(firstSubSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSubSub.attributeName).toEqual("second"); + + expect(secondSubSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSubSub.attributeName).toEqual("third"); }); }); diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index 116c4b719..e66b9c771 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -11,9 +11,9 @@ class Attribute { /** @param {string} */ this.type = row.type; /** @param {string} */ - this.name = row.name; + this.name = row.name.toLowerCase(); /** @param {string} */ - this.value = row.value; + this.value = row.value.toLowerCase(); /** @param {boolean} */ this.isInheritable = !!row.isInheritable; diff --git a/src/services/search/comparator_builder.js b/src/services/search/comparator_builder.js new file mode 100644 index 000000000..72c1103a8 --- /dev/null +++ b/src/services/search/comparator_builder.js @@ -0,0 +1,66 @@ +const dayjs = require("dayjs"); + +const comparators = { + "=": comparedValue => (val => val === comparedValue), + "!=": comparedValue => (val => val !== comparedValue), + ">": comparedValue => (val => val > comparedValue), + ">=": comparedValue => (val => val >= comparedValue), + "<": comparedValue => (val => val < comparedValue), + "<=": comparedValue => (val => val <= comparedValue), + "*=": comparedValue => (val => val.endsWith(comparedValue)), + "=*": comparedValue => (val => val.startsWith(comparedValue)), + "*=*": comparedValue => (val => val.includes(comparedValue)), +} + +const smartValueRegex = /^(NOW|TODAY|WEEK|MONTH|YEAR) *([+\-] *\d+)?$/i; + +function calculateSmartValue(v) { + const match = smartValueRegex.exec(v); + if (match === null) { + return v; + } + + const keyword = match[1].toUpperCase(); + const num = match[2] ? parseInt(match[2].replace(/ /g, "")) : 0; // can contain spaces between sign and digits + + let format, date; + + if (keyword === 'NOW') { + date = dayjs().add(num, 'second'); + format = "YYYY-MM-DD HH:mm:ss"; + } + else if (keyword === 'TODAY') { + date = dayjs().add(num, 'day'); + format = "YYYY-MM-DD"; + } + else if (keyword === 'WEEK') { + // FIXME: this will always use sunday as start of the week + date = dayjs().startOf('week').add(7 * num, 'day'); + format = "YYYY-MM-DD"; + } + else if (keyword === 'MONTH') { + date = dayjs().add(num, 'month'); + format = "YYYY-MM"; + } + else if (keyword === 'YEAR') { + date = dayjs().add(num, 'year'); + format = "YYYY"; + } + else { + throw new Error("Unrecognized keyword: " + keyword); + } + + return date.format(format); +} + +function buildComparator(operator, comparedValue) { + comparedValue = comparedValue.toLowerCase(); + + comparedValue = calculateSmartValue(comparedValue); + + if (operator in comparators) { + return comparators[operator](comparedValue); + } +} + +module.exports = buildComparator; diff --git a/src/services/search/expressions/and.js b/src/services/search/expressions/and.js index 44e19c0af..cbdd4e108 100644 --- a/src/services/search/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -1,19 +1,20 @@ "use strict"; class AndExp { - constructor(subExpressions) { - this.subExpressions = subExpressions; - } - static of(subExpressions) { + subExpressions = subExpressions.filter(exp => !!exp); + if (subExpressions.length === 1) { return subExpressions[0]; - } - else { + } else if (subExpressions.length > 0) { return new AndExp(subExpressions); } } + constructor(subExpressions) { + this.subExpressions = subExpressions; + } + execute(noteSet, searchContext) { for (const subExpression of this.subExpressions) { noteSet = subExpression.execute(noteSet, searchContext); diff --git a/src/services/search/expressions/field_comparison.js b/src/services/search/expressions/field_comparison.js index 8c95f2169..4630087ce 100644 --- a/src/services/search/expressions/field_comparison.js +++ b/src/services/search/expressions/field_comparison.js @@ -4,11 +4,10 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); class FieldComparisonExp { - constructor(attributeType, attributeName, operator, attributeValue) { + constructor(attributeType, attributeName, comparator) { this.attributeType = attributeType; this.attributeName = attributeName; - this.operator = operator; - this.attributeValue = attributeValue; + this.comparator = comparator; } execute(noteSet) { @@ -18,7 +17,7 @@ class FieldComparisonExp { for (const attr of attrs) { const note = attr.note; - if (noteSet.hasNoteId(note.noteId) && attr.value === this.attributeValue) { + if (noteSet.hasNoteId(note.noteId) && this.comparator(attr.value)) { if (attr.isInheritable) { resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); } diff --git a/src/services/search/expressions/or.js b/src/services/search/expressions/or.js index a48bb8bc8..51406bcfc 100644 --- a/src/services/search/expressions/or.js +++ b/src/services/search/expressions/or.js @@ -3,6 +3,17 @@ const NoteSet = require('../note_set'); class OrExp { + static of(subExpressions) { + subExpressions = subExpressions.filter(exp => !!exp); + + if (subExpressions.length === 1) { + return subExpressions[0]; + } + else if (subExpressions.length > 0) { + return new OrExp(subExpressions); + } + } + constructor(subExpressions) { this.subExpressions = subExpressions; } diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 1c692eaaf..bf82e4b21 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -5,28 +5,32 @@ const AttributeExistsExp = require('./expressions/attribute_exists'); const FieldComparisonExp = require('./expressions/field_comparison'); const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); +const comparatorBuilder = require('./comparator_builder'); function getFulltext(tokens, includingNoteContent) { - if (includingNoteContent) { - return [ - new OrExp([ - new NoteCacheFulltextExp(tokens), - new NoteContentFulltextExp(tokens) - ]) - ] + if (tokens.length === 0) { + return null; + } + else if (includingNoteContent) { + return new OrExp([ + new NoteCacheFulltextExp(tokens), + new NoteContentFulltextExp(tokens) + ]); } else { - return [ - new NoteCacheFulltextExp(tokens) - ] + return new NoteCacheFulltextExp(tokens); } } function isOperator(str) { - return str.matches(/^[=<>*]+$/); + return str.match(/^[=<>*]+$/); } -function getExpressions(tokens) { +function getExpression(tokens) { + if (tokens.length === 0) { + return null; + } + const expressions = []; let op = null; @@ -38,13 +42,22 @@ function getExpressions(tokens) { } if (Array.isArray(token)) { - expressions.push(getExpressions(token)); + expressions.push(getExpression(token)); } else if (token.startsWith('#') || token.startsWith('@')) { const type = token.startsWith('#') ? 'label' : 'relation'; if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { - expressions.push(new FieldComparisonExp(type, token.substr(1), tokens[i + 1], tokens[i + 2])); + const operator = tokens[i + 1]; + const comparedValue = tokens[i + 2]; + + const comparator = comparatorBuilder(operator, comparedValue); + + if (!comparator) { + throw new Error(`Can't find operator '${operator}'`); + } + + expressions.push(new FieldComparisonExp(type, token.substr(1), comparator)); i += 2; } @@ -72,13 +85,18 @@ function getExpressions(tokens) { } } - return expressions; + if (op === null || op === 'and') { + return AndExp.of(expressions); + } + else if (op === 'or') { + return OrExp.of(expressions); + } } function parse(fulltextTokens, expressionTokens, includingNoteContent) { return AndExp.of([ - ...getFulltext(fulltextTokens, includingNoteContent), - ...getExpressions(expressionTokens) + getFulltext(fulltextTokens, includingNoteContent), + getExpression(expressionTokens) ]); }