From 32ecb43b5cc0637fdccc59beff16580cbb453605 Mon Sep 17 00:00:00 2001 From: zadam Date: Tue, 21 Jul 2020 00:01:07 +0200 Subject: [PATCH] search refactoring --- spec/search/lexer.spec.js | 38 +++++++++---------- spec/search/parens.spec.js | 4 +- spec/search/parser.spec.js | 32 ++++++++-------- spec/search/search.spec.js | 2 +- src/routes/api/autocomplete.js | 2 +- src/routes/api/search.js | 2 +- src/services/backend_script_api.js | 2 +- .../build_comparator.js} | 0 .../{parens.js => services/handle_parens.js} | 6 +-- .../search/{lexer.js => services/lex.js} | 4 +- .../search/{parser.js => services/parse.js} | 32 ++++++++-------- src/services/search/{ => services}/search.js | 28 +++++++------- 12 files changed, 76 insertions(+), 76 deletions(-) rename src/services/search/{comparator_builder.js => services/build_comparator.js} (100%) rename src/services/search/{parens.js => services/handle_parens.js} (88%) rename src/services/search/{lexer.js => services/lex.js} (98%) rename src/services/search/{parser.js => services/parse.js} (89%) rename src/services/search/{ => services}/search.js (87%) diff --git a/spec/search/lexer.spec.js b/spec/search/lexer.spec.js index 3d259d308..09d27b3d3 100644 --- a/spec/search/lexer.spec.js +++ b/spec/search/lexer.spec.js @@ -1,63 +1,63 @@ -const lexer = require('../../src/services/search/lexer.js'); +const lex = require('../../src/services/search/services/lex.js'); describe("Lexer fulltext", () => { it("simple lexing", () => { - expect(lexer("hello world").fulltextTokens.map(t => t.token)) + expect(lex("hello world").fulltextTokens.map(t => t.token)) .toEqual(["hello", "world"]); }); it("use quotes to keep words together", () => { - expect(lexer("'hello world' my friend").fulltextTokens.map(t => t.token)) + expect(lex("'hello world' my friend").fulltextTokens.map(t => t.token)) .toEqual(["hello world", "my", "friend"]); - expect(lexer('"hello world" my friend').fulltextTokens.map(t => t.token)) + expect(lex('"hello world" my friend').fulltextTokens.map(t => t.token)) .toEqual(["hello world", "my", "friend"]); - expect(lexer('`hello world` my friend').fulltextTokens.map(t => t.token)) + expect(lex('`hello world` my friend').fulltextTokens.map(t => t.token)) .toEqual(["hello world", "my", "friend"]); }); it("you can use different quotes and other special characters inside quotes", () => { - expect(lexer("'i can use \" or ` or #~=*' without problem").fulltextTokens.map(t => t.token)) + expect(lex("'i can use \" or ` or #~=*' without problem").fulltextTokens.map(t => t.token)) .toEqual(["i can use \" or ` or #~=*", "without", "problem"]); }); it("if quote is not ended then it's just one long token", () => { - expect(lexer("'unfinished quote").fulltextTokens.map(t => t.token)) + expect(lex("'unfinished quote").fulltextTokens.map(t => t.token)) .toEqual(["unfinished quote"]); }); it("parenthesis and symbols in fulltext section are just normal characters", () => { - expect(lexer("what's u=p ").fulltextTokens.map(t => t.token)) + expect(lex("what's u=p ").fulltextTokens.map(t => t.token)) .toEqual(["what's", "u=p", ""]); }); it("escaping special characters", () => { - expect(lexer("hello \\#\\~\\'").fulltextTokens.map(t => t.token)) + expect(lex("hello \\#\\~\\'").fulltextTokens.map(t => t.token)) .toEqual(["hello", "#~'"]); }); }); describe("Lexer expression", () => { it("simple attribute existence", () => { - expect(lexer("#label ~relation").expressionTokens.map(t => t.token)) + expect(lex("#label ~relation").expressionTokens.map(t => t.token)) .toEqual(["#label", "~relation"]); }); it("simple label operators", () => { - expect(lexer("#label*=*text").expressionTokens.map(t => t.token)) + expect(lex("#label*=*text").expressionTokens.map(t => t.token)) .toEqual(["#label", "*=*", "text"]); }); it("simple label operator with in quotes and without", () => { - expect(lexer("#label*=*'text'").expressionTokens) + expect(lex("#label*=*'text'").expressionTokens) .toEqual([ {token: "#label", inQuotes: false}, {token: "*=*", inQuotes: false}, {token: "text", inQuotes: true} ]); - expect(lexer("#label*=*text").expressionTokens) + expect(lex("#label*=*text").expressionTokens) .toEqual([ {token: "#label", inQuotes: false}, {token: "*=*", inQuotes: false}, @@ -66,35 +66,35 @@ describe("Lexer expression", () => { }); it("complex expressions with and, or and parenthesis", () => { - expect(lexer(`# (#label=text OR #second=text) AND ~relation`).expressionTokens.map(t => t.token)) + expect(lex(`# (#label=text OR #second=text) AND ~relation`).expressionTokens.map(t => t.token)) .toEqual(["#", "(", "#label", "=", "text", "or", "#second", "=", "text", ")", "and", "~relation"]); }); it("dot separated properties", () => { - expect(lexer(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens.map(t => t.token)) + expect(lex(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens.map(t => t.token)) .toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "book title", "=", "silo"]); }); it("negation of label and relation", () => { - expect(lexer(`#!capital ~!neighbor`).expressionTokens.map(t => t.token)) + expect(lex(`#!capital ~!neighbor`).expressionTokens.map(t => t.token)) .toEqual(["#!capital", "~!neighbor"]); }); it("negation of sub-expression", () => { - expect(lexer(`# not(#capital) and note.noteId != "root"`).expressionTokens.map(t => t.token)) + expect(lex(`# not(#capital) and note.noteId != "root"`).expressionTokens.map(t => t.token)) .toEqual(["#", "not", "(", "#capital", ")", "and", "note", ".", "noteid", "!=", "root"]); }); }); describe("Lexer invalid queries and edge cases", () => { it("concatenated attributes", () => { - expect(lexer("#label~relation").expressionTokens.map(t => t.token)) + expect(lex("#label~relation").expressionTokens.map(t => t.token)) .toEqual(["#label", "~relation"]); }); it("spaces in attribute names and values", () => { // invalid but should be reported by parser as an error - expect(lexer(`#'long label'="hello o' world" ~'long relation'`).expressionTokens.map(t => t.token)) + expect(lex(`#'long label'="hello o' world" ~'long relation'`).expressionTokens.map(t => t.token)) .toEqual(["#long label", "=", "hello o' world", "~long relation"]); }); }); diff --git a/spec/search/parens.spec.js b/spec/search/parens.spec.js index b8133028d..d79cfd6ea 100644 --- a/spec/search/parens.spec.js +++ b/spec/search/parens.spec.js @@ -1,11 +1,11 @@ -const parens = require('../../src/services/search/parens.js'); +const handleParens = require('../../src/services/search/services/handle_parens.js'); describe("Parens handler", () => { it("handles parens", () => { const input = ["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"] .map(token => ({token})); - expect(parens(input)) + expect(handleParens(input)) .toEqual([ [ {token: "hello"} diff --git a/spec/search/parser.spec.js b/spec/search/parser.spec.js index 97c4dd386..c0fceb48b 100644 --- a/spec/search/parser.spec.js +++ b/spec/search/parser.spec.js @@ -1,5 +1,5 @@ const ParsingContext = require("../../src/services/search/parsing_context.js"); -const parser = require('../../src/services/search/parser.js'); +const parse = require('../../src/services/search/services/parse.js'); function tokens(...args) { return args.map(arg => { @@ -17,7 +17,7 @@ function tokens(...args) { describe("Parser", () => { it("fulltext parser without content", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: tokens("hello", "hi"), expressionTokens: [], parsingContext: new ParsingContext({includeNoteContent: false}) @@ -28,7 +28,7 @@ describe("Parser", () => { }); it("fulltext parser with content", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: tokens("hello", "hi"), expressionTokens: [], parsingContext: new ParsingContext({includeNoteContent: true}) @@ -48,7 +48,7 @@ describe("Parser", () => { }); it("simple label comparison", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: [], expressionTokens: tokens("#mylabel", "=", "text"), parsingContext: new ParsingContext() @@ -61,7 +61,7 @@ describe("Parser", () => { }); it("simple attribute negation", () => { - let rootExp = parser({ + let rootExp = parse({ fulltextTokens: [], expressionTokens: tokens("#!mylabel"), parsingContext: new ParsingContext() @@ -72,7 +72,7 @@ describe("Parser", () => { expect(rootExp.subExpression.attributeType).toEqual("label"); expect(rootExp.subExpression.attributeName).toEqual("mylabel"); - rootExp = parser({ + rootExp = parse({ fulltextTokens: [], expressionTokens: tokens("~!myrelation"), parsingContext: new ParsingContext() @@ -85,7 +85,7 @@ describe("Parser", () => { }); it("simple label AND", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: [], expressionTokens: tokens("#first", "=", "text", "and", "#second", "=", "text"), parsingContext: new ParsingContext(true) @@ -102,7 +102,7 @@ describe("Parser", () => { }); it("simple label AND without explicit AND", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: [], expressionTokens: tokens("#first", "=", "text", "#second", "=", "text"), parsingContext: new ParsingContext() @@ -119,7 +119,7 @@ describe("Parser", () => { }); it("simple label OR", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: [], expressionTokens: tokens("#first", "=", "text", "or", "#second", "=", "text"), parsingContext: new ParsingContext() @@ -136,7 +136,7 @@ describe("Parser", () => { }); it("fulltext and simple label", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: tokens("hello"), expressionTokens: tokens("#mylabel", "=", "text"), parsingContext: new ParsingContext() @@ -153,7 +153,7 @@ describe("Parser", () => { }); it("label sub-expression", () => { - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: [], expressionTokens: tokens("#first", "=", "text", "or", tokens("#second", "=", "text", "and", "#third", "=", "text")), parsingContext: new ParsingContext() @@ -176,11 +176,11 @@ describe("Parser", () => { }); }); -describe("Invalid tokens", () => { +describe("Invalid expressions", () => { it("incomplete comparison", () => { const parsingContext = new ParsingContext(); - parser({ + parse({ fulltextTokens: [], expressionTokens: tokens("#first", "="), parsingContext @@ -192,7 +192,7 @@ describe("Invalid tokens", () => { it("comparison between labels is impossible", () => { let parsingContext = new ParsingContext(); - parser({ + parse({ fulltextTokens: [], expressionTokens: tokens("#first", "=", "#second"), parsingContext @@ -202,7 +202,7 @@ describe("Invalid tokens", () => { parsingContext = new ParsingContext(); - parser({ + parse({ fulltextTokens: [], expressionTokens: tokens("#first", "=", "note", ".", "relations", "second"), parsingContext @@ -210,7 +210,7 @@ describe("Invalid tokens", () => { expect(parsingContext.error).toEqual(`Error near token "note", it's possible to compare with constant only.`); - const rootExp = parser({ + const rootExp = parse({ fulltextTokens: [], expressionTokens: [ { token: "#first", inQuotes: false }, diff --git a/spec/search/search.spec.js b/spec/search/search.spec.js index a70005071..47b8ae00f 100644 --- a/spec/search/search.spec.js +++ b/spec/search/search.spec.js @@ -1,4 +1,4 @@ -const searchService = require('../../src/services/search/search.js'); +const searchService = require('../../src/services/search/services/search.js'); const Note = require('../../src/services/note_cache/entities/note.js'); const Branch = require('../../src/services/note_cache/entities/branch.js'); const Attribute = require('../../src/services/note_cache/entities/attribute.js'); diff --git a/src/routes/api/autocomplete.js b/src/routes/api/autocomplete.js index 59739b6fc..a6aad50fa 100644 --- a/src/routes/api/autocomplete.js +++ b/src/routes/api/autocomplete.js @@ -1,7 +1,7 @@ "use strict"; const noteCacheService = require('../../services/note_cache/note_cache_service'); -const searchService = require('../../services/search/search'); +const searchService = require('../../services/search/services/search.js'); const repository = require('../../services/repository'); const log = require('../../services/log'); const utils = require('../../services/utils'); diff --git a/src/routes/api/search.js b/src/routes/api/search.js index 866f9678b..2be781f8d 100644 --- a/src/routes/api/search.js +++ b/src/routes/api/search.js @@ -4,7 +4,7 @@ const repository = require('../../services/repository'); const noteCacheService = require('../../services/note_cache/note_cache.js'); const log = require('../../services/log'); const scriptService = require('../../services/script'); -const searchService = require('../../services/search/search'); +const searchService = require('../../services/search/services/search.js'); function searchNotes(req) { const {count, results} = searchService.searchTrimmedNotes(req.params.searchString); diff --git a/src/services/backend_script_api.js b/src/services/backend_script_api.js index eaf9f10db..c5805de99 100644 --- a/src/services/backend_script_api.js +++ b/src/services/backend_script_api.js @@ -12,7 +12,7 @@ const dayjs = require('dayjs'); const cloningService = require('./cloning'); const ws = require('./ws.js'); const appInfo = require('./app_info'); -const searchService = require('./search/search'); +const searchService = require('./search/services/search.js'); /** * This is the main backend API interface for scripts. It's published in the local "api" object. diff --git a/src/services/search/comparator_builder.js b/src/services/search/services/build_comparator.js similarity index 100% rename from src/services/search/comparator_builder.js rename to src/services/search/services/build_comparator.js diff --git a/src/services/search/parens.js b/src/services/search/services/handle_parens.js similarity index 88% rename from src/services/search/parens.js rename to src/services/search/services/handle_parens.js index 4641e73a6..c391b2892 100644 --- a/src/services/search/parens.js +++ b/src/services/search/services/handle_parens.js @@ -1,7 +1,7 @@ /** * This will create a recursive object from list of tokens - tokens between parenthesis are grouped in a single array */ -function parens(tokens) { +function handle_parens(tokens) { if (tokens.length === 0) { return []; } @@ -34,10 +34,10 @@ function parens(tokens) { tokens = [ ...tokens.slice(0, leftIdx), - parens(tokens.slice(leftIdx + 1, rightIdx)), + handle_parens(tokens.slice(leftIdx + 1, rightIdx)), ...tokens.slice(rightIdx + 1) ]; } } -module.exports = parens; +module.exports = handle_parens; diff --git a/src/services/search/lexer.js b/src/services/search/services/lex.js similarity index 98% rename from src/services/search/lexer.js rename to src/services/search/services/lex.js index fb6f09e4d..65ff8823a 100644 --- a/src/services/search/lexer.js +++ b/src/services/search/services/lex.js @@ -1,4 +1,4 @@ -function lexer(str) { +function lex(str) { str = str.toLowerCase(); const fulltextTokens = []; @@ -130,4 +130,4 @@ function lexer(str) { } } -module.exports = lexer; +module.exports = lex; diff --git a/src/services/search/parser.js b/src/services/search/services/parse.js similarity index 89% rename from src/services/search/parser.js rename to src/services/search/services/parse.js index b18307d0f..020e84fe9 100644 --- a/src/services/search/parser.js +++ b/src/services/search/services/parse.js @@ -1,21 +1,21 @@ "use strict"; -const AndExp = require('./expressions/and'); -const OrExp = require('./expressions/or'); -const NotExp = require('./expressions/not'); -const ChildOfExp = require('./expressions/child_of'); -const DescendantOfExp = require('./expressions/descendant_of'); -const ParentOfExp = require('./expressions/parent_of'); -const RelationWhereExp = require('./expressions/relation_where'); -const PropertyComparisonExp = require('./expressions/property_comparison'); -const AttributeExistsExp = require('./expressions/attribute_exists'); -const LabelComparisonExp = require('./expressions/label_comparison'); -const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); -const NoteContentProtectedFulltextExp = require('./expressions/note_content_protected_fulltext'); -const NoteContentUnprotectedFulltextExp = require('./expressions/note_content_unprotected_fulltext'); -const OrderByAndLimitExp = require('./expressions/order_by_and_limit'); -const comparatorBuilder = require('./comparator_builder'); -const ValueExtractor = require('./value_extractor'); +const AndExp = require('../expressions/and.js'); +const OrExp = require('../expressions/or.js'); +const NotExp = require('../expressions/not.js'); +const ChildOfExp = require('../expressions/child_of.js'); +const DescendantOfExp = require('../expressions/descendant_of.js'); +const ParentOfExp = require('../expressions/parent_of.js'); +const RelationWhereExp = require('../expressions/relation_where.js'); +const PropertyComparisonExp = require('../expressions/property_comparison.js'); +const AttributeExistsExp = require('../expressions/attribute_exists.js'); +const LabelComparisonExp = require('../expressions/label_comparison.js'); +const NoteCacheFulltextExp = require('../expressions/note_cache_fulltext.js'); +const NoteContentProtectedFulltextExp = require('../expressions/note_content_protected_fulltext.js'); +const NoteContentUnprotectedFulltextExp = require('../expressions/note_content_unprotected_fulltext.js'); +const OrderByAndLimitExp = require('../expressions/order_by_and_limit.js'); +const comparatorBuilder = require('./build_comparator.js'); +const ValueExtractor = require('../value_extractor.js'); function getFulltext(tokens, parsingContext) { tokens = tokens.map(t => t.token); diff --git a/src/services/search/search.js b/src/services/search/services/search.js similarity index 87% rename from src/services/search/search.js rename to src/services/search/services/search.js index 415df5ed6..e76481b1c 100644 --- a/src/services/search/search.js +++ b/src/services/search/services/search.js @@ -1,16 +1,16 @@ "use strict"; -const lexer = require('./lexer'); -const parens = require('./parens'); -const parser = require('./parser'); -const NoteSet = require("./note_set"); -const SearchResult = require("./search_result"); -const ParsingContext = require("./parsing_context"); -const noteCache = require('../note_cache/note_cache'); -const noteCacheService = require('../note_cache/note_cache_service'); -const hoistedNoteService = require('../hoisted_note'); -const repository = require('../repository'); -const utils = require('../utils'); +const lex = require('./lex.js'); +const handleParens = require('./handle_parens.js'); +const parse = require('./parse.js'); +const NoteSet = require("../note_set.js"); +const SearchResult = require("../search_result.js"); +const ParsingContext = require("../parsing_context.js"); +const noteCache = require('../../note_cache/note_cache.js'); +const noteCacheService = require('../../note_cache/note_cache_service.js'); +const hoistedNoteService = require('../../hoisted_note.js'); +const repository = require('../../repository.js'); +const utils = require('../../utils.js'); /** * @param {Expression} expression @@ -51,10 +51,10 @@ function findNotesWithExpression(expression) { } function parseQueryToExpression(query, parsingContext) { - const {fulltextTokens, expressionTokens} = lexer(query); - const structuredExpressionTokens = parens(expressionTokens); + const {fulltextTokens, expressionTokens} = lex(query); + const structuredExpressionTokens = handleParens(expressionTokens); - const expression = parser({ + const expression = parse({ fulltextTokens, expressionTokens: structuredExpressionTokens, parsingContext