server-ts: Port services/search/services/search

This commit is contained in:
Elian Doran 2024-02-18 11:16:30 +02:00
parent 15169289f0
commit 2fbd2e3c29
No known key found for this signature in database
18 changed files with 115 additions and 100 deletions

View File

@ -1,4 +1,4 @@
const searchService = require('../../src/services/search/services/search.js'); const searchService = require('../../src/services/search/services/search');
const BNote = require('../../src/becca/entities/bnote.js'); const BNote = require('../../src/becca/entities/bnote.js');
const BBranch = require('../../src/becca/entities/bbranch.js'); const BBranch = require('../../src/becca/entities/bbranch.js');
const SearchContext = require('../../src/services/search/search_context'); const SearchContext = require('../../src/services/search/search_context');

View File

@ -78,13 +78,13 @@ class BNote extends AbstractBeccaEntity<BNote> {
// following attributes are filled during searching in the database // following attributes are filled during searching in the database
/** size of the content in bytes */ /** size of the content in bytes */
private contentSize!: number | null; contentSize!: number | null;
/** size of the note content, attachment contents in bytes */ /** size of the note content, attachment contents in bytes */
private contentAndAttachmentsSize!: number | null; contentAndAttachmentsSize!: number | null;
/** size of the note content, attachment contents and revision contents in bytes */ /** size of the note content, attachment contents and revision contents in bytes */
private contentAndAttachmentsAndRevisionsSize!: number | null; contentAndAttachmentsAndRevisionsSize!: number | null;
/** number of note revisions for this note */ /** number of note revisions for this note */
private revisionCount!: number | null; revisionCount!: number | null;
constructor(row?: Partial<NoteRow>) { constructor(row?: Partial<NoteRow>) {
super(); super();

View File

@ -5,7 +5,7 @@ const mappers = require('./mappers.js');
const noteService = require('../services/notes'); const noteService = require('../services/notes');
const TaskContext = require('../services/task_context'); const TaskContext = require('../services/task_context');
const v = require('./validators.js'); const v = require('./validators.js');
const searchService = require('../services/search/services/search.js'); const searchService = require('../services/search/services/search');
const SearchContext = require('../services/search/search_context'); const SearchContext = require('../services/search/search_context');
const zipExportService = require('../services/export/zip.js'); const zipExportService = require('../services/export/zip.js');
const zipImportService = require('../services/import/zip.js'); const zipImportService = require('../services/import/zip.js');

View File

@ -1,7 +1,7 @@
"use strict"; "use strict";
const beccaService = require('../../becca/becca_service'); const beccaService = require('../../becca/becca_service');
const searchService = require('../../services/search/services/search.js'); const searchService = require('../../services/search/services/search');
const log = require('../../services/log'); const log = require('../../services/log');
const utils = require('../../services/utils'); const utils = require('../../services/utils');
const cls = require('../../services/cls'); const cls = require('../../services/cls');

View File

@ -2,7 +2,7 @@
const optionService = require('../../services/options'); const optionService = require('../../services/options');
const log = require('../../services/log'); const log = require('../../services/log');
const searchService = require('../../services/search/services/search.js'); const searchService = require('../../services/search/services/search');
const ValidationError = require('../../errors/validation_error'); const ValidationError = require('../../errors/validation_error');
// options allowed to be updated directly in the Options dialog // options allowed to be updated directly in the Options dialog

View File

@ -2,7 +2,7 @@
const becca = require('../../becca/becca'); const becca = require('../../becca/becca');
const SearchContext = require('../../services/search/search_context'); const SearchContext = require('../../services/search/search_context');
const searchService = require('../../services/search/services/search.js'); const searchService = require('../../services/search/services/search');
const bulkActionService = require('../../services/bulk_actions.js'); const bulkActionService = require('../../services/bulk_actions.js');
const cls = require('../../services/cls'); const cls = require('../../services/cls');
const {formatAttrForSearch} = require('../../services/attribute_formatter'); const {formatAttrForSearch} = require('../../services/attribute_formatter');

View File

@ -46,7 +46,7 @@ const attributesRoute = require('./api/attributes.js');
const scriptRoute = require('./api/script.js'); const scriptRoute = require('./api/script.js');
const senderRoute = require('./api/sender.js'); const senderRoute = require('./api/sender.js');
const filesRoute = require('./api/files.js'); const filesRoute = require('./api/files.js');
const searchRoute = require('./api/search.js'); const searchRoute = require('./api/search');
const bulkActionRoute = require('./api/bulk_action.js'); const bulkActionRoute = require('./api/bulk_action.js');
const specialNotesRoute = require('./api/special_notes.js'); const specialNotesRoute = require('./api/special_notes.js');
const noteMapRoute = require('./api/note_map.js'); const noteMapRoute = require('./api/note_map.js');

View File

@ -1,6 +1,6 @@
"use strict"; "use strict";
const searchService = require('./search/services/search.js'); const searchService = require('./search/services/search');
const sql = require('./sql'); const sql = require('./sql');
const becca = require('../becca/becca'); const becca = require('../becca/becca');
const BAttribute = require('../becca/entities/battribute'); const BAttribute = require('../becca/entities/battribute');

View File

@ -11,7 +11,7 @@ const dayjs = require('dayjs');
const xml2js = require('xml2js'); const xml2js = require('xml2js');
const cloningService = require('./cloning.js'); const cloningService = require('./cloning.js');
const appInfo = require('./app_info'); const appInfo = require('./app_info');
const searchService = require('./search/services/search.js'); const searchService = require('./search/services/search');
const SearchContext = require('./search/search_context'); const SearchContext = require('./search/search_context');
const becca = require('../becca/becca'); const becca = require('../becca/becca');
const ws = require('./ws'); const ws = require('./ws');

View File

@ -5,7 +5,7 @@ const attributeService = require('./attributes.js');
const dateUtils = require('./date_utils'); const dateUtils = require('./date_utils');
const sql = require('./sql'); const sql = require('./sql');
const protectedSessionService = require('./protected_session'); const protectedSessionService = require('./protected_session');
const searchService = require('../services/search/services/search.js'); const searchService = require('../services/search/services/search');
const SearchContext = require('../services/search/search_context'); const SearchContext = require('../services/search/search_context');
const hoistedNoteService = require('./hoisted_note'); const hoistedNoteService = require('./hoisted_note');

View File

@ -1,20 +1,7 @@
"use strict"; "use strict";
import hoistedNoteService = require('../hoisted_note'); import hoistedNoteService = require('../hoisted_note');
import { SearchParams } from './services/types';
interface SearchParams {
fastSearch?: boolean;
includeArchivedNotes?: boolean;
includeHiddenNotes?: boolean;
ignoreHoistedNote?: boolean;
ancestorNoteId?: string;
ancestorDepth?: string;
orderBy?: string;
orderDirection?: string;
limit?: number;
debug?: boolean;
fuzzyAttributeSearch?: boolean;
}
class SearchContext { class SearchContext {
@ -26,9 +13,9 @@ class SearchContext {
ancestorDepth?: string; ancestorDepth?: string;
orderBy?: string; orderBy?: string;
orderDirection?: string; orderDirection?: string;
limit?: number; limit?: number | null;
debug?: boolean; debug?: boolean;
debugInfo: string | null; debugInfo: {} | null;
fuzzyAttributeSearch: boolean; fuzzyAttributeSearch: boolean;
highlightedTokens: string[]; highlightedTokens: string[];
originalQuery: string; originalQuery: string;

View File

@ -4,13 +4,15 @@ import beccaService = require('../../becca/becca_service');
import becca = require('../../becca/becca'); import becca = require('../../becca/becca');
class SearchResult { class SearchResult {
private notePathArray: string[]; notePathArray: string[];
private notePathTitle: string; score: number;
private score?: number; notePathTitle: string;
highlightedNotePathTitle?: string;
constructor(notePathArray: string[]) { constructor(notePathArray: string[]) {
this.notePathArray = notePathArray; this.notePathArray = notePathArray;
this.notePathTitle = beccaService.getNoteTitleForPath(notePathArray); this.notePathTitle = beccaService.getNoteTitleForPath(notePathArray);
this.score = 0;
} }
get notePath() { get notePath() {

View File

@ -448,13 +448,14 @@ function getExpression(tokens: TokenData[], searchContext: SearchContext, level
function parse({fulltextTokens, expressionTokens, searchContext}: { function parse({fulltextTokens, expressionTokens, searchContext}: {
fulltextTokens: TokenData[], fulltextTokens: TokenData[],
expressionTokens: TokenData[], expressionTokens: (TokenData | TokenData[])[],
searchContext: SearchContext searchContext: SearchContext,
originalQuery: string
}) { }) {
let expression: Expression | undefined | null; let expression: Expression | undefined | null;
try { try {
expression = getExpression(expressionTokens, searchContext); expression = getExpression(expressionTokens as TokenData[], searchContext);
} }
catch (e: any) { catch (e: any) {
searchContext.addError(e.message); searchContext.addError(e.message);
@ -475,7 +476,7 @@ function parse({fulltextTokens, expressionTokens, searchContext}: {
exp = new OrderByAndLimitExp([{ exp = new OrderByAndLimitExp([{
valueExtractor: new ValueExtractor(searchContext, ['note', searchContext.orderBy]), valueExtractor: new ValueExtractor(searchContext, ['note', searchContext.orderBy]),
direction: searchContext.orderDirection direction: searchContext.orderDirection
}], searchContext.limit); }], searchContext.limit || undefined);
(exp as any).subExpression = filterExp; (exp as any).subExpression = filterExp;
} }

View File

@ -1,22 +1,28 @@
"use strict"; "use strict";
const normalizeString = require("normalize-strings"); import normalizeString = require("normalize-strings");
const lex = require('./lex'); import lex = require('./lex');
const handleParens = require('./handle_parens'); import handleParens = require('./handle_parens');
const parse = require('./parse'); import parse = require('./parse');
const SearchResult = require('../search_result'); import SearchResult = require('../search_result');
const SearchContext = require('../search_context'); import SearchContext = require('../search_context');
const becca = require('../../../becca/becca'); import becca = require('../../../becca/becca');
const beccaService = require('../../../becca/becca_service'); import beccaService = require('../../../becca/becca_service');
const utils = require('../../utils'); import utils = require('../../utils');
const log = require('../../log'); import log = require('../../log');
const hoistedNoteService = require('../../hoisted_note'); import hoistedNoteService = require('../../hoisted_note');
import BNote = require("../../../becca/entities/bnote");
import BAttribute = require("../../../becca/entities/battribute");
import { SearchParams, TokenData } from "./types";
import Expression = require("../expressions/expression");
import sql = require("../../sql");
function searchFromNote(note) { function searchFromNote(note: BNote) {
let searchResultNoteIds, highlightedTokens; let searchResultNoteIds;
let highlightedTokens: string[];
const searchScript = note.getRelationValue('searchScript'); const searchScript = note.getRelationValue('searchScript');
const searchString = note.getLabelValue('searchString'); const searchString = note.getLabelValue('searchString') || "";
let error = null; let error = null;
if (searchScript) { if (searchScript) {
@ -25,12 +31,12 @@ function searchFromNote(note) {
} else { } else {
const searchContext = new SearchContext({ const searchContext = new SearchContext({
fastSearch: note.hasLabel('fastSearch'), fastSearch: note.hasLabel('fastSearch'),
ancestorNoteId: note.getRelationValue('ancestor'), ancestorNoteId: note.getRelationValue('ancestor') || undefined,
ancestorDepth: note.getLabelValue('ancestorDepth'), ancestorDepth: note.getLabelValue('ancestorDepth') || undefined,
includeArchivedNotes: note.hasLabel('includeArchivedNotes'), includeArchivedNotes: note.hasLabel('includeArchivedNotes'),
orderBy: note.getLabelValue('orderBy'), orderBy: note.getLabelValue('orderBy') || undefined,
orderDirection: note.getLabelValue('orderDirection'), orderDirection: note.getLabelValue('orderDirection') || undefined,
limit: note.getLabelValue('limit'), limit: parseInt(note.getLabelValue('limit') || "0", 10),
debug: note.hasLabel('debug'), debug: note.hasLabel('debug'),
fuzzyAttributeSearch: false fuzzyAttributeSearch: false
}); });
@ -51,7 +57,7 @@ function searchFromNote(note) {
}; };
} }
function searchFromRelation(note, relationName) { function searchFromRelation(note: BNote, relationName: string) {
const scriptNote = note.getRelationTarget(relationName); const scriptNote = note.getRelationTarget(relationName);
if (!scriptNote) { if (!scriptNote) {
@ -90,18 +96,21 @@ function searchFromRelation(note, relationName) {
} }
function loadNeededInfoFromDatabase() { function loadNeededInfoFromDatabase() {
const sql = require('../../sql');
/** /**
* This complex structure is needed to calculate total occupied space by a note. Several object instances * This complex structure is needed to calculate total occupied space by a note. Several object instances
* (note, revisions, attachments) can point to a single blobId, and thus the blob size should count towards the total * (note, revisions, attachments) can point to a single blobId, and thus the blob size should count towards the total
* only once. * only once.
* *
* @var {Object.<string, Object.<string, int>>} - noteId => { blobId => blobSize } * noteId => { blobId => blobSize }
*/ */
const noteBlobs = {}; const noteBlobs: Record<string, Record<string, number>> = {};
const noteContentLengths = sql.getRows(` type NoteContentLengthsRow = {
noteId: string;
blobId: string;
length: number;
};
const noteContentLengths = sql.getRows<NoteContentLengthsRow>(`
SELECT SELECT
noteId, noteId,
blobId, blobId,
@ -122,7 +131,12 @@ function loadNeededInfoFromDatabase() {
noteBlobs[noteId] = { [blobId]: length }; noteBlobs[noteId] = { [blobId]: length };
} }
const attachmentContentLengths = sql.getRows(` type AttachmentContentLengthsRow = {
noteId: string;
blobId: string;
length: number;
};
const attachmentContentLengths = sql.getRows<AttachmentContentLengthsRow>(`
SELECT SELECT
ownerId AS noteId, ownerId AS noteId,
attachments.blobId, attachments.blobId,
@ -151,7 +165,13 @@ function loadNeededInfoFromDatabase() {
becca.notes[noteId].contentAndAttachmentsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0); becca.notes[noteId].contentAndAttachmentsSize = Object.values(noteBlobs[noteId]).reduce((acc, size) => acc + size, 0);
} }
const revisionContentLengths = sql.getRows(` type RevisionRow = {
noteId: string;
blobId: string;
length: number;
isNoteRevision: true;
};
const revisionContentLengths = sql.getRows<RevisionRow>(`
SELECT SELECT
noteId, noteId,
revisions.blobId, revisions.blobId,
@ -186,8 +206,11 @@ function loadNeededInfoFromDatabase() {
noteBlobs[noteId][blobId] = length; noteBlobs[noteId][blobId] = length;
if (isNoteRevision) { if (isNoteRevision) {
becca.notes[noteId].revisionCount++; const noteRevision = becca.notes[noteId];
if (noteRevision && noteRevision.revisionCount) {
noteRevision.revisionCount++;
}
} }
} }
@ -196,20 +219,16 @@ function loadNeededInfoFromDatabase() {
} }
} }
/** function findResultsWithExpression(expression: Expression, searchContext: SearchContext): SearchResult[] {
* @param {Expression} expression
* @param {SearchContext} searchContext
* @returns {SearchResult[]}
*/
function findResultsWithExpression(expression, searchContext) {
if (searchContext.dbLoadNeeded) { if (searchContext.dbLoadNeeded) {
loadNeededInfoFromDatabase(); loadNeededInfoFromDatabase();
} }
const allNoteSet = becca.getAllNoteSet(); const allNoteSet = becca.getAllNoteSet();
const noteIdToNotePath: Record<string, string[]> = {};
const executionContext = { const executionContext = {
noteIdToNotePath: {} noteIdToNotePath
}; };
const noteSet = expression.execute(allNoteSet, executionContext, searchContext); const noteSet = expression.execute(allNoteSet, executionContext, searchContext);
@ -250,16 +269,16 @@ function findResultsWithExpression(expression, searchContext) {
return searchResults; return searchResults;
} }
function parseQueryToExpression(query, searchContext) { function parseQueryToExpression(query: string, searchContext: SearchContext) {
const {fulltextQuery, fulltextTokens, expressionTokens} = lex(query); const {fulltextQuery, fulltextTokens, expressionTokens} = lex(query);
searchContext.fulltextQuery = fulltextQuery; searchContext.fulltextQuery = fulltextQuery;
let structuredExpressionTokens; let structuredExpressionTokens: (TokenData | TokenData[])[];
try { try {
structuredExpressionTokens = handleParens(expressionTokens); structuredExpressionTokens = handleParens(expressionTokens);
} }
catch (e) { catch (e: any) {
structuredExpressionTokens = []; structuredExpressionTokens = [];
searchContext.addError(e.message); searchContext.addError(e.message);
} }
@ -284,23 +303,13 @@ function parseQueryToExpression(query, searchContext) {
return expression; return expression;
} }
/** function searchNotes(query: string, params: SearchParams = {}): BNote[] {
* @param {string} query
* @param {object} params - see SearchContext
* @returns {BNote[]}
*/
function searchNotes(query, params = {}) {
const searchResults = findResultsWithQuery(query, new SearchContext(params)); const searchResults = findResultsWithQuery(query, new SearchContext(params));
return searchResults.map(sr => becca.notes[sr.noteId]); return searchResults.map(sr => becca.notes[sr.noteId]);
} }
/** function findResultsWithQuery(query: string, searchContext: SearchContext): SearchResult[] {
* @param {string} query
* @param {SearchContext} searchContext
* @returns {SearchResult[]}
*/
function findResultsWithQuery(query, searchContext) {
query = query || ""; query = query || "";
searchContext.originalQuery = query; searchContext.originalQuery = query;
@ -313,18 +322,13 @@ function findResultsWithQuery(query, searchContext) {
return findResultsWithExpression(expression, searchContext); return findResultsWithExpression(expression, searchContext);
} }
/** function findFirstNoteWithQuery(query: string, searchContext: SearchContext): BNote | null {
* @param {string} query
* @param {SearchContext} searchContext
* @returns {BNote|null}
*/
function findFirstNoteWithQuery(query, searchContext) {
const searchResults = findResultsWithQuery(query, searchContext); const searchResults = findResultsWithQuery(query, searchContext);
return searchResults.length > 0 ? becca.notes[searchResults[0].noteId] : null; return searchResults.length > 0 ? becca.notes[searchResults[0].noteId] : null;
} }
function searchNotesForAutocomplete(query) { function searchNotesForAutocomplete(query: string) {
const searchContext = new SearchContext({ const searchContext = new SearchContext({
fastSearch: true, fastSearch: true,
includeArchivedNotes: false, includeArchivedNotes: false,
@ -351,7 +355,7 @@ function searchNotesForAutocomplete(query) {
}); });
} }
function highlightSearchResults(searchResults, highlightedTokens) { function highlightSearchResults(searchResults: SearchResult[], highlightedTokens: string[]) {
highlightedTokens = Array.from(new Set(highlightedTokens)); highlightedTokens = Array.from(new Set(highlightedTokens));
// we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks
@ -387,7 +391,7 @@ function highlightSearchResults(searchResults, highlightedTokens) {
} }
} }
function wrapText(text, start, length, prefix, suffix) { function wrapText(text: string, start: number, length: number, prefix: string, suffix: string) {
return text.substring(0, start) + prefix + text.substr(start, length) + suffix + text.substring(start + length); return text.substring(0, start) + prefix + text.substr(start, length) + suffix + text.substring(start + length);
} }
@ -403,6 +407,7 @@ function highlightSearchResults(searchResults, highlightedTokens) {
let match; let match;
// Find all matches // Find all matches
if (!result.highlightedNotePathTitle) { continue; }
while ((match = tokenRegex.exec(normalizeString(result.highlightedNotePathTitle))) !== null) { while ((match = tokenRegex.exec(normalizeString(result.highlightedNotePathTitle))) !== null) {
result.highlightedNotePathTitle = wrapText(result.highlightedNotePathTitle, match.index, token.length, "{", "}"); result.highlightedNotePathTitle = wrapText(result.highlightedNotePathTitle, match.index, token.length, "{", "}");
@ -413,6 +418,7 @@ function highlightSearchResults(searchResults, highlightedTokens) {
} }
for (const result of searchResults) { for (const result of searchResults) {
if (!result.highlightedNotePathTitle) { continue; }
result.highlightedNotePathTitle = result.highlightedNotePathTitle result.highlightedNotePathTitle = result.highlightedNotePathTitle
.replace(/"/g, "<small>") .replace(/"/g, "<small>")
.replace(/'/g, "</small>") .replace(/'/g, "</small>")
@ -421,7 +427,7 @@ function highlightSearchResults(searchResults, highlightedTokens) {
} }
} }
function formatAttribute(attr) { function formatAttribute(attr: BAttribute) {
if (attr.type === 'relation') { if (attr.type === 'relation') {
return `~${utils.escapeHtml(attr.name)}=…`; return `~${utils.escapeHtml(attr.name)}=…`;
} }
@ -438,7 +444,7 @@ function formatAttribute(attr) {
} }
} }
module.exports = { export = {
searchFromNote, searchFromNote,
searchNotesForAutocomplete, searchNotesForAutocomplete,
findResultsWithQuery, findResultsWithQuery,

View File

@ -3,4 +3,18 @@ export interface TokenData {
inQuotes?: boolean; inQuotes?: boolean;
startIndex?: number; startIndex?: number;
endIndex?: number; endIndex?: number;
}
export interface SearchParams {
fastSearch?: boolean;
includeArchivedNotes?: boolean;
includeHiddenNotes?: boolean;
ignoreHoistedNote?: boolean;
ancestorNoteId?: string;
ancestorDepth?: string;
orderBy?: string;
orderDirection?: string;
limit?: number | null;
debug?: boolean;
fuzzyAttributeSearch?: boolean;
} }

View File

@ -5,7 +5,7 @@ const noteService = require('./notes');
const dateUtils = require('./date_utils'); const dateUtils = require('./date_utils');
const log = require('./log'); const log = require('./log');
const hoistedNoteService = require('./hoisted_note'); const hoistedNoteService = require('./hoisted_note');
const searchService = require('./search/services/search.js'); const searchService = require('./search/services/search');
const SearchContext = require('./search/search_context'); const SearchContext = require('./search/search_context');
const {LBTPL_NOTE_LAUNCHER, LBTPL_CUSTOM_WIDGET, LBTPL_SPACER, LBTPL_SCRIPT} = require('./hidden_subtree'); const {LBTPL_NOTE_LAUNCHER, LBTPL_CUSTOM_WIDGET, LBTPL_SPACER, LBTPL_SCRIPT} = require('./hidden_subtree');

View File

@ -9,7 +9,7 @@ const shareRoot = require('./share_root.js');
const contentRenderer = require('./content_renderer.js'); const contentRenderer = require('./content_renderer.js');
const assetPath = require('../services/asset_path'); const assetPath = require('../services/asset_path');
const appPath = require('../services/app_path'); const appPath = require('../services/app_path');
const searchService = require('../services/search/services/search.js'); const searchService = require('../services/search/services/search');
const SearchContext = require('../services/search/search_context'); const SearchContext = require('../services/search/search_context');
const log = require('../services/log'); const log = require('../services/log');

5
src/types.d.ts vendored
View File

@ -11,4 +11,9 @@ declare module 'unescape' {
declare module 'html2plaintext' { declare module 'html2plaintext' {
function html2plaintext(htmlText: string): string; function html2plaintext(htmlText: string): string;
export = html2plaintext; export = html2plaintext;
}
declare module 'normalize-strings' {
function normalizeString(string: string): string;
export = normalizeString;
} }