mirror of
https://github.com/zadam/trilium.git
synced 2025-12-05 06:54:23 +01:00
feat(search): get the correct comparison and rice out the fts5 search
Some checks failed
Checks / main (push) Has been cancelled
Some checks failed
Checks / main (push) Has been cancelled
This commit is contained in:
parent
253da139de
commit
10988095c2
@ -11,7 +11,7 @@ import auth from "../services/auth.js";
|
||||
import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js";
|
||||
import { safeExtractMessageAndStackFromError } from "../services/utils.js";
|
||||
|
||||
const MAX_ALLOWED_FILE_SIZE_MB = 250;
|
||||
const MAX_ALLOWED_FILE_SIZE_MB = 2500;
|
||||
export const router = express.Router();
|
||||
|
||||
// TODO: Deduplicate with etapi_utils.ts afterwards.
|
||||
|
||||
@ -4,7 +4,7 @@ import packageJson from "../../package.json" with { type: "json" };
|
||||
import dataDir from "./data_dir.js";
|
||||
import { AppInfo } from "@triliumnext/commons";
|
||||
|
||||
const APP_DB_VERSION = 235;
|
||||
const APP_DB_VERSION = 236;
|
||||
const SYNC_VERSION = 36;
|
||||
const CLIPPER_PROTOCOL_VERSION = "1.0";
|
||||
|
||||
|
||||
@ -81,30 +81,40 @@ class NoteContentFulltextExp extends Expression {
|
||||
// Try to use FTS5 if available for better performance
|
||||
if (ftsSearchService.checkFTS5Availability() && this.canUseFTS5()) {
|
||||
try {
|
||||
// Performance comparison logging for FTS5 vs traditional search
|
||||
const searchQuery = this.tokens.join(" ");
|
||||
const isQuickSearch = searchContext.fastSearch === false; // quick-search sets fastSearch to false
|
||||
if (isQuickSearch) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Starting comparison for query: "${searchQuery}" with operator: ${this.operator}`);
|
||||
}
|
||||
|
||||
// Check if we need to search protected notes
|
||||
const searchProtected = protectedSessionService.isProtectedSessionAvailable();
|
||||
|
||||
// Time FTS5 search
|
||||
const ftsStartTime = Date.now();
|
||||
const noteIdSet = inputNoteSet.getNoteIds();
|
||||
const ftsResults = ftsSearchService.searchSync(
|
||||
|
||||
// Determine which FTS5 method to use based on operator
|
||||
let ftsResults;
|
||||
if (this.operator === "*=*" || this.operator === "*=" || this.operator === "=*") {
|
||||
// Substring operators use LIKE queries (optimized by trigram index)
|
||||
// Do NOT pass a limit - we want all results to match traditional search behavior
|
||||
ftsResults = ftsSearchService.searchWithLike(
|
||||
this.tokens,
|
||||
this.operator,
|
||||
noteIdSet.size > 0 ? noteIdSet : undefined,
|
||||
{
|
||||
includeSnippets: false,
|
||||
searchProtected: false
|
||||
// No limit specified - return all results
|
||||
},
|
||||
searchContext // Pass context to track internal timing
|
||||
);
|
||||
} else {
|
||||
// Other operators use MATCH syntax
|
||||
ftsResults = ftsSearchService.searchSync(
|
||||
this.tokens,
|
||||
this.operator,
|
||||
noteIdSet.size > 0 ? noteIdSet : undefined,
|
||||
{
|
||||
includeSnippets: false,
|
||||
searchProtected: false // FTS5 doesn't index protected notes
|
||||
}
|
||||
},
|
||||
searchContext // Pass context to track internal timing
|
||||
);
|
||||
const ftsEndTime = Date.now();
|
||||
const ftsTime = ftsEndTime - ftsStartTime;
|
||||
}
|
||||
|
||||
// Add FTS results to note set
|
||||
for (const result of ftsResults) {
|
||||
@ -113,49 +123,6 @@ class NoteContentFulltextExp extends Expression {
|
||||
}
|
||||
}
|
||||
|
||||
// For quick-search, also run traditional search for comparison
|
||||
if (isQuickSearch) {
|
||||
const traditionalStartTime = Date.now();
|
||||
const traditionalNoteSet = new NoteSet();
|
||||
|
||||
// Run traditional search (use the fallback method)
|
||||
const traditionalResults = this.executeWithFallback(inputNoteSet, traditionalNoteSet, searchContext);
|
||||
|
||||
const traditionalEndTime = Date.now();
|
||||
const traditionalTime = traditionalEndTime - traditionalStartTime;
|
||||
|
||||
// Log performance comparison
|
||||
const speedup = traditionalTime > 0 ? (traditionalTime / ftsTime).toFixed(2) : "N/A";
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] ===== Results for query: "${searchQuery}" =====`);
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 search: ${ftsTime}ms, found ${ftsResults.length} results`);
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Traditional search: ${traditionalTime}ms, found ${traditionalResults.notes.length} results`);
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 is ${speedup}x faster (saved ${traditionalTime - ftsTime}ms)`);
|
||||
|
||||
// Check if results match
|
||||
const ftsNoteIds = new Set(ftsResults.map(r => r.noteId));
|
||||
const traditionalNoteIds = new Set(traditionalResults.notes.map(n => n.noteId));
|
||||
const matchingResults = ftsNoteIds.size === traditionalNoteIds.size &&
|
||||
Array.from(ftsNoteIds).every(id => traditionalNoteIds.has(id));
|
||||
|
||||
if (!matchingResults) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Results differ! FTS5: ${ftsNoteIds.size} notes, Traditional: ${traditionalNoteIds.size} notes`);
|
||||
|
||||
// Find differences
|
||||
const onlyInFTS = Array.from(ftsNoteIds).filter(id => !traditionalNoteIds.has(id));
|
||||
const onlyInTraditional = Array.from(traditionalNoteIds).filter(id => !ftsNoteIds.has(id));
|
||||
|
||||
if (onlyInFTS.length > 0) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Only in FTS5: ${onlyInFTS.slice(0, 5).join(", ")}${onlyInFTS.length > 5 ? "..." : ""}`);
|
||||
}
|
||||
if (onlyInTraditional.length > 0) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Only in Traditional: ${onlyInTraditional.slice(0, 5).join(", ")}${onlyInTraditional.length > 5 ? "..." : ""}`);
|
||||
}
|
||||
} else {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Results match perfectly! ✓`);
|
||||
}
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] ========================================`);
|
||||
}
|
||||
|
||||
// If we need to search protected notes, use the separate method
|
||||
if (searchProtected) {
|
||||
const protectedResults = ftsSearchService.searchProtectedNotesSync(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -54,6 +54,7 @@ export interface FTSSearchOptions {
|
||||
snippetLength?: number;
|
||||
highlightTag?: string;
|
||||
searchProtected?: boolean;
|
||||
skipDiagnostics?: boolean; // Skip diagnostic queries for performance measurements
|
||||
}
|
||||
|
||||
export interface FTSErrorInfo {
|
||||
@ -125,6 +126,11 @@ class FTSSearchService {
|
||||
throw new Error("No search tokens provided");
|
||||
}
|
||||
|
||||
// Substring operators (*=*, *=, =*) use LIKE queries now, not MATCH
|
||||
if (operator === "*=*" || operator === "*=" || operator === "=*") {
|
||||
throw new Error("Substring operators should use searchWithLike(), not MATCH queries");
|
||||
}
|
||||
|
||||
// Trigram tokenizer requires minimum 3 characters
|
||||
const shortTokens = tokens.filter(token => token.length < 3);
|
||||
if (shortTokens.length > 0) {
|
||||
@ -140,33 +146,24 @@ class FTSSearchService {
|
||||
this.sanitizeFTS5Token(token)
|
||||
);
|
||||
|
||||
// Only handle operators that work with MATCH
|
||||
switch (operator) {
|
||||
case "=": // Exact match (phrase search)
|
||||
case "=": // Exact phrase match
|
||||
return `"${sanitizedTokens.join(" ")}"`;
|
||||
|
||||
case "*=*": // Contains all tokens (AND)
|
||||
return sanitizedTokens.join(" AND ");
|
||||
|
||||
case "*=": // Ends with
|
||||
return sanitizedTokens.map(t => `*${t}`).join(" AND ");
|
||||
|
||||
case "=*": // Starts with
|
||||
return sanitizedTokens.map(t => `${t}*`).join(" AND ");
|
||||
|
||||
case "!=": // Does not contain (NOT)
|
||||
case "!=": // Does not contain
|
||||
return `NOT (${sanitizedTokens.join(" OR ")})`;
|
||||
|
||||
case "~=": // Fuzzy match (use OR for more flexible matching)
|
||||
case "~*": // Fuzzy contains
|
||||
case "~=": // Fuzzy match (use OR)
|
||||
case "~*":
|
||||
return sanitizedTokens.join(" OR ");
|
||||
|
||||
case "%=": // Regex match - fallback to OR search
|
||||
log.error(`Regex search operator ${operator} not fully supported in FTS5, using OR search`);
|
||||
return sanitizedTokens.join(" OR ");
|
||||
case "%=": // Regex - fallback to custom function
|
||||
log.error(`Regex search operator ${operator} not supported in FTS5`);
|
||||
throw new FTSNotAvailableError("Regex search not supported in FTS5");
|
||||
|
||||
default:
|
||||
// Default to AND search
|
||||
return sanitizedTokens.join(" AND ");
|
||||
throw new FTSQueryError(`Unsupported MATCH operator: ${operator}`);
|
||||
}
|
||||
}
|
||||
|
||||
@ -197,6 +194,249 @@ class FTSSearchService {
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes LIKE wildcards (% and _) in user input to treat them as literals
|
||||
* @param str - User input string
|
||||
* @returns String with LIKE wildcards escaped
|
||||
*/
|
||||
private escapeLikeWildcards(str: string): string {
|
||||
return str.replace(/[%_]/g, '\\$&');
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs substring search using LIKE queries optimized by trigram index
|
||||
* This is used for *=*, *=, and =* operators with detail='none'
|
||||
*
|
||||
* @param tokens - Search tokens
|
||||
* @param operator - Search operator (*=*, *=, =*)
|
||||
* @param noteIds - Optional set of note IDs to filter
|
||||
* @param options - Search options
|
||||
* @param searchContext - Optional search context to track internal timing
|
||||
* @returns Array of search results (noteIds only, no scoring)
|
||||
*/
|
||||
searchWithLike(
|
||||
tokens: string[],
|
||||
operator: string,
|
||||
noteIds?: Set<string>,
|
||||
options: FTSSearchOptions = {},
|
||||
searchContext?: any
|
||||
): FTSSearchResult[] {
|
||||
if (!this.checkFTS5Availability()) {
|
||||
throw new FTSNotAvailableError();
|
||||
}
|
||||
|
||||
// Normalize tokens to lowercase for case-insensitive search
|
||||
const normalizedTokens = tokens.map(t => t.toLowerCase());
|
||||
|
||||
// Validate token lengths to prevent memory issues
|
||||
const MAX_TOKEN_LENGTH = 1000;
|
||||
const longTokens = normalizedTokens.filter(t => t.length > MAX_TOKEN_LENGTH);
|
||||
if (longTokens.length > 0) {
|
||||
throw new FTSQueryError(
|
||||
`Search tokens too long (max ${MAX_TOKEN_LENGTH} characters). ` +
|
||||
`Long tokens: ${longTokens.map(t => t.substring(0, 50) + '...').join(', ')}`
|
||||
);
|
||||
}
|
||||
|
||||
const {
|
||||
limit, // No default limit - return all results
|
||||
offset = 0,
|
||||
skipDiagnostics = false
|
||||
} = options;
|
||||
|
||||
// Run diagnostics BEFORE the actual search (not counted in performance timing)
|
||||
if (!skipDiagnostics) {
|
||||
log.info('[FTS-DIAGNOSTICS] Running index completeness checks (not counted in search timing)...');
|
||||
const totalInFts = sql.getValue<number>(`SELECT COUNT(*) FROM notes_fts`);
|
||||
const totalNotes = sql.getValue<number>(`
|
||||
SELECT COUNT(*)
|
||||
FROM notes n
|
||||
LEFT JOIN blobs b ON n.blobId = b.blobId
|
||||
WHERE n.type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap')
|
||||
AND n.isDeleted = 0
|
||||
AND n.isProtected = 0
|
||||
AND b.content IS NOT NULL
|
||||
`);
|
||||
|
||||
if (totalInFts < totalNotes) {
|
||||
log.warn(`[FTS-DIAGNOSTICS] FTS index incomplete: ${totalInFts} indexed out of ${totalNotes} total notes. Run syncMissingNotes().`);
|
||||
} else {
|
||||
log.info(`[FTS-DIAGNOSTICS] FTS index complete: ${totalInFts} notes indexed`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Start timing for actual search (excludes diagnostics)
|
||||
const searchStartTime = Date.now();
|
||||
|
||||
// Optimization: If noteIds set is very large, skip filtering to avoid expensive IN clauses
|
||||
// The FTS table already excludes protected notes, so we can search all notes
|
||||
const LARGE_SET_THRESHOLD = 1000;
|
||||
const isLargeNoteSet = noteIds && noteIds.size > LARGE_SET_THRESHOLD;
|
||||
|
||||
if (isLargeNoteSet) {
|
||||
log.info(`[FTS-OPTIMIZATION] Large noteIds set (${noteIds!.size} notes) - skipping IN clause filter, searching all FTS notes`);
|
||||
}
|
||||
|
||||
// Only filter noteIds if the set is small enough to benefit from it
|
||||
const shouldFilterByNoteIds = noteIds && noteIds.size > 0 && !isLargeNoteSet;
|
||||
const nonProtectedNoteIds = shouldFilterByNoteIds
|
||||
? this.filterNonProtectedNoteIds(noteIds)
|
||||
: [];
|
||||
|
||||
let whereConditions: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
// Build LIKE conditions for each token - search BOTH title and content
|
||||
switch (operator) {
|
||||
case "*=*": // Contains (substring)
|
||||
normalizedTokens.forEach(token => {
|
||||
// Search in BOTH title and content with escaped wildcards
|
||||
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
|
||||
const escapedToken = this.escapeLikeWildcards(token);
|
||||
params.push(`%${escapedToken}%`, `%${escapedToken}%`);
|
||||
});
|
||||
break;
|
||||
|
||||
case "*=": // Ends with
|
||||
normalizedTokens.forEach(token => {
|
||||
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
|
||||
const escapedToken = this.escapeLikeWildcards(token);
|
||||
params.push(`%${escapedToken}`, `%${escapedToken}`);
|
||||
});
|
||||
break;
|
||||
|
||||
case "=*": // Starts with
|
||||
normalizedTokens.forEach(token => {
|
||||
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
|
||||
const escapedToken = this.escapeLikeWildcards(token);
|
||||
params.push(`${escapedToken}%`, `${escapedToken}%`);
|
||||
});
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new FTSQueryError(`Unsupported LIKE operator: ${operator}`);
|
||||
}
|
||||
|
||||
// Validate that we have search criteria
|
||||
if (whereConditions.length === 0 && nonProtectedNoteIds.length === 0) {
|
||||
throw new FTSQueryError("No search criteria provided (empty tokens and no note filter)");
|
||||
}
|
||||
|
||||
// SQLite parameter limit handling (999 params max)
|
||||
const MAX_PARAMS_PER_QUERY = 900; // Leave margin for other params
|
||||
|
||||
// Add noteId filter if provided
|
||||
if (nonProtectedNoteIds.length > 0) {
|
||||
const tokenParamCount = params.length;
|
||||
const additionalParams = 2; // For limit and offset
|
||||
|
||||
if (nonProtectedNoteIds.length <= MAX_PARAMS_PER_QUERY - tokenParamCount - additionalParams) {
|
||||
// Normal case: all IDs fit in one query
|
||||
whereConditions.push(`noteId IN (${nonProtectedNoteIds.map(() => '?').join(',')})`);
|
||||
params.push(...nonProtectedNoteIds);
|
||||
} else {
|
||||
// Large noteIds set: split into chunks and execute multiple queries
|
||||
const chunks: string[][] = [];
|
||||
for (let i = 0; i < nonProtectedNoteIds.length; i += MAX_PARAMS_PER_QUERY) {
|
||||
chunks.push(nonProtectedNoteIds.slice(i, i + MAX_PARAMS_PER_QUERY));
|
||||
}
|
||||
|
||||
log.info(`Large noteIds set detected (${nonProtectedNoteIds.length} notes), splitting into ${chunks.length} chunks`);
|
||||
|
||||
// Execute a query for each chunk and combine results
|
||||
const allResults: FTSSearchResult[] = [];
|
||||
let remainingLimit = limit !== undefined ? limit : Number.MAX_SAFE_INTEGER;
|
||||
let currentOffset = offset;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
if (remainingLimit <= 0) break;
|
||||
|
||||
const chunkWhereConditions = [...whereConditions];
|
||||
const chunkParams: any[] = [...params];
|
||||
|
||||
chunkWhereConditions.push(`noteId IN (${chunk.map(() => '?').join(',')})`);
|
||||
chunkParams.push(...chunk);
|
||||
|
||||
// Build chunk query
|
||||
const chunkQuery = `
|
||||
SELECT noteId, title
|
||||
FROM notes_fts
|
||||
WHERE ${chunkWhereConditions.join(' AND ')}
|
||||
${remainingLimit !== Number.MAX_SAFE_INTEGER ? 'LIMIT ?' : ''}
|
||||
${currentOffset > 0 ? 'OFFSET ?' : ''}
|
||||
`;
|
||||
|
||||
if (remainingLimit !== Number.MAX_SAFE_INTEGER) chunkParams.push(remainingLimit);
|
||||
if (currentOffset > 0) chunkParams.push(currentOffset);
|
||||
|
||||
const chunkResults = sql.getRows<{ noteId: string; title: string }>(chunkQuery, chunkParams);
|
||||
allResults.push(...chunkResults.map(row => ({
|
||||
noteId: row.noteId,
|
||||
title: row.title,
|
||||
score: 1.0
|
||||
})));
|
||||
|
||||
if (remainingLimit !== Number.MAX_SAFE_INTEGER) {
|
||||
remainingLimit -= chunkResults.length;
|
||||
}
|
||||
currentOffset = 0; // Only apply offset to first chunk
|
||||
}
|
||||
|
||||
const searchTime = Date.now() - searchStartTime;
|
||||
log.info(`FTS5 LIKE search (chunked) returned ${allResults.length} results in ${searchTime}ms (excluding diagnostics)`);
|
||||
|
||||
// Track internal search time on context for performance comparison
|
||||
if (searchContext) {
|
||||
searchContext.ftsInternalSearchTime = searchTime;
|
||||
}
|
||||
|
||||
return allResults;
|
||||
}
|
||||
}
|
||||
|
||||
// Build query - LIKE queries are automatically optimized by trigram index
|
||||
// Only add LIMIT/OFFSET if specified
|
||||
const query = `
|
||||
SELECT noteId, title
|
||||
FROM notes_fts
|
||||
WHERE ${whereConditions.join(' AND ')}
|
||||
${limit !== undefined ? 'LIMIT ?' : ''}
|
||||
${offset > 0 ? 'OFFSET ?' : ''}
|
||||
`;
|
||||
|
||||
// Only add limit/offset params if specified
|
||||
if (limit !== undefined) params.push(limit);
|
||||
if (offset > 0) params.push(offset);
|
||||
|
||||
// Log the search parameters
|
||||
log.info(`FTS5 LIKE search: tokens=[${normalizedTokens.join(', ')}], operator=${operator}, limit=${limit || 'none'}, offset=${offset}`);
|
||||
|
||||
const rows = sql.getRows<{ noteId: string; title: string }>(query, params);
|
||||
|
||||
const searchTime = Date.now() - searchStartTime;
|
||||
log.info(`FTS5 LIKE search returned ${rows.length} results in ${searchTime}ms (excluding diagnostics)`);
|
||||
|
||||
// Track internal search time on context for performance comparison
|
||||
if (searchContext) {
|
||||
searchContext.ftsInternalSearchTime = searchTime;
|
||||
}
|
||||
|
||||
return rows.map(row => ({
|
||||
noteId: row.noteId,
|
||||
title: row.title,
|
||||
score: 1.0 // LIKE queries don't have ranking
|
||||
}));
|
||||
|
||||
} catch (error: any) {
|
||||
log.error(`FTS5 LIKE search error: ${error}`);
|
||||
throw new FTSQueryError(
|
||||
`FTS5 LIKE search failed: ${error.message}`,
|
||||
undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs a synchronous full-text search using FTS5
|
||||
*
|
||||
@ -204,13 +444,15 @@ class FTSSearchService {
|
||||
* @param operator - Search operator
|
||||
* @param noteIds - Optional set of note IDs to search within
|
||||
* @param options - Search options
|
||||
* @param searchContext - Optional search context to track internal timing
|
||||
* @returns Array of search results
|
||||
*/
|
||||
searchSync(
|
||||
tokens: string[],
|
||||
operator: string,
|
||||
noteIds?: Set<string>,
|
||||
options: FTSSearchOptions = {}
|
||||
options: FTSSearchOptions = {},
|
||||
searchContext?: any
|
||||
): FTSSearchResult[] {
|
||||
if (!this.checkFTS5Availability()) {
|
||||
throw new FTSNotAvailableError();
|
||||
@ -226,6 +468,9 @@ class FTSSearchService {
|
||||
} = options;
|
||||
|
||||
try {
|
||||
// Start timing for actual search
|
||||
const searchStartTime = Date.now();
|
||||
|
||||
const ftsQuery = this.convertToFTS5Query(tokens, operator);
|
||||
|
||||
// Validate query length
|
||||
@ -249,10 +494,20 @@ class FTSSearchService {
|
||||
let whereConditions = [`notes_fts MATCH ?`];
|
||||
const params: any[] = [ftsQuery];
|
||||
|
||||
// Filter by noteIds if provided
|
||||
if (noteIds && noteIds.size > 0) {
|
||||
// Optimization: If noteIds set is very large, skip filtering to avoid expensive IN clauses
|
||||
// The FTS table already excludes protected notes, so we can search all notes
|
||||
const LARGE_SET_THRESHOLD = 1000;
|
||||
const isLargeNoteSet = noteIds && noteIds.size > LARGE_SET_THRESHOLD;
|
||||
|
||||
if (isLargeNoteSet) {
|
||||
log.info(`[FTS-OPTIMIZATION] Large noteIds set (${noteIds!.size} notes) - skipping IN clause filter, searching all FTS notes`);
|
||||
}
|
||||
|
||||
// Filter by noteIds if provided and set is small enough
|
||||
const shouldFilterByNoteIds = noteIds && noteIds.size > 0 && !isLargeNoteSet;
|
||||
if (shouldFilterByNoteIds) {
|
||||
// First filter out any protected notes from the noteIds
|
||||
const nonProtectedNoteIds = this.filterNonProtectedNoteIds(noteIds);
|
||||
const nonProtectedNoteIds = this.filterNonProtectedNoteIds(noteIds!);
|
||||
if (nonProtectedNoteIds.length === 0) {
|
||||
// All provided notes are protected, return empty results
|
||||
return [];
|
||||
@ -287,6 +542,14 @@ class FTSSearchService {
|
||||
snippet?: string;
|
||||
}>(query, params);
|
||||
|
||||
const searchTime = Date.now() - searchStartTime;
|
||||
log.info(`FTS5 MATCH search returned ${results.length} results in ${searchTime}ms`);
|
||||
|
||||
// Track internal search time on context for performance comparison
|
||||
if (searchContext) {
|
||||
searchContext.ftsInternalSearchTime = searchTime;
|
||||
}
|
||||
|
||||
return results;
|
||||
|
||||
} catch (error: any) {
|
||||
|
||||
@ -24,6 +24,7 @@ class SearchContext {
|
||||
fulltextQuery: string;
|
||||
dbLoadNeeded: boolean;
|
||||
error: string | null;
|
||||
ftsInternalSearchTime: number | null; // Time spent in actual FTS search (excluding diagnostics)
|
||||
|
||||
constructor(params: SearchParams = {}) {
|
||||
this.fastSearch = !!params.fastSearch;
|
||||
@ -54,6 +55,7 @@ class SearchContext {
|
||||
// and some extra data needs to be loaded before executing
|
||||
this.dbLoadNeeded = false;
|
||||
this.error = null;
|
||||
this.ftsInternalSearchTime = null;
|
||||
}
|
||||
|
||||
addError(error: string) {
|
||||
|
||||
@ -19,6 +19,7 @@ import sql from "../../sql.js";
|
||||
import scriptService from "../../script.js";
|
||||
import striptags from "striptags";
|
||||
import protectedSessionService from "../../protected_session.js";
|
||||
import ftsSearchService from "../fts_search.js";
|
||||
|
||||
export interface SearchNoteResult {
|
||||
searchResultNoteIds: string[];
|
||||
@ -422,13 +423,83 @@ function findResultsWithQuery(query: string, searchContext: SearchContext): Sear
|
||||
// ordering or other logic that shouldn't be interfered with.
|
||||
const isPureExpressionQuery = query.trim().startsWith('#');
|
||||
|
||||
// Performance comparison for quick-search (fastSearch === false)
|
||||
const isQuickSearch = searchContext.fastSearch === false;
|
||||
let results: SearchResult[];
|
||||
let ftsTime = 0;
|
||||
let traditionalTime = 0;
|
||||
|
||||
if (isPureExpressionQuery) {
|
||||
// For pure expression queries, use standard search without progressive phases
|
||||
results = performSearch(expression, searchContext, searchContext.enableFuzzyMatching);
|
||||
} else {
|
||||
// For quick-search, run both FTS5 and traditional search to compare
|
||||
if (isQuickSearch) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Starting comparison for query: "${query}"`);
|
||||
|
||||
// Time FTS5 search (normal path)
|
||||
const ftsStartTime = Date.now();
|
||||
results = findResultsWithExpression(expression, searchContext);
|
||||
ftsTime = Date.now() - ftsStartTime;
|
||||
|
||||
// Time traditional search (with FTS5 disabled)
|
||||
const traditionalStartTime = Date.now();
|
||||
|
||||
// Create a new search context with FTS5 disabled
|
||||
const traditionalContext = new SearchContext({
|
||||
fastSearch: false,
|
||||
includeArchivedNotes: false,
|
||||
includeHiddenNotes: true,
|
||||
fuzzyAttributeSearch: true,
|
||||
ignoreInternalAttributes: true,
|
||||
ancestorNoteId: searchContext.ancestorNoteId
|
||||
});
|
||||
|
||||
// Temporarily disable FTS5 to force traditional search
|
||||
const originalFtsAvailable = (ftsSearchService as any).isFTS5Available;
|
||||
(ftsSearchService as any).isFTS5Available = false;
|
||||
|
||||
const traditionalResults = findResultsWithExpression(expression, traditionalContext);
|
||||
traditionalTime = Date.now() - traditionalStartTime;
|
||||
|
||||
// Restore FTS5 availability
|
||||
(ftsSearchService as any).isFTS5Available = originalFtsAvailable;
|
||||
|
||||
// Log performance comparison
|
||||
// Use internal FTS search time (excluding diagnostics) if available
|
||||
const ftsInternalTime = searchContext.ftsInternalSearchTime ?? ftsTime;
|
||||
const speedup = traditionalTime > 0 ? (traditionalTime / ftsInternalTime).toFixed(2) : "N/A";
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] ===== Results for query: "${query}" =====`);
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 search: ${ftsInternalTime}ms (excluding diagnostics), found ${results.length} results`);
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Traditional search: ${traditionalTime}ms, found ${traditionalResults.length} results`);
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 is ${speedup}x faster (saved ${traditionalTime - ftsInternalTime}ms)`);
|
||||
|
||||
// Check if results match
|
||||
const ftsNoteIds = new Set(results.map(r => r.noteId));
|
||||
const traditionalNoteIds = new Set(traditionalResults.map(r => r.noteId));
|
||||
const matchingResults = ftsNoteIds.size === traditionalNoteIds.size &&
|
||||
Array.from(ftsNoteIds).every(id => traditionalNoteIds.has(id));
|
||||
|
||||
if (!matchingResults) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Results differ! FTS5: ${ftsNoteIds.size} notes, Traditional: ${traditionalNoteIds.size} notes`);
|
||||
|
||||
// Find differences
|
||||
const onlyInFTS = Array.from(ftsNoteIds).filter(id => !traditionalNoteIds.has(id));
|
||||
const onlyInTraditional = Array.from(traditionalNoteIds).filter(id => !ftsNoteIds.has(id));
|
||||
|
||||
if (onlyInFTS.length > 0) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Only in FTS5: ${onlyInFTS.slice(0, 5).join(", ")}${onlyInFTS.length > 5 ? "..." : ""}`);
|
||||
}
|
||||
if (onlyInTraditional.length > 0) {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Only in Traditional: ${onlyInTraditional.slice(0, 5).join(", ")}${onlyInTraditional.length > 5 ? "..." : ""}`);
|
||||
}
|
||||
} else {
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] Results match perfectly! ✓`);
|
||||
}
|
||||
log.info(`[QUICK-SEARCH-COMPARISON] ========================================`);
|
||||
} else {
|
||||
results = findResultsWithExpression(expression, searchContext);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user