feat(search): get the correct comparison and rice out the fts5 search
Some checks failed
Checks / main (push) Has been cancelled

This commit is contained in:
perf3ct 2025-10-27 14:37:44 -07:00
parent 253da139de
commit 10988095c2
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
7 changed files with 1453 additions and 103 deletions

View File

@ -11,7 +11,7 @@ import auth from "../services/auth.js";
import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js"; import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js";
import { safeExtractMessageAndStackFromError } from "../services/utils.js"; import { safeExtractMessageAndStackFromError } from "../services/utils.js";
const MAX_ALLOWED_FILE_SIZE_MB = 250; const MAX_ALLOWED_FILE_SIZE_MB = 2500;
export const router = express.Router(); export const router = express.Router();
// TODO: Deduplicate with etapi_utils.ts afterwards. // TODO: Deduplicate with etapi_utils.ts afterwards.

View File

@ -4,7 +4,7 @@ import packageJson from "../../package.json" with { type: "json" };
import dataDir from "./data_dir.js"; import dataDir from "./data_dir.js";
import { AppInfo } from "@triliumnext/commons"; import { AppInfo } from "@triliumnext/commons";
const APP_DB_VERSION = 235; const APP_DB_VERSION = 236;
const SYNC_VERSION = 36; const SYNC_VERSION = 36;
const CLIPPER_PROTOCOL_VERSION = "1.0"; const CLIPPER_PROTOCOL_VERSION = "1.0";

View File

@ -81,30 +81,40 @@ class NoteContentFulltextExp extends Expression {
// Try to use FTS5 if available for better performance // Try to use FTS5 if available for better performance
if (ftsSearchService.checkFTS5Availability() && this.canUseFTS5()) { if (ftsSearchService.checkFTS5Availability() && this.canUseFTS5()) {
try { try {
// Performance comparison logging for FTS5 vs traditional search
const searchQuery = this.tokens.join(" ");
const isQuickSearch = searchContext.fastSearch === false; // quick-search sets fastSearch to false
if (isQuickSearch) {
log.info(`[QUICK-SEARCH-COMPARISON] Starting comparison for query: "${searchQuery}" with operator: ${this.operator}`);
}
// Check if we need to search protected notes // Check if we need to search protected notes
const searchProtected = protectedSessionService.isProtectedSessionAvailable(); const searchProtected = protectedSessionService.isProtectedSessionAvailable();
// Time FTS5 search
const ftsStartTime = Date.now();
const noteIdSet = inputNoteSet.getNoteIds(); const noteIdSet = inputNoteSet.getNoteIds();
const ftsResults = ftsSearchService.searchSync(
// Determine which FTS5 method to use based on operator
let ftsResults;
if (this.operator === "*=*" || this.operator === "*=" || this.operator === "=*") {
// Substring operators use LIKE queries (optimized by trigram index)
// Do NOT pass a limit - we want all results to match traditional search behavior
ftsResults = ftsSearchService.searchWithLike(
this.tokens,
this.operator,
noteIdSet.size > 0 ? noteIdSet : undefined,
{
includeSnippets: false,
searchProtected: false
// No limit specified - return all results
},
searchContext // Pass context to track internal timing
);
} else {
// Other operators use MATCH syntax
ftsResults = ftsSearchService.searchSync(
this.tokens, this.tokens,
this.operator, this.operator,
noteIdSet.size > 0 ? noteIdSet : undefined, noteIdSet.size > 0 ? noteIdSet : undefined,
{ {
includeSnippets: false, includeSnippets: false,
searchProtected: false // FTS5 doesn't index protected notes searchProtected: false // FTS5 doesn't index protected notes
} },
searchContext // Pass context to track internal timing
); );
const ftsEndTime = Date.now(); }
const ftsTime = ftsEndTime - ftsStartTime;
// Add FTS results to note set // Add FTS results to note set
for (const result of ftsResults) { for (const result of ftsResults) {
@ -113,49 +123,6 @@ class NoteContentFulltextExp extends Expression {
} }
} }
// For quick-search, also run traditional search for comparison
if (isQuickSearch) {
const traditionalStartTime = Date.now();
const traditionalNoteSet = new NoteSet();
// Run traditional search (use the fallback method)
const traditionalResults = this.executeWithFallback(inputNoteSet, traditionalNoteSet, searchContext);
const traditionalEndTime = Date.now();
const traditionalTime = traditionalEndTime - traditionalStartTime;
// Log performance comparison
const speedup = traditionalTime > 0 ? (traditionalTime / ftsTime).toFixed(2) : "N/A";
log.info(`[QUICK-SEARCH-COMPARISON] ===== Results for query: "${searchQuery}" =====`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 search: ${ftsTime}ms, found ${ftsResults.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] Traditional search: ${traditionalTime}ms, found ${traditionalResults.notes.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 is ${speedup}x faster (saved ${traditionalTime - ftsTime}ms)`);
// Check if results match
const ftsNoteIds = new Set(ftsResults.map(r => r.noteId));
const traditionalNoteIds = new Set(traditionalResults.notes.map(n => n.noteId));
const matchingResults = ftsNoteIds.size === traditionalNoteIds.size &&
Array.from(ftsNoteIds).every(id => traditionalNoteIds.has(id));
if (!matchingResults) {
log.info(`[QUICK-SEARCH-COMPARISON] Results differ! FTS5: ${ftsNoteIds.size} notes, Traditional: ${traditionalNoteIds.size} notes`);
// Find differences
const onlyInFTS = Array.from(ftsNoteIds).filter(id => !traditionalNoteIds.has(id));
const onlyInTraditional = Array.from(traditionalNoteIds).filter(id => !ftsNoteIds.has(id));
if (onlyInFTS.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in FTS5: ${onlyInFTS.slice(0, 5).join(", ")}${onlyInFTS.length > 5 ? "..." : ""}`);
}
if (onlyInTraditional.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in Traditional: ${onlyInTraditional.slice(0, 5).join(", ")}${onlyInTraditional.length > 5 ? "..." : ""}`);
}
} else {
log.info(`[QUICK-SEARCH-COMPARISON] Results match perfectly! ✓`);
}
log.info(`[QUICK-SEARCH-COMPARISON] ========================================`);
}
// If we need to search protected notes, use the separate method // If we need to search protected notes, use the separate method
if (searchProtected) { if (searchProtected) {
const protectedResults = ftsSearchService.searchProtectedNotesSync( const protectedResults = ftsSearchService.searchProtectedNotesSync(

File diff suppressed because it is too large Load Diff

View File

@ -54,6 +54,7 @@ export interface FTSSearchOptions {
snippetLength?: number; snippetLength?: number;
highlightTag?: string; highlightTag?: string;
searchProtected?: boolean; searchProtected?: boolean;
skipDiagnostics?: boolean; // Skip diagnostic queries for performance measurements
} }
export interface FTSErrorInfo { export interface FTSErrorInfo {
@ -125,6 +126,11 @@ class FTSSearchService {
throw new Error("No search tokens provided"); throw new Error("No search tokens provided");
} }
// Substring operators (*=*, *=, =*) use LIKE queries now, not MATCH
if (operator === "*=*" || operator === "*=" || operator === "=*") {
throw new Error("Substring operators should use searchWithLike(), not MATCH queries");
}
// Trigram tokenizer requires minimum 3 characters // Trigram tokenizer requires minimum 3 characters
const shortTokens = tokens.filter(token => token.length < 3); const shortTokens = tokens.filter(token => token.length < 3);
if (shortTokens.length > 0) { if (shortTokens.length > 0) {
@ -140,33 +146,24 @@ class FTSSearchService {
this.sanitizeFTS5Token(token) this.sanitizeFTS5Token(token)
); );
// Only handle operators that work with MATCH
switch (operator) { switch (operator) {
case "=": // Exact match (phrase search) case "=": // Exact phrase match
return `"${sanitizedTokens.join(" ")}"`; return `"${sanitizedTokens.join(" ")}"`;
case "*=*": // Contains all tokens (AND) case "!=": // Does not contain
return sanitizedTokens.join(" AND ");
case "*=": // Ends with
return sanitizedTokens.map(t => `*${t}`).join(" AND ");
case "=*": // Starts with
return sanitizedTokens.map(t => `${t}*`).join(" AND ");
case "!=": // Does not contain (NOT)
return `NOT (${sanitizedTokens.join(" OR ")})`; return `NOT (${sanitizedTokens.join(" OR ")})`;
case "~=": // Fuzzy match (use OR for more flexible matching) case "~=": // Fuzzy match (use OR)
case "~*": // Fuzzy contains case "~*":
return sanitizedTokens.join(" OR "); return sanitizedTokens.join(" OR ");
case "%=": // Regex match - fallback to OR search case "%=": // Regex - fallback to custom function
log.error(`Regex search operator ${operator} not fully supported in FTS5, using OR search`); log.error(`Regex search operator ${operator} not supported in FTS5`);
return sanitizedTokens.join(" OR "); throw new FTSNotAvailableError("Regex search not supported in FTS5");
default: default:
// Default to AND search throw new FTSQueryError(`Unsupported MATCH operator: ${operator}`);
return sanitizedTokens.join(" AND ");
} }
} }
@ -197,6 +194,249 @@ class FTSSearchService {
return sanitized; return sanitized;
} }
/**
* Escapes LIKE wildcards (% and _) in user input to treat them as literals
* @param str - User input string
* @returns String with LIKE wildcards escaped
*/
private escapeLikeWildcards(str: string): string {
return str.replace(/[%_]/g, '\\$&');
}
/**
* Performs substring search using LIKE queries optimized by trigram index
* This is used for *=*, *=, and =* operators with detail='none'
*
* @param tokens - Search tokens
* @param operator - Search operator (*=*, *=, =*)
* @param noteIds - Optional set of note IDs to filter
* @param options - Search options
* @param searchContext - Optional search context to track internal timing
* @returns Array of search results (noteIds only, no scoring)
*/
searchWithLike(
tokens: string[],
operator: string,
noteIds?: Set<string>,
options: FTSSearchOptions = {},
searchContext?: any
): FTSSearchResult[] {
if (!this.checkFTS5Availability()) {
throw new FTSNotAvailableError();
}
// Normalize tokens to lowercase for case-insensitive search
const normalizedTokens = tokens.map(t => t.toLowerCase());
// Validate token lengths to prevent memory issues
const MAX_TOKEN_LENGTH = 1000;
const longTokens = normalizedTokens.filter(t => t.length > MAX_TOKEN_LENGTH);
if (longTokens.length > 0) {
throw new FTSQueryError(
`Search tokens too long (max ${MAX_TOKEN_LENGTH} characters). ` +
`Long tokens: ${longTokens.map(t => t.substring(0, 50) + '...').join(', ')}`
);
}
const {
limit, // No default limit - return all results
offset = 0,
skipDiagnostics = false
} = options;
// Run diagnostics BEFORE the actual search (not counted in performance timing)
if (!skipDiagnostics) {
log.info('[FTS-DIAGNOSTICS] Running index completeness checks (not counted in search timing)...');
const totalInFts = sql.getValue<number>(`SELECT COUNT(*) FROM notes_fts`);
const totalNotes = sql.getValue<number>(`
SELECT COUNT(*)
FROM notes n
LEFT JOIN blobs b ON n.blobId = b.blobId
WHERE n.type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap')
AND n.isDeleted = 0
AND n.isProtected = 0
AND b.content IS NOT NULL
`);
if (totalInFts < totalNotes) {
log.warn(`[FTS-DIAGNOSTICS] FTS index incomplete: ${totalInFts} indexed out of ${totalNotes} total notes. Run syncMissingNotes().`);
} else {
log.info(`[FTS-DIAGNOSTICS] FTS index complete: ${totalInFts} notes indexed`);
}
}
try {
// Start timing for actual search (excludes diagnostics)
const searchStartTime = Date.now();
// Optimization: If noteIds set is very large, skip filtering to avoid expensive IN clauses
// The FTS table already excludes protected notes, so we can search all notes
const LARGE_SET_THRESHOLD = 1000;
const isLargeNoteSet = noteIds && noteIds.size > LARGE_SET_THRESHOLD;
if (isLargeNoteSet) {
log.info(`[FTS-OPTIMIZATION] Large noteIds set (${noteIds!.size} notes) - skipping IN clause filter, searching all FTS notes`);
}
// Only filter noteIds if the set is small enough to benefit from it
const shouldFilterByNoteIds = noteIds && noteIds.size > 0 && !isLargeNoteSet;
const nonProtectedNoteIds = shouldFilterByNoteIds
? this.filterNonProtectedNoteIds(noteIds)
: [];
let whereConditions: string[] = [];
const params: any[] = [];
// Build LIKE conditions for each token - search BOTH title and content
switch (operator) {
case "*=*": // Contains (substring)
normalizedTokens.forEach(token => {
// Search in BOTH title and content with escaped wildcards
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
const escapedToken = this.escapeLikeWildcards(token);
params.push(`%${escapedToken}%`, `%${escapedToken}%`);
});
break;
case "*=": // Ends with
normalizedTokens.forEach(token => {
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
const escapedToken = this.escapeLikeWildcards(token);
params.push(`%${escapedToken}`, `%${escapedToken}`);
});
break;
case "=*": // Starts with
normalizedTokens.forEach(token => {
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
const escapedToken = this.escapeLikeWildcards(token);
params.push(`${escapedToken}%`, `${escapedToken}%`);
});
break;
default:
throw new FTSQueryError(`Unsupported LIKE operator: ${operator}`);
}
// Validate that we have search criteria
if (whereConditions.length === 0 && nonProtectedNoteIds.length === 0) {
throw new FTSQueryError("No search criteria provided (empty tokens and no note filter)");
}
// SQLite parameter limit handling (999 params max)
const MAX_PARAMS_PER_QUERY = 900; // Leave margin for other params
// Add noteId filter if provided
if (nonProtectedNoteIds.length > 0) {
const tokenParamCount = params.length;
const additionalParams = 2; // For limit and offset
if (nonProtectedNoteIds.length <= MAX_PARAMS_PER_QUERY - tokenParamCount - additionalParams) {
// Normal case: all IDs fit in one query
whereConditions.push(`noteId IN (${nonProtectedNoteIds.map(() => '?').join(',')})`);
params.push(...nonProtectedNoteIds);
} else {
// Large noteIds set: split into chunks and execute multiple queries
const chunks: string[][] = [];
for (let i = 0; i < nonProtectedNoteIds.length; i += MAX_PARAMS_PER_QUERY) {
chunks.push(nonProtectedNoteIds.slice(i, i + MAX_PARAMS_PER_QUERY));
}
log.info(`Large noteIds set detected (${nonProtectedNoteIds.length} notes), splitting into ${chunks.length} chunks`);
// Execute a query for each chunk and combine results
const allResults: FTSSearchResult[] = [];
let remainingLimit = limit !== undefined ? limit : Number.MAX_SAFE_INTEGER;
let currentOffset = offset;
for (const chunk of chunks) {
if (remainingLimit <= 0) break;
const chunkWhereConditions = [...whereConditions];
const chunkParams: any[] = [...params];
chunkWhereConditions.push(`noteId IN (${chunk.map(() => '?').join(',')})`);
chunkParams.push(...chunk);
// Build chunk query
const chunkQuery = `
SELECT noteId, title
FROM notes_fts
WHERE ${chunkWhereConditions.join(' AND ')}
${remainingLimit !== Number.MAX_SAFE_INTEGER ? 'LIMIT ?' : ''}
${currentOffset > 0 ? 'OFFSET ?' : ''}
`;
if (remainingLimit !== Number.MAX_SAFE_INTEGER) chunkParams.push(remainingLimit);
if (currentOffset > 0) chunkParams.push(currentOffset);
const chunkResults = sql.getRows<{ noteId: string; title: string }>(chunkQuery, chunkParams);
allResults.push(...chunkResults.map(row => ({
noteId: row.noteId,
title: row.title,
score: 1.0
})));
if (remainingLimit !== Number.MAX_SAFE_INTEGER) {
remainingLimit -= chunkResults.length;
}
currentOffset = 0; // Only apply offset to first chunk
}
const searchTime = Date.now() - searchStartTime;
log.info(`FTS5 LIKE search (chunked) returned ${allResults.length} results in ${searchTime}ms (excluding diagnostics)`);
// Track internal search time on context for performance comparison
if (searchContext) {
searchContext.ftsInternalSearchTime = searchTime;
}
return allResults;
}
}
// Build query - LIKE queries are automatically optimized by trigram index
// Only add LIMIT/OFFSET if specified
const query = `
SELECT noteId, title
FROM notes_fts
WHERE ${whereConditions.join(' AND ')}
${limit !== undefined ? 'LIMIT ?' : ''}
${offset > 0 ? 'OFFSET ?' : ''}
`;
// Only add limit/offset params if specified
if (limit !== undefined) params.push(limit);
if (offset > 0) params.push(offset);
// Log the search parameters
log.info(`FTS5 LIKE search: tokens=[${normalizedTokens.join(', ')}], operator=${operator}, limit=${limit || 'none'}, offset=${offset}`);
const rows = sql.getRows<{ noteId: string; title: string }>(query, params);
const searchTime = Date.now() - searchStartTime;
log.info(`FTS5 LIKE search returned ${rows.length} results in ${searchTime}ms (excluding diagnostics)`);
// Track internal search time on context for performance comparison
if (searchContext) {
searchContext.ftsInternalSearchTime = searchTime;
}
return rows.map(row => ({
noteId: row.noteId,
title: row.title,
score: 1.0 // LIKE queries don't have ranking
}));
} catch (error: any) {
log.error(`FTS5 LIKE search error: ${error}`);
throw new FTSQueryError(
`FTS5 LIKE search failed: ${error.message}`,
undefined
);
}
}
/** /**
* Performs a synchronous full-text search using FTS5 * Performs a synchronous full-text search using FTS5
* *
@ -204,13 +444,15 @@ class FTSSearchService {
* @param operator - Search operator * @param operator - Search operator
* @param noteIds - Optional set of note IDs to search within * @param noteIds - Optional set of note IDs to search within
* @param options - Search options * @param options - Search options
* @param searchContext - Optional search context to track internal timing
* @returns Array of search results * @returns Array of search results
*/ */
searchSync( searchSync(
tokens: string[], tokens: string[],
operator: string, operator: string,
noteIds?: Set<string>, noteIds?: Set<string>,
options: FTSSearchOptions = {} options: FTSSearchOptions = {},
searchContext?: any
): FTSSearchResult[] { ): FTSSearchResult[] {
if (!this.checkFTS5Availability()) { if (!this.checkFTS5Availability()) {
throw new FTSNotAvailableError(); throw new FTSNotAvailableError();
@ -226,6 +468,9 @@ class FTSSearchService {
} = options; } = options;
try { try {
// Start timing for actual search
const searchStartTime = Date.now();
const ftsQuery = this.convertToFTS5Query(tokens, operator); const ftsQuery = this.convertToFTS5Query(tokens, operator);
// Validate query length // Validate query length
@ -249,10 +494,20 @@ class FTSSearchService {
let whereConditions = [`notes_fts MATCH ?`]; let whereConditions = [`notes_fts MATCH ?`];
const params: any[] = [ftsQuery]; const params: any[] = [ftsQuery];
// Filter by noteIds if provided // Optimization: If noteIds set is very large, skip filtering to avoid expensive IN clauses
if (noteIds && noteIds.size > 0) { // The FTS table already excludes protected notes, so we can search all notes
const LARGE_SET_THRESHOLD = 1000;
const isLargeNoteSet = noteIds && noteIds.size > LARGE_SET_THRESHOLD;
if (isLargeNoteSet) {
log.info(`[FTS-OPTIMIZATION] Large noteIds set (${noteIds!.size} notes) - skipping IN clause filter, searching all FTS notes`);
}
// Filter by noteIds if provided and set is small enough
const shouldFilterByNoteIds = noteIds && noteIds.size > 0 && !isLargeNoteSet;
if (shouldFilterByNoteIds) {
// First filter out any protected notes from the noteIds // First filter out any protected notes from the noteIds
const nonProtectedNoteIds = this.filterNonProtectedNoteIds(noteIds); const nonProtectedNoteIds = this.filterNonProtectedNoteIds(noteIds!);
if (nonProtectedNoteIds.length === 0) { if (nonProtectedNoteIds.length === 0) {
// All provided notes are protected, return empty results // All provided notes are protected, return empty results
return []; return [];
@ -287,6 +542,14 @@ class FTSSearchService {
snippet?: string; snippet?: string;
}>(query, params); }>(query, params);
const searchTime = Date.now() - searchStartTime;
log.info(`FTS5 MATCH search returned ${results.length} results in ${searchTime}ms`);
// Track internal search time on context for performance comparison
if (searchContext) {
searchContext.ftsInternalSearchTime = searchTime;
}
return results; return results;
} catch (error: any) { } catch (error: any) {

View File

@ -24,6 +24,7 @@ class SearchContext {
fulltextQuery: string; fulltextQuery: string;
dbLoadNeeded: boolean; dbLoadNeeded: boolean;
error: string | null; error: string | null;
ftsInternalSearchTime: number | null; // Time spent in actual FTS search (excluding diagnostics)
constructor(params: SearchParams = {}) { constructor(params: SearchParams = {}) {
this.fastSearch = !!params.fastSearch; this.fastSearch = !!params.fastSearch;
@ -54,6 +55,7 @@ class SearchContext {
// and some extra data needs to be loaded before executing // and some extra data needs to be loaded before executing
this.dbLoadNeeded = false; this.dbLoadNeeded = false;
this.error = null; this.error = null;
this.ftsInternalSearchTime = null;
} }
addError(error: string) { addError(error: string) {

View File

@ -19,6 +19,7 @@ import sql from "../../sql.js";
import scriptService from "../../script.js"; import scriptService from "../../script.js";
import striptags from "striptags"; import striptags from "striptags";
import protectedSessionService from "../../protected_session.js"; import protectedSessionService from "../../protected_session.js";
import ftsSearchService from "../fts_search.js";
export interface SearchNoteResult { export interface SearchNoteResult {
searchResultNoteIds: string[]; searchResultNoteIds: string[];
@ -422,13 +423,83 @@ function findResultsWithQuery(query: string, searchContext: SearchContext): Sear
// ordering or other logic that shouldn't be interfered with. // ordering or other logic that shouldn't be interfered with.
const isPureExpressionQuery = query.trim().startsWith('#'); const isPureExpressionQuery = query.trim().startsWith('#');
// Performance comparison for quick-search (fastSearch === false)
const isQuickSearch = searchContext.fastSearch === false;
let results: SearchResult[]; let results: SearchResult[];
let ftsTime = 0;
let traditionalTime = 0;
if (isPureExpressionQuery) { if (isPureExpressionQuery) {
// For pure expression queries, use standard search without progressive phases // For pure expression queries, use standard search without progressive phases
results = performSearch(expression, searchContext, searchContext.enableFuzzyMatching); results = performSearch(expression, searchContext, searchContext.enableFuzzyMatching);
} else { } else {
// For quick-search, run both FTS5 and traditional search to compare
if (isQuickSearch) {
log.info(`[QUICK-SEARCH-COMPARISON] Starting comparison for query: "${query}"`);
// Time FTS5 search (normal path)
const ftsStartTime = Date.now();
results = findResultsWithExpression(expression, searchContext); results = findResultsWithExpression(expression, searchContext);
ftsTime = Date.now() - ftsStartTime;
// Time traditional search (with FTS5 disabled)
const traditionalStartTime = Date.now();
// Create a new search context with FTS5 disabled
const traditionalContext = new SearchContext({
fastSearch: false,
includeArchivedNotes: false,
includeHiddenNotes: true,
fuzzyAttributeSearch: true,
ignoreInternalAttributes: true,
ancestorNoteId: searchContext.ancestorNoteId
});
// Temporarily disable FTS5 to force traditional search
const originalFtsAvailable = (ftsSearchService as any).isFTS5Available;
(ftsSearchService as any).isFTS5Available = false;
const traditionalResults = findResultsWithExpression(expression, traditionalContext);
traditionalTime = Date.now() - traditionalStartTime;
// Restore FTS5 availability
(ftsSearchService as any).isFTS5Available = originalFtsAvailable;
// Log performance comparison
// Use internal FTS search time (excluding diagnostics) if available
const ftsInternalTime = searchContext.ftsInternalSearchTime ?? ftsTime;
const speedup = traditionalTime > 0 ? (traditionalTime / ftsInternalTime).toFixed(2) : "N/A";
log.info(`[QUICK-SEARCH-COMPARISON] ===== Results for query: "${query}" =====`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 search: ${ftsInternalTime}ms (excluding diagnostics), found ${results.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] Traditional search: ${traditionalTime}ms, found ${traditionalResults.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 is ${speedup}x faster (saved ${traditionalTime - ftsInternalTime}ms)`);
// Check if results match
const ftsNoteIds = new Set(results.map(r => r.noteId));
const traditionalNoteIds = new Set(traditionalResults.map(r => r.noteId));
const matchingResults = ftsNoteIds.size === traditionalNoteIds.size &&
Array.from(ftsNoteIds).every(id => traditionalNoteIds.has(id));
if (!matchingResults) {
log.info(`[QUICK-SEARCH-COMPARISON] Results differ! FTS5: ${ftsNoteIds.size} notes, Traditional: ${traditionalNoteIds.size} notes`);
// Find differences
const onlyInFTS = Array.from(ftsNoteIds).filter(id => !traditionalNoteIds.has(id));
const onlyInTraditional = Array.from(traditionalNoteIds).filter(id => !ftsNoteIds.has(id));
if (onlyInFTS.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in FTS5: ${onlyInFTS.slice(0, 5).join(", ")}${onlyInFTS.length > 5 ? "..." : ""}`);
}
if (onlyInTraditional.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in Traditional: ${onlyInTraditional.slice(0, 5).join(", ")}${onlyInTraditional.length > 5 ? "..." : ""}`);
}
} else {
log.info(`[QUICK-SEARCH-COMPARISON] Results match perfectly! ✓`);
}
log.info(`[QUICK-SEARCH-COMPARISON] ========================================`);
} else {
results = findResultsWithExpression(expression, searchContext);
}
} }
return results; return results;