feat(search): get the correct comparison and rice out the fts5 search
Some checks failed
Checks / main (push) Has been cancelled

This commit is contained in:
perf3ct 2025-10-27 14:37:44 -07:00
parent 253da139de
commit 10988095c2
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
7 changed files with 1453 additions and 103 deletions

View File

@ -11,7 +11,7 @@ import auth from "../services/auth.js";
import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js"; import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js";
import { safeExtractMessageAndStackFromError } from "../services/utils.js"; import { safeExtractMessageAndStackFromError } from "../services/utils.js";
const MAX_ALLOWED_FILE_SIZE_MB = 250; const MAX_ALLOWED_FILE_SIZE_MB = 2500;
export const router = express.Router(); export const router = express.Router();
// TODO: Deduplicate with etapi_utils.ts afterwards. // TODO: Deduplicate with etapi_utils.ts afterwards.

View File

@ -4,7 +4,7 @@ import packageJson from "../../package.json" with { type: "json" };
import dataDir from "./data_dir.js"; import dataDir from "./data_dir.js";
import { AppInfo } from "@triliumnext/commons"; import { AppInfo } from "@triliumnext/commons";
const APP_DB_VERSION = 235; const APP_DB_VERSION = 236;
const SYNC_VERSION = 36; const SYNC_VERSION = 36;
const CLIPPER_PROTOCOL_VERSION = "1.0"; const CLIPPER_PROTOCOL_VERSION = "1.0";

View File

@ -81,30 +81,40 @@ class NoteContentFulltextExp extends Expression {
// Try to use FTS5 if available for better performance // Try to use FTS5 if available for better performance
if (ftsSearchService.checkFTS5Availability() && this.canUseFTS5()) { if (ftsSearchService.checkFTS5Availability() && this.canUseFTS5()) {
try { try {
// Performance comparison logging for FTS5 vs traditional search
const searchQuery = this.tokens.join(" ");
const isQuickSearch = searchContext.fastSearch === false; // quick-search sets fastSearch to false
if (isQuickSearch) {
log.info(`[QUICK-SEARCH-COMPARISON] Starting comparison for query: "${searchQuery}" with operator: ${this.operator}`);
}
// Check if we need to search protected notes // Check if we need to search protected notes
const searchProtected = protectedSessionService.isProtectedSessionAvailable(); const searchProtected = protectedSessionService.isProtectedSessionAvailable();
// Time FTS5 search
const ftsStartTime = Date.now();
const noteIdSet = inputNoteSet.getNoteIds(); const noteIdSet = inputNoteSet.getNoteIds();
const ftsResults = ftsSearchService.searchSync(
this.tokens, // Determine which FTS5 method to use based on operator
this.operator, let ftsResults;
noteIdSet.size > 0 ? noteIdSet : undefined, if (this.operator === "*=*" || this.operator === "*=" || this.operator === "=*") {
{ // Substring operators use LIKE queries (optimized by trigram index)
includeSnippets: false, // Do NOT pass a limit - we want all results to match traditional search behavior
searchProtected: false // FTS5 doesn't index protected notes ftsResults = ftsSearchService.searchWithLike(
} this.tokens,
); this.operator,
const ftsEndTime = Date.now(); noteIdSet.size > 0 ? noteIdSet : undefined,
const ftsTime = ftsEndTime - ftsStartTime; {
includeSnippets: false,
searchProtected: false
// No limit specified - return all results
},
searchContext // Pass context to track internal timing
);
} else {
// Other operators use MATCH syntax
ftsResults = ftsSearchService.searchSync(
this.tokens,
this.operator,
noteIdSet.size > 0 ? noteIdSet : undefined,
{
includeSnippets: false,
searchProtected: false // FTS5 doesn't index protected notes
},
searchContext // Pass context to track internal timing
);
}
// Add FTS results to note set // Add FTS results to note set
for (const result of ftsResults) { for (const result of ftsResults) {
@ -112,50 +122,7 @@ class NoteContentFulltextExp extends Expression {
resultNoteSet.add(becca.notes[result.noteId]); resultNoteSet.add(becca.notes[result.noteId]);
} }
} }
// For quick-search, also run traditional search for comparison
if (isQuickSearch) {
const traditionalStartTime = Date.now();
const traditionalNoteSet = new NoteSet();
// Run traditional search (use the fallback method)
const traditionalResults = this.executeWithFallback(inputNoteSet, traditionalNoteSet, searchContext);
const traditionalEndTime = Date.now();
const traditionalTime = traditionalEndTime - traditionalStartTime;
// Log performance comparison
const speedup = traditionalTime > 0 ? (traditionalTime / ftsTime).toFixed(2) : "N/A";
log.info(`[QUICK-SEARCH-COMPARISON] ===== Results for query: "${searchQuery}" =====`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 search: ${ftsTime}ms, found ${ftsResults.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] Traditional search: ${traditionalTime}ms, found ${traditionalResults.notes.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 is ${speedup}x faster (saved ${traditionalTime - ftsTime}ms)`);
// Check if results match
const ftsNoteIds = new Set(ftsResults.map(r => r.noteId));
const traditionalNoteIds = new Set(traditionalResults.notes.map(n => n.noteId));
const matchingResults = ftsNoteIds.size === traditionalNoteIds.size &&
Array.from(ftsNoteIds).every(id => traditionalNoteIds.has(id));
if (!matchingResults) {
log.info(`[QUICK-SEARCH-COMPARISON] Results differ! FTS5: ${ftsNoteIds.size} notes, Traditional: ${traditionalNoteIds.size} notes`);
// Find differences
const onlyInFTS = Array.from(ftsNoteIds).filter(id => !traditionalNoteIds.has(id));
const onlyInTraditional = Array.from(traditionalNoteIds).filter(id => !ftsNoteIds.has(id));
if (onlyInFTS.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in FTS5: ${onlyInFTS.slice(0, 5).join(", ")}${onlyInFTS.length > 5 ? "..." : ""}`);
}
if (onlyInTraditional.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in Traditional: ${onlyInTraditional.slice(0, 5).join(", ")}${onlyInTraditional.length > 5 ? "..." : ""}`);
}
} else {
log.info(`[QUICK-SEARCH-COMPARISON] Results match perfectly! ✓`);
}
log.info(`[QUICK-SEARCH-COMPARISON] ========================================`);
}
// If we need to search protected notes, use the separate method // If we need to search protected notes, use the separate method
if (searchProtected) { if (searchProtected) {
const protectedResults = ftsSearchService.searchProtectedNotesSync( const protectedResults = ftsSearchService.searchProtectedNotesSync(
@ -166,7 +133,7 @@ class NoteContentFulltextExp extends Expression {
includeSnippets: false includeSnippets: false
} }
); );
// Add protected note results // Add protected note results
for (const result of protectedResults) { for (const result of protectedResults) {
if (becca.notes[result.noteId]) { if (becca.notes[result.noteId]) {
@ -193,7 +160,7 @@ class NoteContentFulltextExp extends Expression {
} else { } else {
log.error(`FTS5 error: ${error}`); log.error(`FTS5 error: ${error}`);
} }
// Use fallback for recoverable errors // Use fallback for recoverable errors
if (error.recoverable) { if (error.recoverable) {
log.info("Using fallback search implementation"); log.info("Using fallback search implementation");
@ -213,8 +180,8 @@ class NoteContentFulltextExp extends Expression {
for (const row of sql.iterateRows<SearchRow>(` for (const row of sql.iterateRows<SearchRow>(`
SELECT noteId, type, mime, content, isProtected SELECT noteId, type, mime, content, isProtected
FROM notes JOIN blobs USING (blobId) FROM notes JOIN blobs USING (blobId)
WHERE type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap') WHERE type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap')
AND isDeleted = 0 AND isDeleted = 0
AND LENGTH(content) < ${MAX_SEARCH_CONTENT_SIZE}`)) { AND LENGTH(content) < ${MAX_SEARCH_CONTENT_SIZE}`)) {
this.findInText(row, inputNoteSet, resultNoteSet); this.findInText(row, inputNoteSet, resultNoteSet);
} }

File diff suppressed because it is too large Load Diff

View File

@ -54,6 +54,7 @@ export interface FTSSearchOptions {
snippetLength?: number; snippetLength?: number;
highlightTag?: string; highlightTag?: string;
searchProtected?: boolean; searchProtected?: boolean;
skipDiagnostics?: boolean; // Skip diagnostic queries for performance measurements
} }
export interface FTSErrorInfo { export interface FTSErrorInfo {
@ -125,6 +126,11 @@ class FTSSearchService {
throw new Error("No search tokens provided"); throw new Error("No search tokens provided");
} }
// Substring operators (*=*, *=, =*) use LIKE queries now, not MATCH
if (operator === "*=*" || operator === "*=" || operator === "=*") {
throw new Error("Substring operators should use searchWithLike(), not MATCH queries");
}
// Trigram tokenizer requires minimum 3 characters // Trigram tokenizer requires minimum 3 characters
const shortTokens = tokens.filter(token => token.length < 3); const shortTokens = tokens.filter(token => token.length < 3);
if (shortTokens.length > 0) { if (shortTokens.length > 0) {
@ -140,33 +146,24 @@ class FTSSearchService {
this.sanitizeFTS5Token(token) this.sanitizeFTS5Token(token)
); );
// Only handle operators that work with MATCH
switch (operator) { switch (operator) {
case "=": // Exact match (phrase search) case "=": // Exact phrase match
return `"${sanitizedTokens.join(" ")}"`; return `"${sanitizedTokens.join(" ")}"`;
case "*=*": // Contains all tokens (AND) case "!=": // Does not contain
return sanitizedTokens.join(" AND ");
case "*=": // Ends with
return sanitizedTokens.map(t => `*${t}`).join(" AND ");
case "=*": // Starts with
return sanitizedTokens.map(t => `${t}*`).join(" AND ");
case "!=": // Does not contain (NOT)
return `NOT (${sanitizedTokens.join(" OR ")})`; return `NOT (${sanitizedTokens.join(" OR ")})`;
case "~=": // Fuzzy match (use OR for more flexible matching) case "~=": // Fuzzy match (use OR)
case "~*": // Fuzzy contains case "~*":
return sanitizedTokens.join(" OR "); return sanitizedTokens.join(" OR ");
case "%=": // Regex match - fallback to OR search case "%=": // Regex - fallback to custom function
log.error(`Regex search operator ${operator} not fully supported in FTS5, using OR search`); log.error(`Regex search operator ${operator} not supported in FTS5`);
return sanitizedTokens.join(" OR "); throw new FTSNotAvailableError("Regex search not supported in FTS5");
default: default:
// Default to AND search throw new FTSQueryError(`Unsupported MATCH operator: ${operator}`);
return sanitizedTokens.join(" AND ");
} }
} }
@ -180,37 +177,282 @@ class FTSSearchService {
.replace(/["\(\)\*]/g, '') // Remove quotes, parens, wildcards .replace(/["\(\)\*]/g, '') // Remove quotes, parens, wildcards
.replace(/\s+/g, ' ') // Normalize whitespace .replace(/\s+/g, ' ') // Normalize whitespace
.trim(); .trim();
// Validate that token is not empty after sanitization // Validate that token is not empty after sanitization
if (!sanitized || sanitized.length === 0) { if (!sanitized || sanitized.length === 0) {
log.info(`Token became empty after sanitization: "${token}"`); log.info(`Token became empty after sanitization: "${token}"`);
// Return a safe placeholder that won't match anything // Return a safe placeholder that won't match anything
return "__empty_token__"; return "__empty_token__";
} }
// Additional validation: ensure token doesn't contain SQL injection attempts // Additional validation: ensure token doesn't contain SQL injection attempts
if (sanitized.includes(';') || sanitized.includes('--')) { if (sanitized.includes(';') || sanitized.includes('--')) {
log.error(`Potential SQL injection attempt detected in token: "${token}"`); log.error(`Potential SQL injection attempt detected in token: "${token}"`);
return "__invalid_token__"; return "__invalid_token__";
} }
return sanitized; return sanitized;
} }
/**
* Escapes LIKE wildcards (% and _) in user input to treat them as literals
* @param str - User input string
* @returns String with LIKE wildcards escaped
*/
private escapeLikeWildcards(str: string): string {
return str.replace(/[%_]/g, '\\$&');
}
/**
* Performs substring search using LIKE queries optimized by trigram index
* This is used for *=*, *=, and =* operators with detail='none'
*
* @param tokens - Search tokens
* @param operator - Search operator (*=*, *=, =*)
* @param noteIds - Optional set of note IDs to filter
* @param options - Search options
* @param searchContext - Optional search context to track internal timing
* @returns Array of search results (noteIds only, no scoring)
*/
searchWithLike(
tokens: string[],
operator: string,
noteIds?: Set<string>,
options: FTSSearchOptions = {},
searchContext?: any
): FTSSearchResult[] {
if (!this.checkFTS5Availability()) {
throw new FTSNotAvailableError();
}
// Normalize tokens to lowercase for case-insensitive search
const normalizedTokens = tokens.map(t => t.toLowerCase());
// Validate token lengths to prevent memory issues
const MAX_TOKEN_LENGTH = 1000;
const longTokens = normalizedTokens.filter(t => t.length > MAX_TOKEN_LENGTH);
if (longTokens.length > 0) {
throw new FTSQueryError(
`Search tokens too long (max ${MAX_TOKEN_LENGTH} characters). ` +
`Long tokens: ${longTokens.map(t => t.substring(0, 50) + '...').join(', ')}`
);
}
const {
limit, // No default limit - return all results
offset = 0,
skipDiagnostics = false
} = options;
// Run diagnostics BEFORE the actual search (not counted in performance timing)
if (!skipDiagnostics) {
log.info('[FTS-DIAGNOSTICS] Running index completeness checks (not counted in search timing)...');
const totalInFts = sql.getValue<number>(`SELECT COUNT(*) FROM notes_fts`);
const totalNotes = sql.getValue<number>(`
SELECT COUNT(*)
FROM notes n
LEFT JOIN blobs b ON n.blobId = b.blobId
WHERE n.type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap')
AND n.isDeleted = 0
AND n.isProtected = 0
AND b.content IS NOT NULL
`);
if (totalInFts < totalNotes) {
log.warn(`[FTS-DIAGNOSTICS] FTS index incomplete: ${totalInFts} indexed out of ${totalNotes} total notes. Run syncMissingNotes().`);
} else {
log.info(`[FTS-DIAGNOSTICS] FTS index complete: ${totalInFts} notes indexed`);
}
}
try {
// Start timing for actual search (excludes diagnostics)
const searchStartTime = Date.now();
// Optimization: If noteIds set is very large, skip filtering to avoid expensive IN clauses
// The FTS table already excludes protected notes, so we can search all notes
const LARGE_SET_THRESHOLD = 1000;
const isLargeNoteSet = noteIds && noteIds.size > LARGE_SET_THRESHOLD;
if (isLargeNoteSet) {
log.info(`[FTS-OPTIMIZATION] Large noteIds set (${noteIds!.size} notes) - skipping IN clause filter, searching all FTS notes`);
}
// Only filter noteIds if the set is small enough to benefit from it
const shouldFilterByNoteIds = noteIds && noteIds.size > 0 && !isLargeNoteSet;
const nonProtectedNoteIds = shouldFilterByNoteIds
? this.filterNonProtectedNoteIds(noteIds)
: [];
let whereConditions: string[] = [];
const params: any[] = [];
// Build LIKE conditions for each token - search BOTH title and content
switch (operator) {
case "*=*": // Contains (substring)
normalizedTokens.forEach(token => {
// Search in BOTH title and content with escaped wildcards
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
const escapedToken = this.escapeLikeWildcards(token);
params.push(`%${escapedToken}%`, `%${escapedToken}%`);
});
break;
case "*=": // Ends with
normalizedTokens.forEach(token => {
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
const escapedToken = this.escapeLikeWildcards(token);
params.push(`%${escapedToken}`, `%${escapedToken}`);
});
break;
case "=*": // Starts with
normalizedTokens.forEach(token => {
whereConditions.push(`(title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\')`);
const escapedToken = this.escapeLikeWildcards(token);
params.push(`${escapedToken}%`, `${escapedToken}%`);
});
break;
default:
throw new FTSQueryError(`Unsupported LIKE operator: ${operator}`);
}
// Validate that we have search criteria
if (whereConditions.length === 0 && nonProtectedNoteIds.length === 0) {
throw new FTSQueryError("No search criteria provided (empty tokens and no note filter)");
}
// SQLite parameter limit handling (999 params max)
const MAX_PARAMS_PER_QUERY = 900; // Leave margin for other params
// Add noteId filter if provided
if (nonProtectedNoteIds.length > 0) {
const tokenParamCount = params.length;
const additionalParams = 2; // For limit and offset
if (nonProtectedNoteIds.length <= MAX_PARAMS_PER_QUERY - tokenParamCount - additionalParams) {
// Normal case: all IDs fit in one query
whereConditions.push(`noteId IN (${nonProtectedNoteIds.map(() => '?').join(',')})`);
params.push(...nonProtectedNoteIds);
} else {
// Large noteIds set: split into chunks and execute multiple queries
const chunks: string[][] = [];
for (let i = 0; i < nonProtectedNoteIds.length; i += MAX_PARAMS_PER_QUERY) {
chunks.push(nonProtectedNoteIds.slice(i, i + MAX_PARAMS_PER_QUERY));
}
log.info(`Large noteIds set detected (${nonProtectedNoteIds.length} notes), splitting into ${chunks.length} chunks`);
// Execute a query for each chunk and combine results
const allResults: FTSSearchResult[] = [];
let remainingLimit = limit !== undefined ? limit : Number.MAX_SAFE_INTEGER;
let currentOffset = offset;
for (const chunk of chunks) {
if (remainingLimit <= 0) break;
const chunkWhereConditions = [...whereConditions];
const chunkParams: any[] = [...params];
chunkWhereConditions.push(`noteId IN (${chunk.map(() => '?').join(',')})`);
chunkParams.push(...chunk);
// Build chunk query
const chunkQuery = `
SELECT noteId, title
FROM notes_fts
WHERE ${chunkWhereConditions.join(' AND ')}
${remainingLimit !== Number.MAX_SAFE_INTEGER ? 'LIMIT ?' : ''}
${currentOffset > 0 ? 'OFFSET ?' : ''}
`;
if (remainingLimit !== Number.MAX_SAFE_INTEGER) chunkParams.push(remainingLimit);
if (currentOffset > 0) chunkParams.push(currentOffset);
const chunkResults = sql.getRows<{ noteId: string; title: string }>(chunkQuery, chunkParams);
allResults.push(...chunkResults.map(row => ({
noteId: row.noteId,
title: row.title,
score: 1.0
})));
if (remainingLimit !== Number.MAX_SAFE_INTEGER) {
remainingLimit -= chunkResults.length;
}
currentOffset = 0; // Only apply offset to first chunk
}
const searchTime = Date.now() - searchStartTime;
log.info(`FTS5 LIKE search (chunked) returned ${allResults.length} results in ${searchTime}ms (excluding diagnostics)`);
// Track internal search time on context for performance comparison
if (searchContext) {
searchContext.ftsInternalSearchTime = searchTime;
}
return allResults;
}
}
// Build query - LIKE queries are automatically optimized by trigram index
// Only add LIMIT/OFFSET if specified
const query = `
SELECT noteId, title
FROM notes_fts
WHERE ${whereConditions.join(' AND ')}
${limit !== undefined ? 'LIMIT ?' : ''}
${offset > 0 ? 'OFFSET ?' : ''}
`;
// Only add limit/offset params if specified
if (limit !== undefined) params.push(limit);
if (offset > 0) params.push(offset);
// Log the search parameters
log.info(`FTS5 LIKE search: tokens=[${normalizedTokens.join(', ')}], operator=${operator}, limit=${limit || 'none'}, offset=${offset}`);
const rows = sql.getRows<{ noteId: string; title: string }>(query, params);
const searchTime = Date.now() - searchStartTime;
log.info(`FTS5 LIKE search returned ${rows.length} results in ${searchTime}ms (excluding diagnostics)`);
// Track internal search time on context for performance comparison
if (searchContext) {
searchContext.ftsInternalSearchTime = searchTime;
}
return rows.map(row => ({
noteId: row.noteId,
title: row.title,
score: 1.0 // LIKE queries don't have ranking
}));
} catch (error: any) {
log.error(`FTS5 LIKE search error: ${error}`);
throw new FTSQueryError(
`FTS5 LIKE search failed: ${error.message}`,
undefined
);
}
}
/** /**
* Performs a synchronous full-text search using FTS5 * Performs a synchronous full-text search using FTS5
* *
* @param tokens - Search tokens * @param tokens - Search tokens
* @param operator - Search operator * @param operator - Search operator
* @param noteIds - Optional set of note IDs to search within * @param noteIds - Optional set of note IDs to search within
* @param options - Search options * @param options - Search options
* @param searchContext - Optional search context to track internal timing
* @returns Array of search results * @returns Array of search results
*/ */
searchSync( searchSync(
tokens: string[], tokens: string[],
operator: string, operator: string,
noteIds?: Set<string>, noteIds?: Set<string>,
options: FTSSearchOptions = {} options: FTSSearchOptions = {},
searchContext?: any
): FTSSearchResult[] { ): FTSSearchResult[] {
if (!this.checkFTS5Availability()) { if (!this.checkFTS5Availability()) {
throw new FTSNotAvailableError(); throw new FTSNotAvailableError();
@ -226,6 +468,9 @@ class FTSSearchService {
} = options; } = options;
try { try {
// Start timing for actual search
const searchStartTime = Date.now();
const ftsQuery = this.convertToFTS5Query(tokens, operator); const ftsQuery = this.convertToFTS5Query(tokens, operator);
// Validate query length // Validate query length
@ -249,10 +494,20 @@ class FTSSearchService {
let whereConditions = [`notes_fts MATCH ?`]; let whereConditions = [`notes_fts MATCH ?`];
const params: any[] = [ftsQuery]; const params: any[] = [ftsQuery];
// Filter by noteIds if provided // Optimization: If noteIds set is very large, skip filtering to avoid expensive IN clauses
if (noteIds && noteIds.size > 0) { // The FTS table already excludes protected notes, so we can search all notes
const LARGE_SET_THRESHOLD = 1000;
const isLargeNoteSet = noteIds && noteIds.size > LARGE_SET_THRESHOLD;
if (isLargeNoteSet) {
log.info(`[FTS-OPTIMIZATION] Large noteIds set (${noteIds!.size} notes) - skipping IN clause filter, searching all FTS notes`);
}
// Filter by noteIds if provided and set is small enough
const shouldFilterByNoteIds = noteIds && noteIds.size > 0 && !isLargeNoteSet;
if (shouldFilterByNoteIds) {
// First filter out any protected notes from the noteIds // First filter out any protected notes from the noteIds
const nonProtectedNoteIds = this.filterNonProtectedNoteIds(noteIds); const nonProtectedNoteIds = this.filterNonProtectedNoteIds(noteIds!);
if (nonProtectedNoteIds.length === 0) { if (nonProtectedNoteIds.length === 0) {
// All provided notes are protected, return empty results // All provided notes are protected, return empty results
return []; return [];
@ -287,6 +542,14 @@ class FTSSearchService {
snippet?: string; snippet?: string;
}>(query, params); }>(query, params);
const searchTime = Date.now() - searchStartTime;
log.info(`FTS5 MATCH search returned ${results.length} results in ${searchTime}ms`);
// Track internal search time on context for performance comparison
if (searchContext) {
searchContext.ftsInternalSearchTime = searchTime;
}
return results; return results;
} catch (error: any) { } catch (error: any) {

View File

@ -24,6 +24,7 @@ class SearchContext {
fulltextQuery: string; fulltextQuery: string;
dbLoadNeeded: boolean; dbLoadNeeded: boolean;
error: string | null; error: string | null;
ftsInternalSearchTime: number | null; // Time spent in actual FTS search (excluding diagnostics)
constructor(params: SearchParams = {}) { constructor(params: SearchParams = {}) {
this.fastSearch = !!params.fastSearch; this.fastSearch = !!params.fastSearch;
@ -54,6 +55,7 @@ class SearchContext {
// and some extra data needs to be loaded before executing // and some extra data needs to be loaded before executing
this.dbLoadNeeded = false; this.dbLoadNeeded = false;
this.error = null; this.error = null;
this.ftsInternalSearchTime = null;
} }
addError(error: string) { addError(error: string) {

View File

@ -19,6 +19,7 @@ import sql from "../../sql.js";
import scriptService from "../../script.js"; import scriptService from "../../script.js";
import striptags from "striptags"; import striptags from "striptags";
import protectedSessionService from "../../protected_session.js"; import protectedSessionService from "../../protected_session.js";
import ftsSearchService from "../fts_search.js";
export interface SearchNoteResult { export interface SearchNoteResult {
searchResultNoteIds: string[]; searchResultNoteIds: string[];
@ -422,13 +423,83 @@ function findResultsWithQuery(query: string, searchContext: SearchContext): Sear
// ordering or other logic that shouldn't be interfered with. // ordering or other logic that shouldn't be interfered with.
const isPureExpressionQuery = query.trim().startsWith('#'); const isPureExpressionQuery = query.trim().startsWith('#');
// Performance comparison for quick-search (fastSearch === false)
const isQuickSearch = searchContext.fastSearch === false;
let results: SearchResult[]; let results: SearchResult[];
let ftsTime = 0;
let traditionalTime = 0;
if (isPureExpressionQuery) { if (isPureExpressionQuery) {
// For pure expression queries, use standard search without progressive phases // For pure expression queries, use standard search without progressive phases
results = performSearch(expression, searchContext, searchContext.enableFuzzyMatching); results = performSearch(expression, searchContext, searchContext.enableFuzzyMatching);
} else { } else {
results = findResultsWithExpression(expression, searchContext); // For quick-search, run both FTS5 and traditional search to compare
if (isQuickSearch) {
log.info(`[QUICK-SEARCH-COMPARISON] Starting comparison for query: "${query}"`);
// Time FTS5 search (normal path)
const ftsStartTime = Date.now();
results = findResultsWithExpression(expression, searchContext);
ftsTime = Date.now() - ftsStartTime;
// Time traditional search (with FTS5 disabled)
const traditionalStartTime = Date.now();
// Create a new search context with FTS5 disabled
const traditionalContext = new SearchContext({
fastSearch: false,
includeArchivedNotes: false,
includeHiddenNotes: true,
fuzzyAttributeSearch: true,
ignoreInternalAttributes: true,
ancestorNoteId: searchContext.ancestorNoteId
});
// Temporarily disable FTS5 to force traditional search
const originalFtsAvailable = (ftsSearchService as any).isFTS5Available;
(ftsSearchService as any).isFTS5Available = false;
const traditionalResults = findResultsWithExpression(expression, traditionalContext);
traditionalTime = Date.now() - traditionalStartTime;
// Restore FTS5 availability
(ftsSearchService as any).isFTS5Available = originalFtsAvailable;
// Log performance comparison
// Use internal FTS search time (excluding diagnostics) if available
const ftsInternalTime = searchContext.ftsInternalSearchTime ?? ftsTime;
const speedup = traditionalTime > 0 ? (traditionalTime / ftsInternalTime).toFixed(2) : "N/A";
log.info(`[QUICK-SEARCH-COMPARISON] ===== Results for query: "${query}" =====`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 search: ${ftsInternalTime}ms (excluding diagnostics), found ${results.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] Traditional search: ${traditionalTime}ms, found ${traditionalResults.length} results`);
log.info(`[QUICK-SEARCH-COMPARISON] FTS5 is ${speedup}x faster (saved ${traditionalTime - ftsInternalTime}ms)`);
// Check if results match
const ftsNoteIds = new Set(results.map(r => r.noteId));
const traditionalNoteIds = new Set(traditionalResults.map(r => r.noteId));
const matchingResults = ftsNoteIds.size === traditionalNoteIds.size &&
Array.from(ftsNoteIds).every(id => traditionalNoteIds.has(id));
if (!matchingResults) {
log.info(`[QUICK-SEARCH-COMPARISON] Results differ! FTS5: ${ftsNoteIds.size} notes, Traditional: ${traditionalNoteIds.size} notes`);
// Find differences
const onlyInFTS = Array.from(ftsNoteIds).filter(id => !traditionalNoteIds.has(id));
const onlyInTraditional = Array.from(traditionalNoteIds).filter(id => !ftsNoteIds.has(id));
if (onlyInFTS.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in FTS5: ${onlyInFTS.slice(0, 5).join(", ")}${onlyInFTS.length > 5 ? "..." : ""}`);
}
if (onlyInTraditional.length > 0) {
log.info(`[QUICK-SEARCH-COMPARISON] Only in Traditional: ${onlyInTraditional.slice(0, 5).join(", ")}${onlyInTraditional.length > 5 ? "..." : ""}`);
}
} else {
log.info(`[QUICK-SEARCH-COMPARISON] Results match perfectly! ✓`);
}
log.info(`[QUICK-SEARCH-COMPARISON] ========================================`);
} else {
results = findResultsWithExpression(expression, searchContext);
}
} }
return results; return results;