From bb8e5ebd4aadbf4e54df70179cdfec0ad8a33eeb Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Fri, 28 Nov 2025 21:25:24 -0800 Subject: [PATCH] fix(fts): fix suggestions from elian --- .../src/migrations/0234__add_fts5_search.ts | 17 -- apps/server/src/routes/route_api.ts | 4 +- ...{fts_search.test.ts => fts_search.spec.ts} | 0 .../services/search/performance_monitor.ts | 178 ------------------ 4 files changed, 2 insertions(+), 197 deletions(-) rename apps/server/src/services/search/{fts_search.test.ts => fts_search.spec.ts} (100%) delete mode 100644 apps/server/src/services/search/performance_monitor.ts diff --git a/apps/server/src/migrations/0234__add_fts5_search.ts b/apps/server/src/migrations/0234__add_fts5_search.ts index 42034a191..113257937 100644 --- a/apps/server/src/migrations/0234__add_fts5_search.ts +++ b/apps/server/src/migrations/0234__add_fts5_search.ts @@ -17,29 +17,12 @@ import log from "../services/log.js"; export default function addFTS5SearchAndPerformanceIndexes() { log.info("Starting FTS5 and performance optimization migration..."); - // Verify SQLite version supports trigram tokenizer (requires 3.34.0+) - const sqliteVersion = sql.getValue(`SELECT sqlite_version()`); - const [major, minor, patch] = sqliteVersion.split('.').map(Number); - const versionNumber = major * 10000 + minor * 100 + (patch || 0); - const requiredVersion = 3 * 10000 + 34 * 100 + 0; // 3.34.0 - - if (versionNumber < requiredVersion) { - log.error(`SQLite version ${sqliteVersion} does not support trigram tokenizer (requires 3.34.0+)`); - log.info("Skipping FTS5 trigram migration - will use fallback search implementation"); - return; // Skip FTS5 setup, rely on fallback search - } - - log.info(`SQLite version ${sqliteVersion} confirmed - trigram tokenizer available`); - // Part 1: FTS5 Setup log.info("Creating FTS5 virtual table for full-text search..."); // Create FTS5 virtual table // We store noteId, title, and content for searching sql.executeScript(` - -- Drop existing FTS table if it exists (for re-running migration in dev) - DROP TABLE IF EXISTS notes_fts; - -- Create FTS5 virtual table with trigram tokenizer -- Trigram tokenizer provides language-agnostic substring matching: -- 1. Fast substring matching (50-100x speedup for LIKE queries without wildcards) diff --git a/apps/server/src/routes/route_api.ts b/apps/server/src/routes/route_api.ts index fe7033fe7..1b4ea48f2 100644 --- a/apps/server/src/routes/route_api.ts +++ b/apps/server/src/routes/route_api.ts @@ -11,7 +11,7 @@ import auth from "../services/auth.js"; import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js"; import { safeExtractMessageAndStackFromError } from "../services/utils.js"; -const MAX_ALLOWED_FILE_SIZE_MB = 2500; +const MAX_ALLOWED_FILE_SIZE_MB = 250; export const router = express.Router(); // TODO: Deduplicate with etapi_utils.ts afterwards. @@ -183,7 +183,7 @@ export function createUploadMiddleware(): RequestHandler { if (!process.env.TRILIUM_NO_UPLOAD_LIMIT) { multerOptions.limits = { - fileSize: MAX_ALLOWED_FILE_SIZE_MB * 1024 * 1024 * 1024 + fileSize: MAX_ALLOWED_FILE_SIZE_MB * 1024 * 1024 }; } diff --git a/apps/server/src/services/search/fts_search.test.ts b/apps/server/src/services/search/fts_search.spec.ts similarity index 100% rename from apps/server/src/services/search/fts_search.test.ts rename to apps/server/src/services/search/fts_search.spec.ts diff --git a/apps/server/src/services/search/performance_monitor.ts b/apps/server/src/services/search/performance_monitor.ts deleted file mode 100644 index 44936afd8..000000000 --- a/apps/server/src/services/search/performance_monitor.ts +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Performance monitoring utilities for search operations - */ - -import log from "../log.js"; -import optionService from "../options.js"; - -export interface SearchMetrics { - query: string; - backend: "typescript" | "sqlite"; - totalTime: number; - parseTime?: number; - searchTime?: number; - resultCount: number; - memoryUsed?: number; - cacheHit?: boolean; - error?: string; -} - -export interface DetailedMetrics extends SearchMetrics { - phases?: { - name: string; - duration: number; - }[]; - sqliteStats?: { - rowsScanned?: number; - indexUsed?: boolean; - tempBTreeUsed?: boolean; - }; -} - -interface SearchPerformanceAverages { - avgTime: number; - avgResults: number; - totalQueries: number; - errorRate: number; -} - -class PerformanceMonitor { - private metrics: SearchMetrics[] = []; - private maxMetricsStored = 1000; - private metricsEnabled = false; - - constructor() { - // Check if performance logging is enabled - this.updateSettings(); - } - - updateSettings() { - try { - this.metricsEnabled = optionService.getOptionBool("searchSqlitePerformanceLogging"); - } catch { - this.metricsEnabled = false; - } - } - - startTimer(): () => number { - const startTime = process.hrtime.bigint(); - return () => { - const endTime = process.hrtime.bigint(); - return Number(endTime - startTime) / 1_000_000; // Convert to milliseconds - }; - } - - recordMetrics(metrics: SearchMetrics) { - if (!this.metricsEnabled) { - return; - } - - this.metrics.push(metrics); - - // Keep only the last N metrics - if (this.metrics.length > this.maxMetricsStored) { - this.metrics = this.metrics.slice(-this.maxMetricsStored); - } - - // Log significant performance differences - if (metrics.totalTime > 1000) { - log.info(`Slow search query detected: ${metrics.totalTime.toFixed(2)}ms for query "${metrics.query.substring(0, 100)}"`); - } - - // Log to debug for analysis - log.info(`Search metrics: backend=${metrics.backend}, time=${metrics.totalTime.toFixed(2)}ms, results=${metrics.resultCount}, query="${metrics.query.substring(0, 50)}"`); - } - - recordDetailedMetrics(metrics: DetailedMetrics) { - if (!this.metricsEnabled) { - return; - } - - this.recordMetrics(metrics); - - // Log detailed phase information - if (metrics.phases) { - const phaseLog = metrics.phases - .map(p => `${p.name}=${p.duration.toFixed(2)}ms`) - .join(", "); - log.info(`Search phases: ${phaseLog}`); - } - - // Log SQLite specific stats - if (metrics.sqliteStats) { - log.info(`SQLite stats: rows_scanned=${metrics.sqliteStats.rowsScanned}, index_used=${metrics.sqliteStats.indexUsed}`); - } - } - - getRecentMetrics(count: number = 100): SearchMetrics[] { - return this.metrics.slice(-count); - } - - getAverageMetrics(backend?: "typescript" | "sqlite"): SearchPerformanceAverages | null { - let relevantMetrics = this.metrics; - - if (backend) { - relevantMetrics = this.metrics.filter(m => m.backend === backend); - } - - if (relevantMetrics.length === 0) { - return null; - } - - const totalTime = relevantMetrics.reduce((sum, m) => sum + m.totalTime, 0); - const totalResults = relevantMetrics.reduce((sum, m) => sum + m.resultCount, 0); - const errorCount = relevantMetrics.filter(m => m.error).length; - - return { - avgTime: totalTime / relevantMetrics.length, - avgResults: totalResults / relevantMetrics.length, - totalQueries: relevantMetrics.length, - errorRate: errorCount / relevantMetrics.length - }; - } - - compareBackends(): { - typescript: SearchPerformanceAverages; - sqlite: SearchPerformanceAverages; - recommendation?: string; - } { - const tsMetrics = this.getAverageMetrics("typescript"); - const sqliteMetrics = this.getAverageMetrics("sqlite"); - - let recommendation: string | undefined; - - if (tsMetrics && sqliteMetrics) { - const speedupFactor = tsMetrics.avgTime / sqliteMetrics.avgTime; - - if (speedupFactor > 1.5) { - recommendation = `SQLite is ${speedupFactor.toFixed(1)}x faster on average`; - } else if (speedupFactor < 0.67) { - recommendation = `TypeScript is ${(1/speedupFactor).toFixed(1)}x faster on average`; - } else { - recommendation = "Both backends perform similarly"; - } - - // Consider error rates - if (sqliteMetrics.errorRate > tsMetrics.errorRate + 0.1) { - recommendation += " (but SQLite has higher error rate)"; - } else if (tsMetrics.errorRate > sqliteMetrics.errorRate + 0.1) { - recommendation += " (but TypeScript has higher error rate)"; - } - } - - return { - typescript: tsMetrics || { avgTime: 0, avgResults: 0, totalQueries: 0, errorRate: 0 }, - sqlite: sqliteMetrics || { avgTime: 0, avgResults: 0, totalQueries: 0, errorRate: 0 }, - recommendation - }; - } - - reset() { - this.metrics = []; - } -} - -// Singleton instance -const performanceMonitor = new PerformanceMonitor(); - -export default performanceMonitor; \ No newline at end of file