gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-12-04 22:44:25 +01:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

fix(fts): fix suggestions from elian

Browse Source
This commit is contained in:
perfectra1n 2025-11-28 21:25:24 -08:00
parent 6b8b71f7d1
commit bb8e5ebd4a
4 changed files with 2 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
apps/server/src/migrations/0234__add_fts5_search.ts
View File

@ -17,29 +17,12 @@ import log from "../services/log.js";
export default function addFTS5SearchAndPerformanceIndexes() {
log.info("Starting FTS5 and performance optimization migration...");
// Verify SQLite version supports trigram tokenizer (requires 3.34.0+)
const sqliteVersion = sql.getValue<string>(`SELECT sqlite_version()`);
const [major, minor, patch] = sqliteVersion.split('.').map(Number);
const versionNumber = major * 10000 + minor * 100 + (patch || 0);
const requiredVersion = 3 * 10000 + 34 * 100 + 0; // 3.34.0
if (versionNumber < requiredVersion) {
log.error(`SQLite version ${sqliteVersion} does not support trigram tokenizer (requires 3.34.0+)`);
log.info("Skipping FTS5 trigram migration - will use fallback search implementation");
return; // Skip FTS5 setup, rely on fallback search
}
log.info(`SQLite version ${sqliteVersion} confirmed - trigram tokenizer available`);
// Part 1: FTS5 Setup
log.info("Creating FTS5 virtual table for full-text search...");
// Create FTS5 virtual table
// We store noteId, title, and content for searching
sql.executeScript(`
-- Drop existing FTS table if it exists (for re-running migration in dev)
DROP TABLE IF EXISTS notes_fts;
-- Create FTS5 virtual table with trigram tokenizer
-- Trigram tokenizer provides language-agnostic substring matching:
-- 1. Fast substring matching (50-100x speedup for LIKE queries without wildcards)

4
apps/server/src/routes/route_api.ts
View File

@ -11,7 +11,7 @@ import auth from "../services/auth.js";
import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js";
import { safeExtractMessageAndStackFromError } from "../services/utils.js";
const MAX_ALLOWED_FILE_SIZE_MB = 2500;
const MAX_ALLOWED_FILE_SIZE_MB = 250;
export const router = express.Router();
// TODO: Deduplicate with etapi_utils.ts afterwards.
@ -183,7 +183,7 @@ export function createUploadMiddleware(): RequestHandler {
if (!process.env.TRILIUM_NO_UPLOAD_LIMIT) {
multerOptions.limits = {
fileSize: MAX_ALLOWED_FILE_SIZE_MB * 1024 * 1024 * 1024
fileSize: MAX_ALLOWED_FILE_SIZE_MB * 1024 * 1024
};
}

0
apps/server/src/services/search/fts_search.test.ts → apps/server/src/services/search/fts_search.spec.ts
View File

178
apps/server/src/services/search/performance_monitor.ts
View File

@ -1,178 +0,0 @@
/**
* Performance monitoring utilities for search operations
*/
import log from "../log.js";
import optionService from "../options.js";
export interface SearchMetrics {
query: string;
backend: "typescript" | "sqlite";
totalTime: number;
parseTime?: number;
searchTime?: number;
resultCount: number;
memoryUsed?: number;
cacheHit?: boolean;
error?: string;
}
export interface DetailedMetrics extends SearchMetrics {
phases?: {
name: string;
duration: number;
}[];
sqliteStats?: {
rowsScanned?: number;
indexUsed?: boolean;
tempBTreeUsed?: boolean;
};
}
interface SearchPerformanceAverages {
avgTime: number;
avgResults: number;
totalQueries: number;
errorRate: number;
}
class PerformanceMonitor {
private metrics: SearchMetrics[] = [];
private maxMetricsStored = 1000;
private metricsEnabled = false;
constructor() {
// Check if performance logging is enabled
this.updateSettings();
}
updateSettings() {
try {
this.metricsEnabled = optionService.getOptionBool("searchSqlitePerformanceLogging");
} catch {
this.metricsEnabled = false;
}
}
startTimer(): () => number {
const startTime = process.hrtime.bigint();
return () => {
const endTime = process.hrtime.bigint();
return Number(endTime - startTime) / 1_000_000; // Convert to milliseconds
};
}
recordMetrics(metrics: SearchMetrics) {
if (!this.metricsEnabled) {
return;
}
this.metrics.push(metrics);
// Keep only the last N metrics
if (this.metrics.length > this.maxMetricsStored) {
this.metrics = this.metrics.slice(-this.maxMetricsStored);
}
// Log significant performance differences
if (metrics.totalTime > 1000) {
log.info(`Slow search query detected: ${metrics.totalTime.toFixed(2)}ms for query "${metrics.query.substring(0, 100)}"`);
}
// Log to debug for analysis
log.info(`Search metrics: backend=${metrics.backend}, time=${metrics.totalTime.toFixed(2)}ms, results=${metrics.resultCount}, query="${metrics.query.substring(0, 50)}"`);
}
recordDetailedMetrics(metrics: DetailedMetrics) {
if (!this.metricsEnabled) {
return;
}
this.recordMetrics(metrics);
// Log detailed phase information
if (metrics.phases) {
const phaseLog = metrics.phases
.map(p => `${p.name}=${p.duration.toFixed(2)}ms`)
.join(", ");
log.info(`Search phases: ${phaseLog}`);
}
// Log SQLite specific stats
if (metrics.sqliteStats) {
log.info(`SQLite stats: rows_scanned=${metrics.sqliteStats.rowsScanned}, index_used=${metrics.sqliteStats.indexUsed}`);
}
}
getRecentMetrics(count: number = 100): SearchMetrics[] {
return this.metrics.slice(-count);
}
getAverageMetrics(backend?: "typescript" | "sqlite"): SearchPerformanceAverages | null {
let relevantMetrics = this.metrics;
if (backend) {
relevantMetrics = this.metrics.filter(m => m.backend === backend);
}
if (relevantMetrics.length === 0) {
return null;
}
const totalTime = relevantMetrics.reduce((sum, m) => sum + m.totalTime, 0);
const totalResults = relevantMetrics.reduce((sum, m) => sum + m.resultCount, 0);
const errorCount = relevantMetrics.filter(m => m.error).length;
return {
avgTime: totalTime / relevantMetrics.length,
avgResults: totalResults / relevantMetrics.length,
totalQueries: relevantMetrics.length,
errorRate: errorCount / relevantMetrics.length
};
}
compareBackends(): {
typescript: SearchPerformanceAverages;
sqlite: SearchPerformanceAverages;
recommendation?: string;
} {
const tsMetrics = this.getAverageMetrics("typescript");
const sqliteMetrics = this.getAverageMetrics("sqlite");
let recommendation: string | undefined;
if (tsMetrics && sqliteMetrics) {
const speedupFactor = tsMetrics.avgTime / sqliteMetrics.avgTime;
if (speedupFactor > 1.5) {
recommendation = `SQLite is ${speedupFactor.toFixed(1)}x faster on average`;
} else if (speedupFactor < 0.67) {
recommendation = `TypeScript is ${(1/speedupFactor).toFixed(1)}x faster on average`;
} else {
recommendation = "Both backends perform similarly";
}
// Consider error rates
if (sqliteMetrics.errorRate > tsMetrics.errorRate + 0.1) {
recommendation += " (but SQLite has higher error rate)";
} else if (tsMetrics.errorRate > sqliteMetrics.errorRate + 0.1) {
recommendation += " (but TypeScript has higher error rate)";
}
}
return {
typescript: tsMetrics || { avgTime: 0, avgResults: 0, totalQueries: 0, errorRate: 0 },
sqlite: sqliteMetrics || { avgTime: 0, avgResults: 0, totalQueries: 0, errorRate: 0 },
recommendation
};
}
reset() {
this.metrics = [];
}
}
// Singleton instance
const performanceMonitor = new PerformanceMonitor();
export default performanceMonitor;