From a67464b4a0328ecaac301c33fd13fa6924145a8a Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Mon, 5 Jan 2026 14:03:03 +0200 Subject: [PATCH] refactor(server): decouple bettersqlite3 from sql service --- apps/server/src/services/sql.ts | 105 +++++++++++++------------ apps/server/src/services/sql_nodejs.ts | 71 +++++++++++++++++ 2 files changed, 126 insertions(+), 50 deletions(-) create mode 100644 apps/server/src/services/sql_nodejs.ts diff --git a/apps/server/src/services/sql.ts b/apps/server/src/services/sql.ts index 206a828d6..61bc9c6d7 100644 --- a/apps/server/src/services/sql.ts +++ b/apps/server/src/services/sql.ts @@ -1,44 +1,65 @@ -"use strict"; - -/** - * @module sql - */ - -import log from "./log.js"; -import type { Statement, Database as DatabaseType, RunResult } from "better-sqlite3"; -import dataDir from "./data_dir.js"; -import cls from "./cls.js"; import fs from "fs"; -import Database from "better-sqlite3"; -import ws from "./ws.js"; + import becca_loader from "../becca/becca_loader.js"; -import entity_changes from "./entity_changes.js"; +import cls from "./cls.js"; import config from "./config.js"; +import dataDir from "./data_dir.js"; +import entity_changes from "./entity_changes.js"; +import log from "./log.js"; +import BetterSqlite3Provider from "./sql_nodejs.js"; +import ws from "./ws.js"; -const dbOpts: Database.Options = { - nativeBinding: process.env.BETTERSQLITE3_NATIVE_PATH || undefined -}; +type Params = any; -let dbConnection: DatabaseType = buildDatabase(); +export interface Statement { + run(...params: Params): RunResult; + get(params: Params): unknown; + all(...params: Params): unknown[]; + iterate(...params: Params): IterableIterator; + raw(toggleState?: boolean): this; + pluck(toggleState?: boolean): this; +} + +export interface Transaction { + deferred(): void; +} + +export interface RunResult { + changes: number; + lastInsertRowid: number | bigint; +} + +export interface DatabaseProvider { + loadFromFile(path: string, isReadOnly: boolean): void; + loadFromMemory(): void; + loadFromBuffer(buffer: NonSharedBuffer): void; + backup(destinationFile: string): void; + prepare(query: string): Statement; + transaction(func: (statement: Statement) => T): Transaction; + get inTransaction(): boolean; + exec(query: string): void; + close(): void; +} + +const dbConnection: DatabaseProvider = new BetterSqlite3Provider(); let statementCache: Record = {}; function buildDatabase() { // for integration tests, ignore the config's readOnly setting if (process.env.TRILIUM_INTEGRATION_TEST === "memory") { - return buildIntegrationTestDatabase(); + buildIntegrationTestDatabase(); } else if (process.env.TRILIUM_INTEGRATION_TEST === "memory-no-store") { - return new Database(":memory:", dbOpts); + dbConnection.loadFromMemory(); } - return new Database(dataDir.DOCUMENT_PATH, { - ...dbOpts, - readonly: config.General.readOnly - }); + dbConnection.loadFromFile(dataDir.DOCUMENT_PATH, config.General.readOnly); } +buildDatabase(); + function buildIntegrationTestDatabase(dbPath?: string) { - const dbBuffer = fs.readFileSync(dbPath ?? dataDir.DOCUMENT_PATH); - return new Database(dbBuffer, dbOpts); + const buffer = fs.readFileSync(dbPath ?? dataDir.DOCUMENT_PATH); + dbConnection.loadFromBuffer(buffer); } function rebuildIntegrationTestDatabase(dbPath?: string) { @@ -47,28 +68,12 @@ function rebuildIntegrationTestDatabase(dbPath?: string) { } // This allows a database that is read normally but is kept in memory and discards all modifications. - dbConnection = buildIntegrationTestDatabase(dbPath); + buildIntegrationTestDatabase(dbPath); statementCache = {}; } -if (!process.env.TRILIUM_INTEGRATION_TEST) { - dbConnection.pragma("journal_mode = WAL"); -} - const LOG_ALL_QUERIES = false; -type Params = any; - -[`exit`, `SIGINT`, `SIGUSR1`, `SIGUSR2`, `SIGTERM`].forEach((eventType) => { - process.on(eventType, () => { - if (dbConnection) { - // closing connection is especially important to fold -wal file into the main DB file - // (see https://sqlite.org/tempfiles.html for details) - dbConnection.close(); - } - }); -}); - function insert(tableName: string, rec: T, replace = false) { const keys = Object.keys(rec || {}); if (keys.length === 0) { @@ -129,7 +134,7 @@ function upsert(tableName: string, primaryKey: string, rec: T) { * @returns the corresponding {@link Statement}. */ function stmt(sql: string, isRaw?: boolean) { - const key = (isRaw ? "raw/" + sql : sql); + const key = (isRaw ? `raw/${sql}` : sql); if (!(key in statementCache)) { statementCache[key] = dbConnection.prepare(sql); @@ -169,11 +174,11 @@ function getManyRows(query: string, params: Params): T[] { let j = 1; for (const param of curParams) { - curParamsObj["param" + j++] = param; + curParamsObj[`param${j++}`] = param; } let i = 1; - const questionMarks = curParams.map(() => ":param" + i++).join(","); + const questionMarks = curParams.map(() => `:param${i++}`).join(","); const curQuery = query.replace(/\?\?\?/g, questionMarks); const statement = curParams.length === PARAM_LIMIT ? stmt(curQuery) : dbConnection.prepare(curQuery); @@ -240,23 +245,23 @@ function executeMany(query: string, params: Params) { let j = 1; for (const param of curParams) { - curParamsObj["param" + j++] = param; + curParamsObj[`param${j++}`] = param; } let i = 1; - const questionMarks = curParams.map(() => ":param" + i++).join(","); + const questionMarks = curParams.map(() => `:param${i++}`).join(","); const curQuery = query.replace(/\?\?\?/g, questionMarks); dbConnection.prepare(curQuery).run(curParamsObj); } } -function executeScript(query: string): DatabaseType { +function executeScript(query: string) { if (LOG_ALL_QUERIES) { console.log(query); } - return dbConnection.exec(query); + dbConnection.exec(query); } /** @@ -350,7 +355,7 @@ function fillParamList(paramIds: string[] | Set, truncate = true) { async function copyDatabase(targetFilePath: string) { try { fs.unlinkSync(targetFilePath); - } catch (e) {} // unlink throws exception if the file did not exist + } catch (e) { } // unlink throws exception if the file did not exist await dbConnection.backup(targetFilePath); } diff --git a/apps/server/src/services/sql_nodejs.ts b/apps/server/src/services/sql_nodejs.ts new file mode 100644 index 000000000..77cf4b8ed --- /dev/null +++ b/apps/server/src/services/sql_nodejs.ts @@ -0,0 +1,71 @@ +import Database, { type Database as DatabaseType } from "better-sqlite3"; +import { readFileSync } from "fs"; + +import dataDirs from "./data_dir"; +import type { DatabaseProvider, Statement, Transaction } from "./sql"; + +const dbOpts: Database.Options = { + nativeBinding: process.env.BETTERSQLITE3_NATIVE_PATH || undefined +}; + +export default class BetterSqlite3Provider implements DatabaseProvider { + + private dbConnection?: DatabaseType; + + constructor() { + [`exit`, `SIGINT`, `SIGUSR1`, `SIGUSR2`, `SIGTERM`].forEach((eventType) => { + // closing connection is especially important to fold -wal file into the main DB file + // (see https://sqlite.org/tempfiles.html for details) + process.on(eventType, () => this.close()); + }); + } + + loadFromFile(path: string, isReadOnly: boolean) { + this.dbConnection = new Database(path, { + ...dbOpts, + readonly: isReadOnly + }); + this.dbConnection.pragma("journal_mode = WAL"); + } + + loadFromMemory() { + this.dbConnection = new Database(":memory:", dbOpts); + } + + loadFromBuffer(buffer: NonSharedBuffer) { + this.dbConnection = new Database(buffer, dbOpts); + } + + backup(destinationFile: string) { + this.dbConnection?.backup(destinationFile); + } + + prepare(query: string): Statement { + if (!this.dbConnection) throw new Error("DB not open."); + return this.dbConnection.prepare(query); + } + + transaction(func: (statement: Statement) => T): Transaction { + if (!this.dbConnection) throw new Error("DB not open."); + return this.dbConnection.transaction(func); + } + + get inTransaction() { + if (!this.dbConnection) throw new Error("DB not open."); + return this.dbConnection.inTransaction; + } + + exec(query: string): void { + this.dbConnection?.exec(query); + } + + close() { + this.dbConnection?.close(); + } + +} + +function buildIntegrationTestDatabase(dbPath?: string) { + const dbBuffer = readFileSync(dbPath ?? dataDirs.DOCUMENT_PATH); + return new Database(dbBuffer, dbOpts); +}