From f745e21e0d3e67b256a9bcd99374cc0063a1cb9d Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 17 Jun 2020 23:03:46 +0200 Subject: [PATCH] use better-sqlite3 --- electron.js | 2 - package-lock.json | 155 ++++++++++++++++++++++++++++++++++++++- package.json | 3 +- src/routes/api/export.js | 4 + src/routes/api/import.js | 6 +- src/services/sql.js | 65 ++++++++++------ src/services/sql_init.js | 30 ++------ 7 files changed, 210 insertions(+), 55 deletions(-) diff --git a/electron.js b/electron.js index 51649c837..a476c0de0 100644 --- a/electron.js +++ b/electron.js @@ -24,8 +24,6 @@ app.on('window-all-closed', () => { app.on('ready', async () => { app.setAppUserModelId('com.github.zadam.trilium'); - await sqlInit.dbConnection; - // if db is not initialized -> setup process // if db is initialized, then we need to wait until the migration process is finished if (await sqlInit.isDbInitialized()) { diff --git a/package-lock.json b/package-lock.json index 16d75f6d1..edd4cd200 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "trilium", - "version": "0.42.7", + "version": "0.43.0-beta", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -1487,6 +1487,32 @@ "resolved": "https://registry.npmjs.org/beeper/-/beeper-1.1.1.tgz", "integrity": "sha1-5tXqjF2tABMEpwsiY4RH9pyy+Ak=" }, + "better-sqlite3": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.1.0.tgz", + "integrity": "sha512-FV/snQ8F/kyqhdxsevzbojVtMowDWOfe1A5N3lYu1KJwoho2t7JgITmdlSc7DkOh3Zq65I+ZyeNWXQrkLEDFTg==", + "requires": { + "bindings": "^1.5.0", + "prebuild-install": "^5.3.3", + "tar": "4.4.10" + }, + "dependencies": { + "tar": { + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz", + "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.3.5", + "minizlib": "^1.2.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.3" + } + } + } + }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -1591,6 +1617,14 @@ "os-filter-obj": "^1.0.0" } }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "bl": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", @@ -4186,6 +4220,11 @@ } } }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" + }, "expect-ct": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.2.0.tgz", @@ -4458,6 +4497,11 @@ "typedarray-to-buffer": "^3.1.5" } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "filelist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.1.tgz", @@ -4978,6 +5022,11 @@ "logalot": "^2.0.0" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" + }, "glob": { "version": "5.0.15", "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", @@ -7243,6 +7292,11 @@ } } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "modify-filename": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/modify-filename/-/modify-filename-1.1.0.tgz", @@ -7876,6 +7930,11 @@ "integrity": "sha512-s/snB+WGm6uwi0WjsZdaVcuf3KJXlfGl2LcxgwkEwJF0D/BWzVWAZW/XY4bFaiR7s0Jk3FPvlnepg1H1b1UwlA==", "dev": true }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "needle": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/needle/-/needle-2.4.1.tgz", @@ -7923,9 +7982,9 @@ "integrity": "sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==" }, "node-abi": { - "version": "2.16.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.16.0.tgz", - "integrity": "sha512-+sa0XNlWDA6T+bDLmkCUYn6W5k5W6BPRL6mqzSCs6H/xUgtl4D5x2fORKDzopKiU6wsyn/+wXlRXwXeSp+mtoA==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.18.0.tgz", + "integrity": "sha512-yi05ZoiuNNEbyT/xXfSySZE+yVnQW6fxPZuFbLyS1s6b5Kw3HzV2PHOM4XR+nsjzkHxByK+2Wg+yCQbe35l8dw==", "requires": { "semver": "^5.4.1" }, @@ -8037,6 +8096,11 @@ "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz", "integrity": "sha1-WuVUHQJGRdMqWPzdyc7s6nrjrC8=" }, + "noop-logger": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", + "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=" + }, "nopt": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", @@ -9393,6 +9457,43 @@ "is-number-like": "^1.0.3" } }, + "prebuild-install": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.4.tgz", + "integrity": "sha512-AkKN+pf4fSEihjapLEEj8n85YIw/tN6BQqkhzbDc0RvEZGdkpJBGMUYx66AAMcPG2KzmPQS7Cm16an4HVBRRMA==", + "requires": { + "detect-libc": "^1.0.3", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp": "^0.5.1", + "napi-build-utils": "^1.0.1", + "node-abi": "^2.7.0", + "noop-logger": "^0.1.1", + "npmlog": "^4.0.1", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^3.0.3", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0", + "which-pm-runs": "^1.0.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + } + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -10225,6 +10326,36 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "simple-concat": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", + "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" + }, + "simple-get": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "requires": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + }, + "dependencies": { + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + } + } + }, "simple-node-logger": { "version": "18.12.24", "resolved": "https://registry.npmjs.org/simple-node-logger/-/simple-node-logger-18.12.24.tgz", @@ -10633,6 +10764,17 @@ "yallist": "^3.0.3" } }, + "tar-fs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", + "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, "tar-stream": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz", @@ -11777,6 +11919,11 @@ "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" }, + "which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=" + }, "wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", diff --git a/package.json b/package.json index 38c4e225f..3dc3fe868 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "dependencies": { "async-mutex": "0.2.2", "axios": "0.19.2", + "better-sqlite3": "^7.1.0", "body-parser": "1.19.0", "cls-hooked": "4.2.2", "commonmark": "0.29.1", @@ -54,7 +55,7 @@ "jimp": "0.10.3", "mime-types": "2.1.27", "multer": "1.4.2", - "node-abi": "2.16.0", + "node-abi": "2.18.0", "open": "7.0.3", "portscanner": "2.2.0", "rand-token": "1.0.1", diff --git a/src/routes/api/export.js b/src/routes/api/export.js index 109b9bb7b..02390b7a0 100644 --- a/src/routes/api/export.js +++ b/src/routes/api/export.js @@ -23,7 +23,11 @@ async function exportBranch(req, res) { try { if (type === 'subtree' && (format === 'html' || format === 'markdown')) { + const start = Date.now(); + await zipExportService.exportToZip(taskContext, branch, format, res); + + console.log("Export took", Date.now() - start, "ms"); } else if (type === 'single') { await singleExportService.exportSingleNote(taskContext, branch, format, res); diff --git a/src/routes/api/import.js b/src/routes/api/import.js index 7237450d3..007d8599d 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -50,7 +50,11 @@ async function importToBranch(req) { if (extension === '.tar' && options.explodeArchives) { note = await tarImportService.importTar(taskContext, file.buffer, parentNote); } else if (extension === '.zip' && options.explodeArchives) { + const start = Date.now(); + note = await zipImportService.importZip(taskContext, file.buffer, parentNote); + + console.log("Import took", Date.now() - start, "ms"); } else if (extension === '.opml' && options.explodeArchives) { note = await opmlImportService.importOpml(taskContext, file.buffer, parentNote); } else if (extension === '.enex' && options.explodeArchives) { @@ -85,4 +89,4 @@ async function importToBranch(req) { module.exports = { importToBranch -}; \ No newline at end of file +}; diff --git a/src/services/sql.js b/src/services/sql.js index f07144e97..33e5106f7 100644 --- a/src/services/sql.js +++ b/src/services/sql.js @@ -33,7 +33,7 @@ async function insert(tableName, rec, replace = false) { const res = await execute(query, Object.values(rec)); - return res.lastID; + return res.lastInsertRowid; } async function replace(tableName, rec) { @@ -49,34 +49,46 @@ async function upsert(tableName, primaryKey, rec) { const columns = keys.join(", "); - let i = 0; + const questionMarks = keys.map(colName => "@" + colName).join(", "); - const questionMarks = keys.map(p => ":" + i++).join(", "); - - i = 0; - - const updateMarks = keys.map(key => `${key} = :${i++}`).join(", "); + const updateMarks = keys.map(colName => `${colName} = @${colName}`).join(", "); const query = `INSERT INTO ${tableName} (${columns}) VALUES (${questionMarks}) ON CONFLICT (${primaryKey}) DO UPDATE SET ${updateMarks}`; - await execute(query, Object.values(rec)); + for (const idx in rec) { + if (rec[idx] === true || rec[idx] === false) { + rec[idx] = rec[idx] ? 1 : 0; + } + } + + await execute(query, rec); } -async function beginTransaction() { - return await dbConnection.run("BEGIN"); +const statementCache = {}; + +function stmt(sql) { + if (!(sql in statementCache)) { + statementCache[sql] = dbConnection.prepare(sql); + } + + return statementCache[sql]; } -async function commit() { - return await dbConnection.run("COMMIT"); +function beginTransaction() { + return stmt("BEGIN").run(); } -async function rollback() { - return await dbConnection.run("ROLLBACK"); +function commit() { + return stmt("COMMIT").run(); +} + +function rollback() { + return stmt("ROLLBACK").run(); } async function getRow(query, params = []) { - return await wrap(async db => db.get(query, ...params), query); + return wrap(() => stmt(query).get(params), query); } async function getRowOrNull(query, params = []) { @@ -105,18 +117,25 @@ async function getManyRows(query, params) { const curParams = params.slice(0, Math.min(params.length, PARAM_LIMIT)); params = params.slice(curParams.length); + const curParamsObj = {}; + + let j = 1; + for (const param of curParams) { + curParamsObj['param' + j++] = param; + } + let i = 1; - const questionMarks = curParams.map(() => "?" + i++).join(","); + const questionMarks = curParams.map(() => ":param" + i++).join(","); const curQuery = query.replace(/\?\?\?/g, questionMarks); - results = results.concat(await getRows(curQuery, curParams)); + results = results.concat(await getRows(curQuery, curParamsObj)); } return results; } async function getRows(query, params = []) { - return await wrap(async db => db.all(query, ...params), query); + return wrap(() => stmt(query).all(params), query); } async function getMap(query, params = []) { @@ -152,11 +171,11 @@ async function getColumn(query, params = []) { async function execute(query, params = []) { await startTransactionIfNecessary(); - return await wrap(async db => db.run(query, ...params), query); + return wrap(() => stmt(query).run(params), query); } async function executeWithoutTransaction(query, params = []) { - await dbConnection.run(query, ...params); + await dbConnection.run(query, params); } async function executeMany(query, params) { @@ -169,10 +188,10 @@ async function executeMany(query, params) { async function executeScript(query) { await startTransactionIfNecessary(); - return await wrap(async db => db.exec(query), query); + return wrap(() => stmt.run(query), query); } -async function wrap(func, query) { +function wrap(func, query) { if (!dbConnection) { throw new Error("DB connection not initialized yet"); } @@ -182,7 +201,7 @@ async function wrap(func, query) { try { const startTimestamp = Date.now(); - const result = await func(dbConnection); + const result = func(dbConnection); const milliseconds = Date.now() - startTimestamp; if (milliseconds >= 300) { diff --git a/src/services/sql_init.js b/src/services/sql_init.js index eaf0aaf23..a8f8cc97d 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -1,8 +1,6 @@ const log = require('./log'); const dataDir = require('./data_dir'); const fs = require('fs'); -const sqlite = require('sqlite'); -const sqlite3 = require('sqlite3'); const resourceDir = require('./resource_dir'); const appInfo = require('./app_info'); const sql = require('./sql'); @@ -12,28 +10,14 @@ const optionService = require('./options'); const port = require('./port'); const Option = require('../entities/option'); const TaskContext = require('./task_context.js'); +const Database = require('better-sqlite3'); -const dbConnection = new Promise(async (resolve, reject) => { - const db = await sqlite.open({ - filename: dataDir.DOCUMENT_PATH, - driver: sqlite3.Database - }); +const dbConnection = new Database(dataDir.DOCUMENT_PATH); +dbConnection.pragma('journal_mode = WAL'); - db.run('PRAGMA journal_mode = WAL;'); +sql.setDbConnection(dbConnection); - sql.setDbConnection(db); - - resolve(); -}); - -let dbReadyResolve = null; -const dbReady = new Promise(async (resolve, reject) => { - dbReadyResolve = resolve; - - await dbConnection; - - initDbConnection(); -}); +const dbReady = initDbConnection(); async function schemaExists() { const tableResults = await sql.getRows("SELECT name FROM sqlite_master WHERE type='table' AND name='options'"); @@ -78,7 +62,6 @@ async function initDbConnection() { await require('./options_init').initStartupOptions(); log.info("DB ready."); - dbReadyResolve(); }); } @@ -189,7 +172,6 @@ dbReady.then(async () => { module.exports = { dbReady, - dbConnection, schemaExists, isDbInitialized, initDbConnection, @@ -197,4 +179,4 @@ module.exports = { createInitialDatabase, createDatabaseForSync, dbInitialized -}; \ No newline at end of file +};