From 969f31dde281fea8dd21675009e6650eae3fe189 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 20 Jun 2020 23:09:34 +0200 Subject: [PATCH] fixed backup and anonymization with better-sqlite3 --- src/routes/api/database.js | 8 ++--- src/routes/routes.js | 7 +++- src/services/backup.js | 70 +++++++++++++------------------------- src/services/import/tar.js | 27 ++++++++------- src/services/options.js | 4 +-- src/services/repository.js | 1 - src/services/source_id.js | 5 +-- src/services/sql.js | 1 + 8 files changed, 53 insertions(+), 70 deletions(-) diff --git a/src/routes/api/database.js b/src/routes/api/database.js index f2574ddb3..6dac3020b 100644 --- a/src/routes/api/database.js +++ b/src/routes/api/database.js @@ -5,13 +5,13 @@ const log = require('../../services/log'); const backupService = require('../../services/backup'); const consistencyChecksService = require('../../services/consistency_checks'); -function anonymize() { - return backupService.anonymize(); +async function anonymize() { + return await backupService.anonymize(); } -function backupDatabase() { +async function backupDatabase() { return { - backupFile: backupService.backupNow("now") + backupFile: await backupService.backupNow("now") }; } diff --git a/src/routes/routes.js b/src/routes/routes.js index 23fba4a3d..0cde69324 100644 --- a/src/routes/routes.js +++ b/src/routes/routes.js @@ -97,7 +97,12 @@ function route(method, path, middleware, routeHandler, resultHandler, transactio }); if (resultHandler) { - resultHandler(req, res, result); + if (result && result.then) { + result.then(actualResult => resultHandler(req, res, actualResult)) + } + else { + resultHandler(req, res, result); + } } } catch (e) { diff --git a/src/services/backup.js b/src/services/backup.js index caadf9d21..133b13e7b 100644 --- a/src/services/backup.js +++ b/src/services/backup.js @@ -10,6 +10,7 @@ const syncMutexService = require('./sync_mutex'); const attributeService = require('./attributes'); const cls = require('./cls'); const utils = require('./utils'); +const Database = require('better-sqlite3'); function regularBackup() { periodBackup('lastDailyBackupDate', 'daily', 24 * 3600); @@ -32,7 +33,7 @@ function periodBackup(optionName, fileName, periodInSeconds) { const COPY_ATTEMPT_COUNT = 50; -function copyFile(backupFile) { +async function copyFile(backupFile) { const sql = require('./sql'); try { @@ -40,79 +41,54 @@ function copyFile(backupFile) { } catch (e) { } // unlink throws exception if the file did not exist - let success = false; - let attemptCount = 0 - - for (; attemptCount < COPY_ATTEMPT_COUNT && !success; attemptCount++) { - try { - sql.executeWithoutTransaction(`VACUUM INTO '${backupFile}'`); - - success = true; - } catch (e) { - log.info(`Copy DB attempt ${attemptCount + 1} failed with "${e.message}", retrying...`); - } - // we re-try since VACUUM is very picky and it can't run if there's any other query currently running - // which is difficult to guarantee so we just re-try - } - - return attemptCount !== COPY_ATTEMPT_COUNT; + await sql.dbConnection.backup(backupFile); } async function backupNow(name) { // we don't want to backup DB in the middle of sync with potentially inconsistent DB state - return await syncMutexService.doExclusively(() => { + return await syncMutexService.doExclusively(async () => { const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`; - const success = copyFile(backupFile); + await copyFile(backupFile); - if (success) { - log.info("Created backup at " + backupFile); - } - else { - log.error(`Creating backup ${backupFile} failed`); - } + log.info("Created backup at " + backupFile); return backupFile; }); } -function anonymize() { +async function anonymize() { if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) { fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700); } const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db"; - const success = copyFile(anonymizedFile); + await copyFile(anonymizedFile); - if (!success) { - return { success: false }; - } + const db = new Database(anonymizedFile); - const db = sqlite.open({ - filename: anonymizedFile, - driver: sqlite3.Database - }); - - db.run("UPDATE api_tokens SET token = 'API token value'"); - db.run("UPDATE notes SET title = 'title'"); - db.run("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL"); - db.run("UPDATE note_revisions SET title = 'title'"); - db.run("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL"); + db.prepare("UPDATE api_tokens SET token = 'API token value'").run(); + db.prepare("UPDATE notes SET title = 'title'").run(); + db.prepare("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL").run(); + db.prepare("UPDATE note_revisions SET title = 'title'").run(); + db.prepare("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL").run(); // we want to delete all non-builtin attributes because they can contain sensitive names and values // on the other hand builtin/system attrs should not contain any sensitive info - const builtinAttrs = attributeService.getBuiltinAttributeNames().map(name => "'" + utils.sanitizeSql(name) + "'").join(', '); + const builtinAttrs = attributeService + .getBuiltinAttributeNames() + .map(name => "'" + utils.sanitizeSql(name) + "'").join(', '); - db.run(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`); - db.run(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`); - db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL"); - db.run(`UPDATE options SET value = 'anonymized' WHERE name IN + db.prepare(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`).run(); + db.prepare(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`).run(); + db.prepare("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL").run(); + db.prepare(`UPDATE options SET value = 'anonymized' WHERE name IN ('documentId', 'documentSecret', 'encryptedDataKey', 'passwordVerificationHash', 'passwordVerificationSalt', 'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') - AND value != ''`); - db.run("VACUUM"); + AND value != ''`).run(); + db.prepare("VACUUM").run(); db.close(); diff --git a/src/services/import/tar.js b/src/services/import/tar.js index aca7f0b4f..c668bcd7d 100644 --- a/src/services/import/tar.js +++ b/src/services/import/tar.js @@ -14,6 +14,7 @@ const commonmark = require('commonmark'); const TaskContext = require('../task_context.js'); const protectedSessionService = require('../protected_session'); const mimeService = require("./mime"); +const sql = require("../sql"); const treeService = require("../tree"); /** @@ -166,19 +167,21 @@ function importTar(taskContext, fileBuffer, importRootNote) { return; } - ({note} = noteService.createNewNote({ - parentNoteId: parentNoteId, - title: noteTitle, - content: '', - noteId: noteId, - type: noteMeta ? noteMeta.type : 'text', - mime: noteMeta ? noteMeta.mime : 'text/html', - prefix: noteMeta ? noteMeta.prefix : '', - isExpanded: noteMeta ? noteMeta.isExpanded : false, - isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), - })); + sql.transactional(() => { + ({note} = noteService.createNewNote({ + parentNoteId: parentNoteId, + title: noteTitle, + content: '', + noteId: noteId, + type: noteMeta ? noteMeta.type : 'text', + mime: noteMeta ? noteMeta.mime : 'text/html', + prefix: noteMeta ? noteMeta.prefix : '', + isExpanded: noteMeta ? noteMeta.isExpanded : false, + isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), + })); - saveAttributes(note, noteMeta); + saveAttributes(note, noteMeta); + }); if (!firstNote) { firstNote = note; diff --git a/src/services/options.js b/src/services/options.js index 29e2e410b..efbd3ef44 100644 --- a/src/services/options.js +++ b/src/services/options.js @@ -67,9 +67,7 @@ function getOptions() { } function getOptionsMap() { - const options = getOptions(); - - return utils.toObject(options, opt => [opt.name, opt.value]); + return require('./sql').getMap("SELECT name, value FROM options ORDER BY name"); } module.exports = { diff --git a/src/services/repository.js b/src/services/repository.js index 61b60b093..1a32d30de 100644 --- a/src/services/repository.js +++ b/src/services/repository.js @@ -131,7 +131,6 @@ function updateEntity(entity) { eventService.emit(eventService.ENTITY_CREATED, eventPayload); } - // it seems to be better to handle deletion and update separately eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload); } } diff --git a/src/services/source_id.js b/src/services/source_id.js index 788d9a5a9..c7496af17 100644 --- a/src/services/source_id.js +++ b/src/services/source_id.js @@ -2,7 +2,6 @@ const utils = require('./utils'); const dateUtils = require('./date_utils'); const log = require('./log'); const sql = require('./sql'); -const sqlInit = require('./sql_init'); const cls = require('./cls'); function saveSourceId(sourceId) { @@ -49,8 +48,10 @@ const currentSourceId = createSourceId(); // very ugly setTimeout(() => { + const sqlInit = require('./sql_init'); + sqlInit.dbReady.then(cls.wrap(() => saveSourceId(currentSourceId))); -}, 1000); +}, 5000); function getCurrentSourceId() { return currentSourceId; diff --git a/src/services/sql.js b/src/services/sql.js index f4f4f52cc..c67ddce1f 100644 --- a/src/services/sql.js +++ b/src/services/sql.js @@ -253,6 +253,7 @@ function transactional(func) { } module.exports = { + dbConnection, insert, replace, getValue,