gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixed backup and anonymization with better-sqlite3

Browse Source
This commit is contained in:
zadam 2020-06-20 23:09:34 +02:00
parent 027afab6b1
commit 969f31dde2
8 changed files with 53 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/routes/api/database.js
View File

@ -5,13 +5,13 @@ const log = require('../../services/log');
const backupService = require('../../services/backup');
const consistencyChecksService = require('../../services/consistency_checks');
function anonymize() {
return backupService.anonymize();
async function anonymize() {
return await backupService.anonymize();
}
function backupDatabase() {
async function backupDatabase() {
return {
backupFile: backupService.backupNow("now")
backupFile: await backupService.backupNow("now")
};
}

7
src/routes/routes.js
View File

@ -97,7 +97,12 @@ function route(method, path, middleware, routeHandler, resultHandler, transactio
});
if (resultHandler) {
resultHandler(req, res, result);
if (result && result.then) {
result.then(actualResult => resultHandler(req, res, actualResult))
}
else {
resultHandler(req, res, result);
}
}
}
catch (e) {

70
src/services/backup.js
View File

@ -10,6 +10,7 @@ const syncMutexService = require('./sync_mutex');
const attributeService = require('./attributes');
const cls = require('./cls');
const utils = require('./utils');
const Database = require('better-sqlite3');
function regularBackup() {
periodBackup('lastDailyBackupDate', 'daily', 24 * 3600);
@ -32,7 +33,7 @@ function periodBackup(optionName, fileName, periodInSeconds) {
const COPY_ATTEMPT_COUNT = 50;
function copyFile(backupFile) {
async function copyFile(backupFile) {
const sql = require('./sql');
try {
@ -40,79 +41,54 @@ function copyFile(backupFile) {
} catch (e) {
} // unlink throws exception if the file did not exist
let success = false;
let attemptCount = 0
for (; attemptCount < COPY_ATTEMPT_COUNT && !success; attemptCount++) {
try {
sql.executeWithoutTransaction(`VACUUM INTO '${backupFile}'`);
success = true;
} catch (e) {
log.info(`Copy DB attempt ${attemptCount + 1} failed with "${e.message}", retrying...`);
}
// we re-try since VACUUM is very picky and it can't run if there's any other query currently running
// which is difficult to guarantee so we just re-try
}
return attemptCount !== COPY_ATTEMPT_COUNT;
await sql.dbConnection.backup(backupFile);
}
async function backupNow(name) {
// we don't want to backup DB in the middle of sync with potentially inconsistent DB state
return await syncMutexService.doExclusively(() => {
return await syncMutexService.doExclusively(async () => {
const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`;
const success = copyFile(backupFile);
await copyFile(backupFile);
if (success) {
log.info("Created backup at " + backupFile);
}
else {
log.error(`Creating backup ${backupFile} failed`);
}
log.info("Created backup at " + backupFile);
return backupFile;
});
}
function anonymize() {
async function anonymize() {
if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
}
const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
const success = copyFile(anonymizedFile);
await copyFile(anonymizedFile);
if (!success) {
return { success: false };
}
const db = new Database(anonymizedFile);
const db = sqlite.open({
filename: anonymizedFile,
driver: sqlite3.Database
});
db.run("UPDATE api_tokens SET token = 'API token value'");
db.run("UPDATE notes SET title = 'title'");
db.run("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL");
db.run("UPDATE note_revisions SET title = 'title'");
db.run("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL");
db.prepare("UPDATE api_tokens SET token = 'API token value'").run();
db.prepare("UPDATE notes SET title = 'title'").run();
db.prepare("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL").run();
db.prepare("UPDATE note_revisions SET title = 'title'").run();
db.prepare("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL").run();
// we want to delete all non-builtin attributes because they can contain sensitive names and values
// on the other hand builtin/system attrs should not contain any sensitive info
const builtinAttrs = attributeService.getBuiltinAttributeNames().map(name => "'" + utils.sanitizeSql(name) + "'").join(', ');
const builtinAttrs = attributeService
.getBuiltinAttributeNames()
.map(name => "'" + utils.sanitizeSql(name) + "'").join(', ');
db.run(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`);
db.run(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`);
db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL");
db.run(`UPDATE options SET value = 'anonymized' WHERE name IN
db.prepare(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`).run();
db.prepare(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`).run();
db.prepare("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL").run();
db.prepare(`UPDATE options SET value = 'anonymized' WHERE name IN
('documentId', 'documentSecret', 'encryptedDataKey',
'passwordVerificationHash', 'passwordVerificationSalt',
'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy')
AND value != ''`);
db.run("VACUUM");
AND value != ''`).run();
db.prepare("VACUUM").run();
db.close();

27
src/services/import/tar.js
View File

@ -14,6 +14,7 @@ const commonmark = require('commonmark');
const TaskContext = require('../task_context.js');
const protectedSessionService = require('../protected_session');
const mimeService = require("./mime");
const sql = require("../sql");
const treeService = require("../tree");
/**
@ -166,19 +167,21 @@ function importTar(taskContext, fileBuffer, importRootNote) {
return;
}
({note} = noteService.createNewNote({
parentNoteId: parentNoteId,
title: noteTitle,
content: '',
noteId: noteId,
type: noteMeta ? noteMeta.type : 'text',
mime: noteMeta ? noteMeta.mime : 'text/html',
prefix: noteMeta ? noteMeta.prefix : '',
isExpanded: noteMeta ? noteMeta.isExpanded : false,
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
}));
sql.transactional(() => {
({note} = noteService.createNewNote({
parentNoteId: parentNoteId,
title: noteTitle,
content: '',
noteId: noteId,
type: noteMeta ? noteMeta.type : 'text',
mime: noteMeta ? noteMeta.mime : 'text/html',
prefix: noteMeta ? noteMeta.prefix : '',
isExpanded: noteMeta ? noteMeta.isExpanded : false,
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
}));
saveAttributes(note, noteMeta);
saveAttributes(note, noteMeta);
});
if (!firstNote) {
firstNote = note;

4
src/services/options.js
View File

@ -67,9 +67,7 @@ function getOptions() {
}
function getOptionsMap() {
const options = getOptions();
return utils.toObject(options, opt => [opt.name, opt.value]);
return require('./sql').getMap("SELECT name, value FROM options ORDER BY name");
}
module.exports = {

1
src/services/repository.js
View File

@ -131,7 +131,6 @@ function updateEntity(entity) {
eventService.emit(eventService.ENTITY_CREATED, eventPayload);
}
// it seems to be better to handle deletion and update separately
eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload);
}
}

5
src/services/source_id.js
View File

@ -2,7 +2,6 @@ const utils = require('./utils');
const dateUtils = require('./date_utils');
const log = require('./log');
const sql = require('./sql');
const sqlInit = require('./sql_init');
const cls = require('./cls');
function saveSourceId(sourceId) {
@ -49,8 +48,10 @@ const currentSourceId = createSourceId();
// very ugly
setTimeout(() => {
const sqlInit = require('./sql_init');
sqlInit.dbReady.then(cls.wrap(() => saveSourceId(currentSourceId)));
}, 1000);
}, 5000);
function getCurrentSourceId() {
return currentSourceId;

1
src/services/sql.js
View File

@ -253,6 +253,7 @@ function transactional(func) {
}
module.exports = {
dbConnection,
insert,
replace,
getValue,