safer backup to file using VACUUM INTO + possibility to explicitly ask for backup now

src/public/app/dialogs/options/advanced.js

@@ -24,6 +24,13 @@ const TPL = `
 <p>This action will create a new copy of the database and anonymise it (remove all note content and leave only structure and metadata)
     for sharing online for debugging purposes without fear of leaking your personal data.</p>
 
+<h4>Backup database</h4>
+
+<button id="backup-database-button" class="btn">Backup database</button>
+
+<br/>
+<br/>
+
 <h4>Vacuum database</h4>
 
 <p>This will rebuild database which will typically result in smaller database file. No data will be actually changed.</p>
@@ -37,6 +44,7 @@ export default class AdvancedOptions {
         this.$forceFullSyncButton = $("#force-full-sync-button");
         this.$fillSyncRowsButton = $("#fill-sync-rows-button");
         this.$anonymizeButton = $("#anonymize-button");
+        this.$backupDatabaseButton = $("#backup-database-button");
         this.$vacuumDatabaseButton = $("#vacuum-database-button");
         this.$findAndFixConsistencyIssuesButton = $("#find-and-fix-consistency-issues-button");
 
@@ -58,16 +66,22 @@ export default class AdvancedOptions {
             toastService.showMessage("Created anonymized database");
         });
 
+        this.$backupDatabaseButton.on('click', async () => {
+            const {backupFile} = await server.post('database/backup-database');
+
+            toastService.showMessage("Database has been backed up to " + backupFile, 10000);
+        });
+
         this.$vacuumDatabaseButton.on('click', async () => {
-            await server.post('cleanup/vacuum-database');
+            await server.post('database/vacuum-database');
 
             toastService.showMessage("Database has been vacuumed");
         });
 
         this.$findAndFixConsistencyIssuesButton.on('click', async () => {
-            await server.post('cleanup/find-and-fix-consistency-issues');
+            await server.post('database/find-and-fix-consistency-issues');
 
             toastService.showMessage("Consistency issues should be fixed.");
         });
     }
-}
+}

src/routes/api/database.js

@@ -2,8 +2,15 @@
 
 const sql = require('../../services/sql');
 const log = require('../../services/log');
+const backupService = require('../../services/backup');
 const consistencyChecksService = require('../../services/consistency_checks');
 
+async function backupDatabase() {
+    return {
+        backupFile: await backupService.backupNow("now")
+    };
+}
+
 async function vacuumDatabase() {
     await sql.execute("VACUUM");
 
@@ -15,6 +22,7 @@ async function findAndFixConsistencyIssues() {
 }
 
 module.exports = {
+    backupDatabase,
     vacuumDatabase,
     findAndFixConsistencyIssues
-};
+};

src/routes/routes.js

@@ -25,7 +25,7 @@ const importRoute = require('./api/import');
 const setupApiRoute = require('./api/setup');
 const sqlRoute = require('./api/sql');
 const anonymizationRoute = require('./api/anonymization');
-const cleanupRoute = require('./api/cleanup');
+const databaseRoute = require('./api/database');
 const imageRoute = require('./api/image');
 const attributesRoute = require('./api/attributes');
 const scriptRoute = require('./api/script');
@@ -222,10 +222,13 @@ function register(app) {
     apiRoute(POST, '/api/sql/execute', sqlRoute.execute);
     apiRoute(POST, '/api/anonymization/anonymize', anonymizationRoute.anonymize);
 
-    // VACUUM requires execution outside of transaction
-    route(POST, '/api/cleanup/vacuum-database', [auth.checkApiAuthOrElectron, csrfMiddleware], cleanupRoute.vacuumDatabase, apiResultHandler, false);
+    // backup requires execution outside of transaction
+    route(POST, '/api/database/backup-database', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.backupDatabase, apiResultHandler, false);
 
-    route(POST, '/api/cleanup/find-and-fix-consistency-issues', [auth.checkApiAuthOrElectron, csrfMiddleware], cleanupRoute.findAndFixConsistencyIssues, apiResultHandler, false);
+    // VACUUM requires execution outside of transaction
+    route(POST, '/api/database/vacuum-database', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.vacuumDatabase, apiResultHandler, false);
+
+    route(POST, '/api/database/find-and-fix-consistency-issues', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.findAndFixConsistencyIssues, apiResultHandler, false);
 
     apiRoute(POST, '/api/script/exec', scriptRoute.exec);
     apiRoute(POST, '/api/script/run/:noteId', scriptRoute.run);
@@ -267,4 +270,4 @@ function register(app) {
 
 module.exports = {
     register
-};
+};

src/services/backup.js

@@ -29,13 +29,38 @@ async function periodBackup(optionName, fileName, periodInSeconds) {
 }
 
 async function backupNow(name) {
+    const sql = require('./sql');
+
     // we don't want to backup DB in the middle of sync with potentially inconsistent DB state
-    await syncMutexService.doExclusively(async () => {
+    return await syncMutexService.doExclusively(async () => {
         const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`;
 
-        fs.copySync(dataDir.DOCUMENT_PATH, backupFile);
+        try {
+            fs.unlinkSync(backupFile);
+        }
+        catch (e) {} // unlink throws exception if the file did not exist
 
-        log.info("Created backup at " + backupFile);
+        let success = false;
+        let attemptCount = 0
+
+        for (; attemptCount < 50 && !success; attemptCount++) {
+            try {
+                await sql.executeNoWrap(`VACUUM INTO '${backupFile}'`);
+                success++;
+            }
+            catch (e) {}
+            // we re-try since VACUUM is very picky and it can't run if there's any other query currently running
+            // which is difficult to guarantee so we just re-try
+        }
+
+        if (attemptCount === 10) {
+            log.error(`Creating backup ${backupFile} failed`);
+        }
+        else {
+            log.info("Created backup at " + backupFile);
+        }
+
+        return backupFile;
     });
 }
 
@@ -52,4 +77,4 @@ sqlInit.dbReady.then(() => {
 
 module.exports = {
     backupNow
-};
+};

src/services/sql.js

@@ -153,6 +153,10 @@ async function execute(query, params = []) {
     return await wrap(async db => db.run(query, ...params), query);
 }
 
+async function executeNoWrap(query, params = []) {
+    await dbConnection.run(query, ...params);
+}
+
 async function executeMany(query, params) {
     // essentially just alias
     await getManyRows(query, params);
@@ -264,6 +268,7 @@ module.exports = {
     getMap,
     getColumn,
     execute,
+    executeNoWrap,
     executeMany,
     executeScript,
     transactional,