cleanup of soft deleted items

vacuuming database
consolidation of "advanced" operations in settings

public/javascripts/dialogs/settings.js

@@ -155,6 +155,9 @@ settings.addModule((async function () {
 settings.addModule((async function () {
     const forceFullSyncButton = $("#force-full-sync-button");
     const fillSyncRowsButton = $("#fill-sync-rows-button");
+    const anonymizeButton = $("#anonymize-button");
+    const cleanupSoftDeletedButton = $("#cleanup-soft-deleted-items-button");
+    const vacuumDatabaseButton = $("#vacuum-database-button");
 
     forceFullSyncButton.click(async () => {
         await server.post('sync/force-full-sync');
@@ -168,11 +171,6 @@ settings.addModule((async function () {
         showMessage("Sync rows filled successfully");
     });
 
-    return {};
-})());
-
-settings.addModule((async function () {
-    const anonymizeButton = $("#anonymize-button");
 
     anonymizeButton.click(async () => {
         await server.post('anonymization/anonymize');
@@ -180,5 +178,19 @@ settings.addModule((async function () {
         showMessage("Created anonymized database");
     });
 
+    cleanupSoftDeletedButton.click(async () => {
+        if (confirm("Do you really want to clean up soft-deleted items?")) {
+            await server.post('cleanup/cleanup-soft-deleted-items');
+
+            showMessage("Soft deleted items have been cleaned up");
+        }
+    });
+
+    vacuumDatabaseButton.click(async () => {
+        await server.post('cleanup/vacuum-database');
+
+        showMessage("Database has been vacuumed");
+    });
+
     return {};
 })());

routes/api/cleanup.js

@@ -0,0 +1,43 @@
+"use strict";
+
+const express = require('express');
+const router = express.Router();
+const sql = require('../../services/sql');
+const utils = require('../../services/utils');
+const sync_table = require('../../services/sync_table');
+
+router.post('/cleanup-soft-deleted-items', async (req, res, next) => {
+    await sql.doInTransaction(async () => {
+        const noteIdsToDelete = await sql.getFlattenedResults("SELECT note_id FROM notes WHERE is_deleted = 1");
+        const noteIdsSql = noteIdsToDelete
+            .map(noteId => "'" + utils.sanitizeSql(noteId) + "'")
+            .join(', ');
+
+        console.log("Note IDS for deletion", noteIdsSql);
+
+        await sql.execute(`DELETE FROM event_log WHERE note_id IN (${noteIdsSql})`);
+
+        await sql.execute(`DELETE FROM notes_history WHERE note_id IN (${noteIdsSql})`);
+
+        await sql.execute("DELETE FROM notes_tree WHERE is_deleted = 1");
+
+        await sql.execute("DELETE FROM notes WHERE is_deleted = 1");
+
+        await sql.execute("DELETE FROM recent_notes");
+
+        await sync_table.cleanupSyncRowsForMissingEntities("notes", "note_id");
+        await sync_table.cleanupSyncRowsForMissingEntities("notes_tree", "note_tree_id");
+        await sync_table.cleanupSyncRowsForMissingEntities("notes_history", "note_history_id");
+        await sync_table.cleanupSyncRowsForMissingEntities("recent_notes", "note_tree_id");
+    });
+
+    res.send({});
+});
+
+router.post('/vacuum-database', async (req, res, next) => {
+    await sql.execute("VACUUM");
+
+    res.send({});
+});
+
+module.exports = router;

routes/api/import.js

@@ -2,7 +2,6 @@
 
 const express = require('express');
 const router = express.Router();
-const rimraf = require('rimraf');
 const fs = require('fs');
 const sql = require('../../services/sql');
 const data_dir = require('../../services/data_dir');

routes/api/sync.js

@@ -5,11 +5,10 @@ const router = express.Router();
 const auth = require('../../services/auth');
 const sync = require('../../services/sync');
 const syncUpdate = require('../../services/sync_update');
+const sync_table = require('../../services/sync_table');
 const sql = require('../../services/sql');
 const options = require('../../services/options');
 const content_hash = require('../../services/content_hash');
-const utils = require('../../services/utils');
-const log = require('../../services/log');
 
 router.get('/check', auth.checkApiAuth, async (req, res, next) => {
     res.send({
@@ -22,39 +21,12 @@ router.post('/now', auth.checkApiAuth, async (req, res, next) => {
     res.send(await sync.sync());
 });
 
-async function fillSyncRows(entityName, entityKey) {
-    // cleanup sync rows for missing entities
-    await sql.execute(`
-      DELETE 
-      FROM sync 
-      WHERE sync.entity_name = '${entityName}' 
-        AND sync.entity_id NOT IN (SELECT ${entityKey} FROM ${entityName})`);
-
-    const entityIds = await sql.getFlattenedResults(`SELECT ${entityKey} FROM ${entityName}`);
-
-    for (const entityId of entityIds) {
-        const existingRows = await sql.getSingleValue("SELECT COUNT(id) FROM sync WHERE entity_name = ? AND entity_id = ?", [entityName, entityId]);
-
-        // we don't want to replace existing entities (which would effectively cause full resync)
-        if (existingRows === 0) {
-            log.info(`Creating missing sync record for ${entityName} ${entityId}`);
-
-            await sql.insert("sync", {
-                entity_name: entityName,
-                entity_id: entityId,
-                source_id: "SYNC_FILL",
-                sync_date: utils.nowDate()
-            });
-        }
-    }
-}
-
 router.post('/fill-sync-rows', auth.checkApiAuth, async (req, res, next) => {
     await sql.doInTransaction(async () => {
-        await fillSyncRows("notes", "note_id");
+        await sync_table.fillSyncRows("notes", "note_id");
-        await fillSyncRows("notes_tree", "note_tree_id");
+        await sync_table.fillSyncRows("notes_tree", "note_tree_id");
-        await fillSyncRows("notes_history", "note_history_id");
+        await sync_table.fillSyncRows("notes_history", "note_history_id");
-        await fillSyncRows("recent_notes", "note_tree_id");
+        await sync_table.fillSyncRows("recent_notes", "note_tree_id");
     });
 
     res.send({});

routes/routes.js

@@ -23,6 +23,7 @@ const importRoute = require('./api/import');
 const setupApiRoute = require('./api/setup');
 const sqlRoute = require('./api/sql');
 const anonymizationRoute = require('./api/anonymization');
+const cleanupRoute = require('./api/cleanup');
 
 function register(app) {
     app.use('/', indexRoute);
@@ -49,6 +50,7 @@ function register(app) {
     app.use('/api/setup', setupApiRoute);
     app.use('/api/sql', sqlRoute);
     app.use('/api/anonymization', anonymizationRoute);
+    app.use('/api/cleanup', cleanupRoute);
 }
 
 module.exports = {

services/sync_table.js

@@ -2,6 +2,7 @@ const sql = require('./sql');
 const source_id = require('./source_id');
 const utils = require('./utils');
 const sync_setup = require('./sync_setup');
+const log = require('./log');
 
 async function addNoteSync(noteId, sourceId) {
     await addEntitySync("notes", noteId, sourceId)
@@ -42,11 +43,43 @@ async function addEntitySync(entityName, entityId, sourceId) {
     }
 }
 
+async function cleanupSyncRowsForMissingEntities(entityName, entityKey) {
+    await sql.execute(`
+      DELETE 
+      FROM sync 
+      WHERE sync.entity_name = '${entityName}' 
+        AND sync.entity_id NOT IN (SELECT ${entityKey} FROM ${entityName})`);
+}
+
+async function fillSyncRows(entityName, entityKey) {
+    await cleanupSyncRowsForMissingEntities(entityName, entityKey);
+
+    const entityIds = await sql.getFlattenedResults(`SELECT ${entityKey} FROM ${entityName}`);
+
+    for (const entityId of entityIds) {
+        const existingRows = await sql.getSingleValue("SELECT COUNT(id) FROM sync WHERE entity_name = ? AND entity_id = ?", [entityName, entityId]);
+
+        // we don't want to replace existing entities (which would effectively cause full resync)
+        if (existingRows === 0) {
+            log.info(`Creating missing sync record for ${entityName} ${entityId}`);
+
+            await sql.insert("sync", {
+                entity_name: entityName,
+                entity_id: entityId,
+                source_id: "SYNC_FILL",
+                sync_date: utils.nowDate()
+            });
+        }
+    }
+}
+
 module.exports = {
     addNoteSync,
     addNoteTreeSync,
     addNoteReorderingSync,
     addNoteHistorySync,
     addOptionsSync,
-    addRecentNoteSync
+    addRecentNoteSync,
+    cleanupSyncRowsForMissingEntities,
+    fillSyncRows
 };

services/utils.js

@@ -74,6 +74,11 @@ function getDateTimeForFile() {
     return new Date().toISOString().substr(0, 19).replace(/:/g, '');
 }
 
+function sanitizeSql(str) {
+    // should be improved or usage eliminated
+    return str.replace(/'/g, "\\'");
+}
+
 module.exports = {
     randomSecureToken,
     randomString,
@@ -89,5 +94,6 @@ module.exports = {
     isElectron,
     hash,
     isEmptyOrWhitespace,
-    getDateTimeForFile
+    getDateTimeForFile,
+    sanitizeSql
 };

views/index.ejs

@@ -183,8 +183,7 @@
           <li><a href="#change-password">Change password</a></li>
           <li><a href="#protected-session-timeout">Protected session</a></li>
           <li><a href="#history-snapshot-time-interval">History snapshots</a></li>
-          <li><a href="#sync">Sync</a></li>
+          <li><a href="#advanced">Advanced</a></li>
-          <li><a href="#debugging">Debugging</a></li>
           <li><a href="#about">About Trilium</a></li>
         </ul>
         <div id="change-password">
@@ -232,16 +231,30 @@
             <button class="btn btn-sm">Save</button>
           </form>
         </div>
-        <div id="sync">
+        <div id="advanced">
+          <h4 style="margin-top: 0px;">Sync</h4>
           <button id="force-full-sync-button" class="btn btn-sm">Force full sync</button>
 
           <br/>
           <br/>
 
           <button id="fill-sync-rows-button" class="btn btn-sm">Fill sync rows</button>
-        </div>
+
-        <div id="debugging">
+          <h4>Debugging</h4>
-          <button id="anonymize-button" class="btn btn-sm">Save anonymized database</button>
+
+          <button id="anonymize-button" class="btn btn-sm">Save anonymized database</button><br/><br/>
+
+          <p>This action will create a new copy of the database and anonymise it (remove all note content and leave only structure and metadata)
+            for sharing online for debugging purposes without fear of leaking your personal data.</p>
+
+          <h4>Cleanup</h4>
+
+          <button id="cleanup-soft-deleted-items-button" class="btn btn-danger btn-sm">Permanently cleanup soft-deleted items</button>
+
+          <br/>
+          <br/>
+
+          <button id="vacuum-database-button" class="btn btn-sm">Vacuum database</button>
         </div>
         <div id="about">
           <table class="table">