implemented consistency checks

app.js

@@ -72,6 +72,9 @@ require('./services/sync');
 // triggers backup timer
 require('./services/backup');
 
+// trigger consistency checks timer
+require('./services/consistency_checks');
+
 module.exports = {
     app,
     sessionParser

migrations/0051__note_id_index_on_notes_tree.sql

@@ -0,0 +1,5 @@
+DROP index IDX_notes_tree_note_tree_id;
+
+CREATE INDEX `IDX_notes_tree_note_id` ON `notes_tree` (
+  `note_id`
+);

routes/api/notes_move.js

@@ -188,7 +188,7 @@ async function checkCycle(parentNoteId, childNoteId) {
         return false;
     }
 
-    const parentNoteIds = await sql.getFlattenedResults("note_pid", "SELECT DISTINCT note_pid FROM notes_tree WHERE note_id = ?", [parentNoteId]);
+    const parentNoteIds = await sql.getFlattenedResults("SELECT DISTINCT note_pid FROM notes_tree WHERE note_id = ?", [parentNoteId]);
 
     for (const pid of parentNoteIds) {
         if (!await checkCycle(pid, childNoteId)) {

schema.sql

@@ -78,8 +78,8 @@ CREATE TABLE IF NOT EXISTS "notes_tree" (
   `date_modified`	TEXT NOT NULL,
   PRIMARY KEY(`note_tree_id`)
 );
-CREATE INDEX `IDX_notes_tree_note_tree_id` ON `notes_tree` (
+CREATE INDEX `IDX_notes_tree_note_id` ON `notes_tree` (
-  `note_tree_id`
+  `note_id`
 );
 CREATE INDEX `IDX_notes_tree_note_id_note_pid` ON `notes_tree` (
   `note_id`,

services/app_info.js

@@ -3,7 +3,7 @@
 const build = require('./build');
 const packageJson = require('../package');
 
-const APP_DB_VERSION = 50;
+const APP_DB_VERSION = 51;
 
 module.exports = {
     app_version: packageJson.version,

services/consistency_checks.js

@@ -0,0 +1,55 @@
+"use strict";
+
+const sql = require('./sql');
+const log = require('./log');
+
+async function runCheck(query, errorText, errorList) {
+    const result = await sql.getFlattenedResults(query);
+
+    if (result.length > 0) {
+        const err = errorText + ": " + result;
+        errorList.push(err);
+
+        log.error(err);
+    }
+}
+
+async function runMissingSyncRowCheck(table, key, errorList) {
+    await runCheck("SELECT " + key + " FROM " + table + " LEFT JOIN sync ON sync.entity_name = '" + table + "' AND entity_id = " + key + " WHERE sync.id IS NULL",
+        "Missing sync records for " + key + " in table " + table, errorList);
+}
+
+async function runChecks() {
+    const errorList = [];
+
+    await runCheck("SELECT note_id FROM notes LEFT JOIN notes_tree USING(note_id) WHERE notes_tree.note_tree_id IS NULL",
+        "Missing notes_tree records for following note IDs", errorList);
+
+    await runCheck("SELECT note_tree_id FROM notes_tree LEFT JOIN notes USING(note_id) WHERE notes.note_id IS NULL",
+        "Missing notes records for following note tree IDs", errorList);
+
+    await runCheck("SELECT note_tree_id FROM notes_tree JOIN notes USING(note_id) WHERE notes.is_deleted = 1 AND notes_tree.is_deleted = 0",
+        "Note tree is not deleted even though main note is deleted for following note tree IDs", errorList);
+
+    await runCheck("SELECT child.note_pid || ' > ' || child.note_id FROM notes_tree AS child LEFT JOIN notes_tree AS parent ON parent.note_id = child.note_pid WHERE parent.note_id IS NULL AND child.note_pid != 'root'",
+        "Not existing parent in the following parent > child relations", errorList);
+
+    await runCheck("SELECT note_history_id || ' > ' || notes_history.note_id FROM notes_history LEFT JOIN notes USING(note_id) WHERE notes.note_id IS NULL",
+        "Missing notes records for following note history ID > note ID", errorList);
+
+    await runMissingSyncRowCheck("notes", "note_id", errorList);
+    await runMissingSyncRowCheck("notes_history", "note_history_id", errorList);
+    await runMissingSyncRowCheck("notes_tree", "note_tree_id", errorList);
+    await runMissingSyncRowCheck("recent_notes", "note_tree_id", errorList);
+}
+
+sql.dbReady.then(() => {
+    setInterval(runChecks, 60 * 60 * 1000);
+
+    // kickoff backup immediately
+    setTimeout(runChecks, 5000);
+});
+
+module.exports = {
+    runChecks
+};

services/notes.js

@@ -73,7 +73,7 @@ async function protectNoteRecursively(noteId, dataKey, protect) {
 
     await protectNote(note, dataKey, protect);
 
-    const children = await sql.getFlattenedResults("note_id", "SELECT note_id FROM notes_tree WHERE note_pid = ?", [noteId]);
+    const children = await sql.getFlattenedResults("SELECT note_id FROM notes_tree WHERE note_pid = ?", [noteId]);
 
     for (const childNoteId of children) {
         await protectNoteRecursively(childNoteId, dataKey, protect);

services/source_id.js

@@ -17,7 +17,7 @@ sql.dbReady.then(async () => {
             });
         });
 
-        allSourceIds = await sql.getFlattenedResults("source_id", "SELECT source_id FROM source_ids ORDER BY date_created DESC");
+        allSourceIds = await sql.getFlattenedResults("SELECT source_id FROM source_ids ORDER BY date_created DESC");
     }
     catch (e) {}
 });

services/sql.js

@@ -155,10 +155,16 @@ async function getMap(query, params = []) {
     return map;
 }
 
-async function getFlattenedResults(key, query, params = []) {
+async function getFlattenedResults(query, params = []) {
     const list = [];
     const result = await getResults(query, params);
 
+    if (result.length === 0) {
+        return list;
+    }
+
+    const key = Object.keys(result[0])[0];
+
     for (const row of result) {
         list.push(row[key]);
     }