diff --git a/app.js b/app.js index a3def3edf..6c9c01f70 100644 --- a/app.js +++ b/app.js @@ -72,6 +72,9 @@ require('./services/sync'); // triggers backup timer require('./services/backup'); +// trigger consistency checks timer +require('./services/consistency_checks'); + module.exports = { app, sessionParser diff --git a/migrations/0051__note_id_index_on_notes_tree.sql b/migrations/0051__note_id_index_on_notes_tree.sql new file mode 100644 index 000000000..bc08c5d19 --- /dev/null +++ b/migrations/0051__note_id_index_on_notes_tree.sql @@ -0,0 +1,5 @@ +DROP index IDX_notes_tree_note_tree_id; + +CREATE INDEX `IDX_notes_tree_note_id` ON `notes_tree` ( + `note_id` +); \ No newline at end of file diff --git a/routes/api/notes_move.js b/routes/api/notes_move.js index e431e34a1..1c02cdc6a 100644 --- a/routes/api/notes_move.js +++ b/routes/api/notes_move.js @@ -188,7 +188,7 @@ async function checkCycle(parentNoteId, childNoteId) { return false; } - const parentNoteIds = await sql.getFlattenedResults("note_pid", "SELECT DISTINCT note_pid FROM notes_tree WHERE note_id = ?", [parentNoteId]); + const parentNoteIds = await sql.getFlattenedResults("SELECT DISTINCT note_pid FROM notes_tree WHERE note_id = ?", [parentNoteId]); for (const pid of parentNoteIds) { if (!await checkCycle(pid, childNoteId)) { diff --git a/schema.sql b/schema.sql index 87a137804..be727bc2d 100644 --- a/schema.sql +++ b/schema.sql @@ -78,8 +78,8 @@ CREATE TABLE IF NOT EXISTS "notes_tree" ( `date_modified` TEXT NOT NULL, PRIMARY KEY(`note_tree_id`) ); -CREATE INDEX `IDX_notes_tree_note_tree_id` ON `notes_tree` ( - `note_tree_id` +CREATE INDEX `IDX_notes_tree_note_id` ON `notes_tree` ( + `note_id` ); CREATE INDEX `IDX_notes_tree_note_id_note_pid` ON `notes_tree` ( `note_id`, diff --git a/services/app_info.js b/services/app_info.js index 0876a135e..60a9b2d9b 100644 --- a/services/app_info.js +++ b/services/app_info.js @@ -3,7 +3,7 @@ const build = require('./build'); const packageJson = require('../package'); -const APP_DB_VERSION = 50; +const APP_DB_VERSION = 51; module.exports = { app_version: packageJson.version, diff --git a/services/consistency_checks.js b/services/consistency_checks.js new file mode 100644 index 000000000..163e6a017 --- /dev/null +++ b/services/consistency_checks.js @@ -0,0 +1,55 @@ +"use strict"; + +const sql = require('./sql'); +const log = require('./log'); + +async function runCheck(query, errorText, errorList) { + const result = await sql.getFlattenedResults(query); + + if (result.length > 0) { + const err = errorText + ": " + result; + errorList.push(err); + + log.error(err); + } +} + +async function runMissingSyncRowCheck(table, key, errorList) { + await runCheck("SELECT " + key + " FROM " + table + " LEFT JOIN sync ON sync.entity_name = '" + table + "' AND entity_id = " + key + " WHERE sync.id IS NULL", + "Missing sync records for " + key + " in table " + table, errorList); +} + +async function runChecks() { + const errorList = []; + + await runCheck("SELECT note_id FROM notes LEFT JOIN notes_tree USING(note_id) WHERE notes_tree.note_tree_id IS NULL", + "Missing notes_tree records for following note IDs", errorList); + + await runCheck("SELECT note_tree_id FROM notes_tree LEFT JOIN notes USING(note_id) WHERE notes.note_id IS NULL", + "Missing notes records for following note tree IDs", errorList); + + await runCheck("SELECT note_tree_id FROM notes_tree JOIN notes USING(note_id) WHERE notes.is_deleted = 1 AND notes_tree.is_deleted = 0", + "Note tree is not deleted even though main note is deleted for following note tree IDs", errorList); + + await runCheck("SELECT child.note_pid || ' > ' || child.note_id FROM notes_tree AS child LEFT JOIN notes_tree AS parent ON parent.note_id = child.note_pid WHERE parent.note_id IS NULL AND child.note_pid != 'root'", + "Not existing parent in the following parent > child relations", errorList); + + await runCheck("SELECT note_history_id || ' > ' || notes_history.note_id FROM notes_history LEFT JOIN notes USING(note_id) WHERE notes.note_id IS NULL", + "Missing notes records for following note history ID > note ID", errorList); + + await runMissingSyncRowCheck("notes", "note_id", errorList); + await runMissingSyncRowCheck("notes_history", "note_history_id", errorList); + await runMissingSyncRowCheck("notes_tree", "note_tree_id", errorList); + await runMissingSyncRowCheck("recent_notes", "note_tree_id", errorList); +} + +sql.dbReady.then(() => { + setInterval(runChecks, 60 * 60 * 1000); + + // kickoff backup immediately + setTimeout(runChecks, 5000); +}); + +module.exports = { + runChecks +}; \ No newline at end of file diff --git a/services/notes.js b/services/notes.js index f02fee4d8..9718751ff 100644 --- a/services/notes.js +++ b/services/notes.js @@ -73,7 +73,7 @@ async function protectNoteRecursively(noteId, dataKey, protect) { await protectNote(note, dataKey, protect); - const children = await sql.getFlattenedResults("note_id", "SELECT note_id FROM notes_tree WHERE note_pid = ?", [noteId]); + const children = await sql.getFlattenedResults("SELECT note_id FROM notes_tree WHERE note_pid = ?", [noteId]); for (const childNoteId of children) { await protectNoteRecursively(childNoteId, dataKey, protect); diff --git a/services/source_id.js b/services/source_id.js index fd6c01503..2ab6ae149 100644 --- a/services/source_id.js +++ b/services/source_id.js @@ -17,7 +17,7 @@ sql.dbReady.then(async () => { }); }); - allSourceIds = await sql.getFlattenedResults("source_id", "SELECT source_id FROM source_ids ORDER BY date_created DESC"); + allSourceIds = await sql.getFlattenedResults("SELECT source_id FROM source_ids ORDER BY date_created DESC"); } catch (e) {} }); diff --git a/services/sql.js b/services/sql.js index 2a788f0ae..650afe9ef 100644 --- a/services/sql.js +++ b/services/sql.js @@ -155,10 +155,16 @@ async function getMap(query, params = []) { return map; } -async function getFlattenedResults(key, query, params = []) { +async function getFlattenedResults(query, params = []) { const list = []; const result = await getResults(query, params); + if (result.length === 0) { + return list; + } + + const key = Object.keys(result[0])[0]; + for (const row of result) { list.push(row[key]); }