gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

final fixes and refactorings for consistency checks

Browse Source
This commit is contained in:
zadam 2019-02-02 12:41:20 +01:00
parent 40d2e6ea83
commit 89344a6eda
1 changed files with 83 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
src/services/consistency_checks.js
View File

@ -10,7 +10,7 @@ const cls = require('./cls');
const syncTableService = require('./sync_table'); const syncTableService = require('./sync_table');
const Branch = require('../entities/branch'); const Branch = require('../entities/branch');
let outstandingConsistencyErrors = false; let unrecoverableConsistencyErrors = false;
let fixedIssues = false; let fixedIssues = false;
async function findIssues(query, errorCb) { async function findIssues(query, errorCb) {
@ -19,7 +19,7 @@ async function findIssues(query, errorCb) {
for (const res of results) { for (const res of results) {
logError(errorCb(res)); logError(errorCb(res));
outstandingConsistencyErrors = true; unrecoverableConsistencyErrors = true;
} }
return results; return results;
@ -57,7 +57,7 @@ async function checkTreeCycles() {
if (!childToParents[noteId] || childToParents[noteId].length === 0) { if (!childToParents[noteId] || childToParents[noteId].length === 0) {
logError(`No parents found for note ${noteId}`); logError(`No parents found for note ${noteId}`);
outstandingConsistencyErrors = true; unrecoverableConsistencyErrors = true;
return; return;
} }
@ -65,7 +65,7 @@ async function checkTreeCycles() {
if (path.includes(parentNoteId)) { if (path.includes(parentNoteId)) {
logError(`Tree cycle detected at parent-child relationship: ${parentNoteId} - ${noteId}, whole path: ${path}`); logError(`Tree cycle detected at parent-child relationship: ${parentNoteId} - ${noteId}, whole path: ${path}`);
outstandingConsistencyErrors = true; unrecoverableConsistencyErrors = true;
} }
else { else {
const newPath = path.slice(); const newPath = path.slice();
@ -84,7 +84,7 @@ async function checkTreeCycles() {
if (childToParents['root'].length !== 1 || childToParents['root'][0] !== 'none') { if (childToParents['root'].length !== 1 || childToParents['root'][0] !== 'none') {
logError('Incorrect root parent: ' + JSON.stringify(childToParents['root'])); logError('Incorrect root parent: ' + JSON.stringify(childToParents['root']));
outstandingConsistencyErrors = true; unrecoverableConsistencyErrors = true;
} }
} }
@ -137,30 +137,14 @@ async function findExistencyIssues() {
// principle for fixing inconsistencies is that if the note itself is deleted (isDeleted=true) then all related entities should be also deleted (branches, links, attributes) // principle for fixing inconsistencies is that if the note itself is deleted (isDeleted=true) then all related entities should be also deleted (branches, links, attributes)
// but if note is not deleted, then at least one branch should exist. // but if note is not deleted, then at least one branch should exist.
await findAndFixIssues(` // the order here is important - first we might need to delete inconsistent branches and after that
SELECT // another check might create missing branch
DISTINCT noteId
FROM
notes
WHERE
(SELECT COUNT(*) FROM branches WHERE notes.noteId = branches.noteId AND branches.isDeleted = 0) = 0
AND notes.isDeleted = 0
`, async ({noteId}) => {
const branch = await new Branch({
parentNoteId: 'root',
noteId: noteId,
prefix: 'recovered'
}).save();
logFix(`Created missing branch ${branch.branchId} for note ${noteId}`);
});
await findAndFixIssues(` await findAndFixIssues(`
SELECT SELECT
branchId, noteId branchId, noteId
FROM FROM
branches branches
JOIN notes USING(noteId) JOIN notes USING(noteId)
WHERE WHERE
notes.isDeleted = 1 notes.isDeleted = 1
AND branches.isDeleted = 0`, AND branches.isDeleted = 0`,
@ -172,40 +156,78 @@ async function findExistencyIssues() {
logFix(`Branch ${branchId} has been deleted since associated note ${noteId} is deleted.`); logFix(`Branch ${branchId} has been deleted since associated note ${noteId} is deleted.`);
}); });
await findAndFixIssues(`
SELECT
branchId, parentNoteId
FROM
branches
JOIN notes AS parentNote ON parentNote.noteId = branches.parentNoteId
WHERE
parentNote.isDeleted = 1
AND branches.isDeleted = 0
`, async ({branchId, parentNoteId}) => {
const branch = await repository.getBranch(branchId);
branch.isDeleted = true;
await branch.save();
logFix(`Branch ${branchId} has been deleted since associated parent note ${parentNoteId} is deleted.`);
});
await findAndFixIssues(`
SELECT
DISTINCT notes.noteId
FROM
notes
LEFT JOIN branches ON notes.noteId = branches.noteId AND branches.isDeleted = 0
WHERE
notes.isDeleted = 0
AND branches.branchId IS NULL
`, async ({noteId}) => {
const branch = await new Branch({
parentNoteId: 'root',
noteId: noteId,
prefix: 'recovered'
}).save();
logFix(`Created missing branch ${branch.branchId} for note ${noteId}`);
});
// there should be a unique relationship between note and its parent // there should be a unique relationship between note and its parent
await findAndFixIssues(` await findAndFixIssues(`
SELECT SELECT
noteId, parentNoteId noteId, parentNoteId
FROM FROM
branches branches
WHERE WHERE
branches.isDeleted = 0 branches.isDeleted = 0
GROUP BY GROUP BY
branches.parentNoteId, branches.parentNoteId,
branches.noteId branches.noteId
HAVING HAVING
COUNT(*) > 1`, COUNT(*) > 1`,
async ({noteId, parentNoteId}) => { async ({noteId, parentNoteId}) => {
const branches = await repository.getEntities(`SELECT * FROM branches WHERE noteId = ? and parentNoteId = ? and isDeleted = 1`, [noteId, parentNoteId]); const branches = await repository.getEntities(`SELECT * FROM branches WHERE noteId = ? and parentNoteId = ? and isDeleted = 1`, [noteId, parentNoteId]);
// it's not necessarily "original" branch, it's just the only one which will survive // it's not necessarily "original" branch, it's just the only one which will survive
const origBranch = branches.get(0); const origBranch = branches.get(0);
// delete all but the first branch // delete all but the first branch
for (const branch of branches.slice(1)) { for (const branch of branches.slice(1)) {
branch.isDeleted = true; branch.isDeleted = true;
await branch.save(); await branch.save();
logFix(`Removing branch ${branch.branchId} since it's parent-child duplicate of branch ${origBranch.branchId}`); logFix(`Removing branch ${branch.branchId} since it's parent-child duplicate of branch ${origBranch.branchId}`);
} }
}); });
} }
async function findLogicIssues() { async function findLogicIssues() {
await findIssues( ` await findIssues( `
SELECT noteId, type SELECT noteId, type
FROM notes FROM notes
WHERE type NOT IN ('text', 'code', 'render', 'file', 'image', 'search', 'relation-map')`, WHERE
isDeleted = 0
AND type NOT IN ('text', 'code', 'render', 'file', 'image', 'search', 'relation-map')`,
({noteId, type}) => `Note ${noteId} has invalid type=${type}`); ({noteId, type}) => `Note ${noteId} has invalid type=${type}`);
await findIssues(` await findIssues(`
@ -221,8 +243,10 @@ async function findLogicIssues() {
FROM FROM
branches branches
JOIN notes ON notes.noteId = branches.parentNoteId JOIN notes ON notes.noteId = branches.parentNoteId
WHERE WHERE
type == 'search'`, notes.isDeleted = 0
AND notes.type == 'search'
AND branches.isDeleted = 0`,
({parentNoteId}) => `Search note ${parentNoteId} has children`); ({parentNoteId}) => `Search note ${parentNoteId} has children`);
await findAndFixIssues(` await findAndFixIssues(`
@ -246,24 +270,13 @@ async function findLogicIssues() {
type type
FROM attributes FROM attributes
WHERE WHERE
type != 'label' isDeleted = 0
AND type != 'label'
AND type != 'label-definition' AND type != 'label-definition'
AND type != 'relation' AND type != 'relation'
AND type != 'relation-definition'`, AND type != 'relation-definition'`,
({attributeId, type}) => `Attribute ${attributeId} has invalid type '${type}'`); ({attributeId, type}) => `Attribute ${attributeId} has invalid type '${type}'`);
await findIssues(`
SELECT
attributeId,
attributes.noteId
FROM
attributes
LEFT JOIN notes ON attributes.noteId = notes.noteId
WHERE
attributes.isDeleted = 0
AND notes.noteId IS NULL`,
({attributeId, noteId}) => `Attribute ${attributeId} reference to the owning note ${noteId} is broken`);
await findAndFixIssues(` await findAndFixIssues(`
SELECT SELECT
attributeId, attributeId,
@ -307,7 +320,7 @@ async function findLogicIssues() {
WHERE type NOT IN ('image', 'hyper', 'relation-map')`, WHERE type NOT IN ('image', 'hyper', 'relation-map')`,
({linkId, type}) => `Link ${linkId} has invalid type '${type}'`); ({linkId, type}) => `Link ${linkId} has invalid type '${type}'`);
await findIssues(` await findAndFixIssues(`
SELECT SELECT
linkId, linkId,
links.noteId AS sourceNoteId links.noteId AS sourceNoteId
@ -387,7 +400,8 @@ async function findSyncRowsIssues() {
} }
async function runAllChecks() { async function runAllChecks() {
outstandingConsistencyErrors = false; unrecoverableConsistencyErrors = false;
fixedIssues = false;
await findBrokenReferenceIssues(); await findBrokenReferenceIssues();
@ -397,23 +411,22 @@ async function runAllChecks() {
await findSyncRowsIssues(); await findSyncRowsIssues();
if (outstandingConsistencyErrors) { if (unrecoverableConsistencyErrors) {
// we run this only if basic checks passed since this assumes basic data consistency // we run this only if basic checks passed since this assumes basic data consistency
await checkTreeCycles(); await checkTreeCycles();
} }
return !outstandingConsistencyErrors; return !unrecoverableConsistencyErrors;
} }
async function runChecks() { async function runChecks() {
let elapsedTimeMs; let elapsedTimeMs;
let dbConsistent;
await syncMutexService.doExclusively(async () => { await syncMutexService.doExclusively(async () => {
const startTime = new Date(); const startTime = new Date();
dbConsistent = await runAllChecks(); await runAllChecks();
elapsedTimeMs = new Date().getTime() - startTime.getTime(); elapsedTimeMs = new Date().getTime() - startTime.getTime();
}); });
@ -422,7 +435,7 @@ async function runChecks() {
messagingService.sendMessageToAllClients({ type: 'refresh-tree' }); messagingService.sendMessageToAllClients({ type: 'refresh-tree' });
} }
if (!dbConsistent) { if (unrecoverableConsistencyErrors) {
log.info(`Consistency checks failed (took ${elapsedTimeMs}ms)`); log.info(`Consistency checks failed (took ${elapsedTimeMs}ms)`);
messagingService.sendMessageToAllClients({type: 'consistency-checks-failed'}); messagingService.sendMessageToAllClients({type: 'consistency-checks-failed'});
@ -443,7 +456,7 @@ function logError(message) {
sqlInit.dbReady.then(() => { sqlInit.dbReady.then(() => {
setInterval(cls.wrap(runChecks), 60 * 60 * 1000); setInterval(cls.wrap(runChecks), 60 * 60 * 1000);
// kickoff backup immediately // kickoff checks soon after startup (to not block the initial load)
setTimeout(cls.wrap(runChecks), 0); setTimeout(cls.wrap(runChecks), 0);
}); });