gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactoring of consistency checks plus some autofixers

Browse Source
This commit is contained in:
zadam 2019-02-01 22:48:51 +01:00
parent 5619088c41
commit 4a2319cb33
1 changed files with 205 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

299
src/services/consistency_checks.js
View File

@ -5,23 +5,36 @@ const sqlInit = require('./sql_init');
const log = require('./log'); const log = require('./log');
const messagingService = require('./messaging'); const messagingService = require('./messaging');
const syncMutexService = require('./sync_mutex'); const syncMutexService = require('./sync_mutex');
const repository = require('./repository.js'); const repository = require('./repository');
const cls = require('./cls'); const cls = require('./cls');
const syncTableService = require('./sync_table');
const Branch = require('../entities/branch');
async function runCheck(query, errorText, errorList) { let outstandingConsistencyErrors = false;
const result = await sql.getColumn(query);
if (result.length > 0) { async function findIssues(query, errorCb) {
const resultText = result.map(val => "'" + val + "'").join(', '); const results = await sql.getRows(query);
const err = errorText + ": " + resultText; for (const res of results) {
errorList.push(err); log.error(errorCb(res));
log.error(err); outstandingConsistencyErrors = true;
}
} }
async function checkTreeCycles(errorList) { return results;
}
async function findAndFixIssues(query, fixerCb) {
const results = await sql.getRows(query);
for (const res of results) {
await fixerCb(res);
}
return results;
}
async function checkTreeCycles() {
const childToParents = {}; const childToParents = {};
const rows = await sql.getRows("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0"); const rows = await sql.getRows("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0");
@ -33,25 +46,29 @@ async function checkTreeCycles(errorList) {
childToParents[childNoteId].push(parentNoteId); childToParents[childNoteId].push(parentNoteId);
} }
function checkTreeCycle(noteId, path, errorList) { function checkTreeCycle(noteId, path) {
if (noteId === 'root') { if (noteId === 'root') {
return; return;
} }
if (!childToParents[noteId] || childToParents[noteId].length === 0) { if (!childToParents[noteId] || childToParents[noteId].length === 0) {
errorList.push(`No parents found for noteId=${noteId}`); log.error(`No parents found for noteId=${noteId}`);
outstandingConsistencyErrors = true;
return; return;
} }
for (const parentNoteId of childToParents[noteId]) { for (const parentNoteId of childToParents[noteId]) {
if (path.includes(parentNoteId)) { if (path.includes(parentNoteId)) {
errorList.push(`Tree cycle detected at parent-child relationship: ${parentNoteId} - ${noteId}, whole path: ${path}`); log.error(`Tree cycle detected at parent-child relationship: ${parentNoteId} - ${noteId}, whole path: ${path}`);
outstandingConsistencyErrors = true;
} }
else { else {
const newPath = path.slice(); const newPath = path.slice();
newPath.push(noteId); newPath.push(noteId);
checkTreeCycle(parentNoteId, newPath, errorList); checkTreeCycle(parentNoteId, newPath);
} }
} }
} }
@ -59,72 +76,62 @@ async function checkTreeCycles(errorList) {
const noteIds = Object.keys(childToParents); const noteIds = Object.keys(childToParents);
for (const noteId of noteIds) { for (const noteId of noteIds) {
checkTreeCycle(noteId, [], errorList); checkTreeCycle(noteId, []);
} }
if (childToParents['root'].length !== 1 || childToParents['root'][0] !== 'none') { if (childToParents['root'].length !== 1 || childToParents['root'][0] !== 'none') {
errorList.push('Incorrect root parent: ' + JSON.stringify(childToParents['root'])); log.error('Incorrect root parent: ' + JSON.stringify(childToParents['root']));
outstandingConsistencyErrors = true;
} }
} }
async function runSyncRowChecks(table, key, errorList) { async function runSyncRowChecks(entityName, key) {
await runCheck(` await findAndFixIssues(`
SELECT SELECT
${key} ${key} as entityId
FROM FROM
${table} ${entityName}
LEFT JOIN sync ON sync.entityName = '${table}' AND entityId = ${key} LEFT JOIN sync ON sync.entityName = '${entityName}' AND entityId = ${key}
WHERE WHERE
sync.id IS NULL AND ` + (table === 'options' ? 'isSynced = 1' : '1'), sync.id IS NULL AND ` + (entityName === 'options' ? 'isSynced = 1' : '1'),
`Missing sync records for ${key} in table ${table}`, errorList); async ({entityId}) => {
await syncTableService.addEntitySync(entityName, entityId);
await runCheck(` log.info(`Created missing sync record entityName=${entityName}, entityId=${entityId}`);
});
await findAndFixIssues(`
SELECT SELECT
entityId id, entityId
FROM FROM
sync sync
LEFT JOIN ${table} ON entityId = ${key} LEFT JOIN ${entityName} ON entityId = ${key}
WHERE WHERE
sync.entityName = '${table}' sync.entityName = '${entityName}'
AND ${key} IS NULL`, AND ${key} IS NULL`,
`Missing ${table} records for existing sync rows`, errorList); async ({id, entityId}) => {
await sql.execute("DELETE FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
log.error(`Deleted extra sync record id=${id}, entityName=${entityName}, entityId=${entityId}`);
});
} }
async function fixEmptyRelationTargets(errorList) { async function fixEmptyRelationTargets() {
const emptyRelations = await repository.getEntities("SELECT * FROM attributes WHERE isDeleted = 0 AND type = 'relation' AND value = ''"); const emptyRelations = await repository.getEntities("SELECT * FROM attributes WHERE isDeleted = 0 AND type = 'relation' AND value = ''");
for (const relation of emptyRelations) { for (const relation of emptyRelations) {
relation.isDeleted = true; relation.isDeleted = true;
await relation.save(); await relation.save();
errorList.push(`Relation ${relation.attributeId} of name "${relation.name} has empty target. Autofixed.`); log.info(`Removed relation ${relation.attributeId} of name "${relation.name} with empty target..`);
}
}
async function fixUndeletedBranches() {
const undeletedBranches = await sql.getRows(`
SELECT
branchId, noteId
FROM
branches
JOIN notes USING(noteId)
WHERE
notes.isDeleted = 1
AND branches.isDeleted = 0`);
for (const {branchId, noteId} of undeletedBranches) {
const branch = await repository.getBranch(branchId);
branch.isDeleted = true;
await branch.save();
log.info(`Branch ${branchId} has been deleted since associated note ${noteId} is deleted.`);
} }
} }
async function runAllChecks() { async function runAllChecks() {
const errorList = []; outstandingConsistencyErrors = false;
await runCheck(` await findAndFixIssues(`
SELECT SELECT
noteId noteId
FROM FROM
@ -133,34 +140,52 @@ async function runAllChecks() {
WHERE WHERE
noteId != 'root' noteId != 'root'
AND branches.branchId IS NULL`, AND branches.branchId IS NULL`,
"Missing branches records for following note IDs", errorList); async ({noteId}) => {
const branch = await new Branch({
parentNoteId: 'root',
noteId: noteId,
prefix: 'recovered'
}).save();
await runCheck(` log.info(`Created missing branch id=${branch.branchId} for note id=${noteId}`);
});
await findAndFixIssues(`
SELECT SELECT
branchId || ' > ' || branches.noteId branchId,
branches.noteId
FROM FROM
branches branches
LEFT JOIN notes USING(noteId) LEFT JOIN notes USING(noteId)
WHERE WHERE
notes.noteId IS NULL`, notes.noteId IS NULL`,
"Missing notes records for following branch ID > note ID", errorList); async ({branchId, noteId}) => {
const branch = await repository.getBranch(branchId);
branch.isDeleted = true;
await branch.save();
await fixUndeletedBranches(); log.info(`Removed ${branchId} because it pointed to the missing ${noteId}`);
});
await runCheck(` await findAndFixIssues(`
SELECT SELECT
child.branchId branchId, noteId
FROM FROM
branches AS child branches
JOIN notes USING(noteId)
WHERE WHERE
child.isDeleted = 0 notes.isDeleted = 1
AND child.parentNoteId != 'none' AND branches.isDeleted = 0`,
AND (SELECT COUNT(*) FROM branches AS parent WHERE parent.noteId = child.parentNoteId async ({branchId, noteId}) => {
AND parent.isDeleted = 0) = 0`, const branch = await repository.getBranch(branchId);
"All parent branches are deleted but child branch is not for these child branch IDs", errorList); branch.isDeleted = true;
await branch.save();
log.info(`Branch ${branchId} has been deleted since associated note ${noteId} is deleted.`);
});
// we do extra JOIN to eliminate orphan notes without branches (which are reported separately) // we do extra JOIN to eliminate orphan notes without branches (which are reported separately)
await runCheck(` await findAndFixIssues(`
SELECT SELECT
DISTINCT noteId DISTINCT noteId
FROM FROM
@ -169,31 +194,29 @@ async function runAllChecks() {
WHERE WHERE
(SELECT COUNT(*) FROM branches WHERE notes.noteId = branches.noteId AND branches.isDeleted = 0) = 0 (SELECT COUNT(*) FROM branches WHERE notes.noteId = branches.noteId AND branches.isDeleted = 0) = 0
AND notes.isDeleted = 0 AND notes.isDeleted = 0
`, 'No undeleted branches for note IDs', errorList); `, async ({noteId}) => {
const branch = await new Branch({
parentNoteId: 'root',
noteId: noteId,
prefix: 'recovered'
}).save();
await runCheck(` log.info(`Created missing branch ${branch.branchId} for note ${noteId}`);
SELECT });
child.parentNoteId || ' > ' || child.noteId
FROM branches
AS child
LEFT JOIN branches AS parent ON parent.noteId = child.parentNoteId
WHERE
parent.noteId IS NULL
AND child.parentNoteId != 'none'`,
"Not existing parent in the following parent > child relations", errorList);
await runCheck(` await findIssues(`
SELECT SELECT
noteRevisionId || ' > ' || note_revisions.noteId note_revisions.noteId,
noteRevisionId
FROM FROM
note_revisions LEFT JOIN notes USING(noteId) note_revisions LEFT JOIN notes USING(noteId)
WHERE WHERE
notes.noteId IS NULL`, notes.noteId IS NULL`,
"Missing notes records for following note revision ID > note ID", errorList); ({noteId, noteRevisionId}) => `Missing note id=${noteId} for note revision id = ${noteRevisionId}`);
await runCheck(` await findAndFixIssues(`
SELECT SELECT
branches.parentNoteId || ' > ' || branches.noteId noteId, parentNoteId
FROM FROM
branches branches
WHERE WHERE
@ -203,11 +226,25 @@ async function runAllChecks() {
branches.noteId branches.noteId
HAVING HAVING
COUNT(*) > 1`, COUNT(*) > 1`,
"Duplicate undeleted parent note <-> note relationship - parent note ID > note ID", errorList); async ({noteId, parentNoteId}) => {
const branches = await repository.getEntities(`SELECT * FROM branches WHERE noteId = ? and parentNoteId = ? and isDeleted = 1`, [noteId, parentNoteId]);
await runCheck(` // it's not necessarily "original" branch, it's just the only one which will survive
const origBranch = branches.get(0);
// delete all but the first branch
for (const branch of branches.slice(1)) {
branch.isDeleted = true;
await branch.save();
log.info(`Removing branch id=${branch.branchId} since it's parent-child duplicate of branch id=${origBranch.branchId}`);
}
});
await findIssues( `
SELECT SELECT
noteId noteId,
type
FROM FROM
notes notes
WHERE WHERE
@ -218,9 +255,9 @@ async function runAllChecks() {
AND type != 'image' AND type != 'image'
AND type != 'search' AND type != 'search'
AND type != 'relation-map'`, AND type != 'relation-map'`,
"Note has invalid type", errorList); ({noteId, type}) => `Note id=${noteId} has invalid type=${type}`);
await runCheck(` await findIssues(`
SELECT SELECT
noteId noteId
FROM FROM
@ -228,9 +265,9 @@ async function runAllChecks() {
WHERE WHERE
isDeleted = 0 isDeleted = 0
AND content IS NULL`, AND content IS NULL`,
"Note content is null even though it is not deleted", errorList); ({noteId}) => `Note id=${noteId} content is null even though it is not deleted`);
await runCheck(` await findIssues(`
SELECT SELECT
parentNoteId parentNoteId
FROM FROM
@ -238,13 +275,14 @@ async function runAllChecks() {
JOIN notes ON notes.noteId = branches.parentNoteId JOIN notes ON notes.noteId = branches.parentNoteId
WHERE WHERE
type == 'search'`, type == 'search'`,
"Search note has children", errorList); ({parentNoteId}) => `Search note id=${parentNoteId} has children`);
await fixEmptyRelationTargets(errorList); await fixEmptyRelationTargets();
await runCheck(` await findIssues(`
SELECT SELECT
attributeId attributeId,
type
FROM FROM
attributes attributes
WHERE WHERE
@ -252,22 +290,24 @@ async function runAllChecks() {
AND type != 'label-definition' AND type != 'label-definition'
AND type != 'relation' AND type != 'relation'
AND type != 'relation-definition'`, AND type != 'relation-definition'`,
"Attribute has invalid type", errorList); ({attributeId, type}) => `Attribute id=${attributeId}, type=${type} has invalid type`);
await runCheck(` await findIssues(`
SELECT SELECT
attributeId attributeId,
attributes.noteId
FROM FROM
attributes attributes
LEFT JOIN notes ON attributes.noteId = notes.noteId AND notes.isDeleted = 0 LEFT JOIN notes ON attributes.noteId = notes.noteId AND notes.isDeleted = 0
WHERE WHERE
attributes.isDeleted = 0 attributes.isDeleted = 0
AND notes.noteId IS NULL`, AND notes.noteId IS NULL`,
"Attribute reference to the owning note is broken", errorList); ({attributeId, noteId}) => `Attribute id=${attributeId} reference to the owning note id=${noteId} is broken`);
await runCheck(` await findIssues(`
SELECT SELECT
attributeId attributeId,
value as targetNoteId
FROM FROM
attributes attributes
LEFT JOIN notes AS targetNote ON attributes.value = targetNote.noteId AND targetNote.isDeleted = 0 LEFT JOIN notes AS targetNote ON attributes.value = targetNote.noteId AND targetNote.isDeleted = 0
@ -275,9 +315,9 @@ async function runAllChecks() {
attributes.type = 'relation' attributes.type = 'relation'
AND attributes.isDeleted = 0 AND attributes.isDeleted = 0
AND targetNote.noteId IS NULL`, AND targetNote.noteId IS NULL`,
"Relation reference to the target note is broken", errorList); ({attributeId, targetNoteId}) => `Relation id=${attributeId} reference to the target note id=${targetNoteId} is broken`);
await runCheck(` await findIssues(`
SELECT SELECT
linkId linkId
FROM FROM
@ -286,52 +326,63 @@ async function runAllChecks() {
type != 'image' type != 'image'
AND type != 'hyper' AND type != 'hyper'
AND type != 'relation-map'`, AND type != 'relation-map'`,
"Link type is invalid", errorList); ({linkId, type}) => `Link id=${linkId} type=${type} is invalid`);
await runCheck(` await findIssues(`
SELECT SELECT
linkId linkId,
links.targetNoteId
FROM
links
LEFT JOIN notes AS targetNote ON targetNote.noteId = links.targetNoteId AND targetNote.isDeleted = 0
WHERE
links.isDeleted = 0
AND targetNote.noteId IS NULL`,
({linkId, targetNoteId}) => `Link id=${linkId} to target note id=${targetNoteId} is broken`);
await findIssues(`
SELECT
linkId,
links.noteId AS sourceNoteId
FROM FROM
links links
LEFT JOIN notes AS sourceNote ON sourceNote.noteId = links.noteId AND sourceNote.isDeleted = 0 LEFT JOIN notes AS sourceNote ON sourceNote.noteId = links.noteId AND sourceNote.isDeleted = 0
LEFT JOIN notes AS targetNote ON targetNote.noteId = links.noteId AND targetNote.isDeleted = 0
WHERE WHERE
links.isDeleted = 0 links.isDeleted = 0
AND (sourceNote.noteId IS NULL AND sourceNote.noteId IS NULL`,
OR targetNote.noteId IS NULL)`, ({linkId, sourceNoteId}) => `Link id=${linkId} to source note id=${sourceNoteId} is broken`);
"Link to source/target note link is broken", errorList);
await runSyncRowChecks("notes", "noteId", errorList); await runSyncRowChecks("notes", "noteId");
await runSyncRowChecks("note_revisions", "noteRevisionId", errorList); await runSyncRowChecks("note_revisions", "noteRevisionId");
await runSyncRowChecks("branches", "branchId", errorList); await runSyncRowChecks("branches", "branchId");
await runSyncRowChecks("recent_notes", "branchId", errorList); await runSyncRowChecks("recent_notes", "branchId");
await runSyncRowChecks("attributes", "attributeId", errorList); await runSyncRowChecks("attributes", "attributeId");
await runSyncRowChecks("api_tokens", "apiTokenId", errorList); await runSyncRowChecks("api_tokens", "apiTokenId");
await runSyncRowChecks("options", "name", errorList); await runSyncRowChecks("options", "name");
if (errorList.length === 0) { if (outstandingConsistencyErrors) {
// we run this only if basic checks passed since this assumes basic data consistency // we run this only if basic checks passed since this assumes basic data consistency
await checkTreeCycles(errorList); await checkTreeCycles();
} }
return errorList; return !outstandingConsistencyErrors;
} }
async function runChecks() { async function runChecks() {
let errorList;
let elapsedTimeMs; let elapsedTimeMs;
let dbConsistent;
await syncMutexService.doExclusively(async () => { await syncMutexService.doExclusively(async () => {
const startTime = new Date(); const startTime = new Date();
errorList = await runAllChecks(); dbConsistent = await runAllChecks();
elapsedTimeMs = new Date().getTime() - startTime.getTime(); elapsedTimeMs = new Date().getTime() - startTime.getTime();
}); });
if (errorList.length > 0) { if (!dbConsistent) {
log.info(`Consistency checks failed (took ${elapsedTimeMs}ms) with these errors: ` + JSON.stringify(errorList)); log.info(`Consistency checks failed (took ${elapsedTimeMs}ms)`);
messagingService.sendMessageToAllClients({type: 'consistency-checks-failed'}); messagingService.sendMessageToAllClients({type: 'consistency-checks-failed'});
} }
@ -347,6 +398,4 @@ sqlInit.dbReady.then(() => {
setTimeout(cls.wrap(runChecks), 10000); setTimeout(cls.wrap(runChecks), 10000);
}); });
module.exports = { module.exports = {};
runChecks
};