gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-06-06 18:08:33 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

run consistency checks on demand

Browse Source
This commit is contained in:
zadam 2019-12-10 22:03:00 +01:00
parent ee15db0ae1
commit dce54c7af3
4 changed files with 613 additions and 627 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/public/javascripts/dialogs/options/advanced.js
View File

@ -13,6 +13,10 @@ const TPL = `
<br/> <br/>
<br/> <br/>
<h4>Consistency checks</h4>
<button id="find-and-fix-consistency-issues-button" class="btn">Find and fix consistency issues</button><br/><br/>
<h4>Debugging</h4> <h4>Debugging</h4>
<button id="anonymize-button" class="btn">Save anonymized database</button><br/><br/> <button id="anonymize-button" class="btn">Save anonymized database</button><br/><br/>
@ -33,9 +37,8 @@ export default class AdvancedOptions {
this.$forceFullSyncButton = $("#force-full-sync-button"); this.$forceFullSyncButton = $("#force-full-sync-button");
this.$fillSyncRowsButton = $("#fill-sync-rows-button"); this.$fillSyncRowsButton = $("#fill-sync-rows-button");
this.$anonymizeButton = $("#anonymize-button"); this.$anonymizeButton = $("#anonymize-button");
this.$cleanupSoftDeletedButton = $("#cleanup-soft-deleted-items-button");
this.$cleanupUnusedImagesButton = $("#cleanup-unused-images-button");
this.$vacuumDatabaseButton = $("#vacuum-database-button"); this.$vacuumDatabaseButton = $("#vacuum-database-button");
this.$findAndFixConsistencyIssuesButton = $("#find-and-fix-consistency-issues-button");
this.$forceFullSyncButton.on('click', async () => { this.$forceFullSyncButton.on('click', async () => {
await server.post('sync/force-full-sync'); await server.post('sync/force-full-sync');
@ -55,26 +58,16 @@ export default class AdvancedOptions {
toastService.showMessage("Created anonymized database"); toastService.showMessage("Created anonymized database");
}); });
this.$cleanupSoftDeletedButton.on('click', async () => {
if (confirm("Do you really want to clean up soft-deleted items?")) {
await server.post('cleanup/cleanup-soft-deleted-items');
toastService.showMessage("Soft deleted items have been cleaned up");
}
});
this.$cleanupUnusedImagesButton.on('click', async () => {
if (confirm("Do you really want to clean up unused images?")) {
await server.post('cleanup/cleanup-unused-images');
toastService.showMessage("Unused images have been cleaned up");
}
});
this.$vacuumDatabaseButton.on('click', async () => { this.$vacuumDatabaseButton.on('click', async () => {
await server.post('cleanup/vacuum-database'); await server.post('cleanup/vacuum-database');
toastService.showMessage("Database has been vacuumed"); toastService.showMessage("Database has been vacuumed");
}); });
this.$findAndFixConsistencyIssuesButton.on('click', async () => {
await server.post('cleanup/find-and-fix-consistency-issues');
toastService.showMessage("Consistency issues should be fixed.");
});
} }
} }

8
src/routes/api/cleanup.js
View File

@ -2,6 +2,7 @@
const sql = require('../../services/sql'); const sql = require('../../services/sql');
const log = require('../../services/log'); const log = require('../../services/log');
const consistencyChecksService = require('../../services/consistency_checks');
async function vacuumDatabase() { async function vacuumDatabase() {
await sql.execute("VACUUM"); await sql.execute("VACUUM");
@ -9,6 +10,11 @@ async function vacuumDatabase() {
log.info("Database has been vacuumed."); log.info("Database has been vacuumed.");
} }
async function findAndFixConsistencyIssues() {
await consistencyChecksService.runOnDemandChecks(true);
}
module.exports = { module.exports = {
vacuumDatabase vacuumDatabase,
findAndFixConsistencyIssues
}; };

3
src/routes/routes.js
View File

@ -215,10 +215,11 @@ function register(app) {
apiRoute(POST, '/api/sql/execute', sqlRoute.execute); apiRoute(POST, '/api/sql/execute', sqlRoute.execute);
apiRoute(POST, '/api/anonymization/anonymize', anonymizationRoute.anonymize); apiRoute(POST, '/api/anonymization/anonymize', anonymizationRoute.anonymize);
apiRoute(POST, '/api/cleanup/cleanup-unused-images', cleanupRoute.cleanupUnusedImages);
// VACUUM requires execution outside of transaction // VACUUM requires execution outside of transaction
route(POST, '/api/cleanup/vacuum-database', [auth.checkApiAuthOrElectron, csrfMiddleware], cleanupRoute.vacuumDatabase, apiResultHandler, false); route(POST, '/api/cleanup/vacuum-database', [auth.checkApiAuthOrElectron, csrfMiddleware], cleanupRoute.vacuumDatabase, apiResultHandler, false);
route(POST, '/api/cleanup/find-and-fix-consistency-issues', [auth.checkApiAuthOrElectron, csrfMiddleware], cleanupRoute.findAndFixConsistencyIssues, apiResultHandler, false);
apiRoute(POST, '/api/script/exec', scriptRoute.exec); apiRoute(POST, '/api/script/exec', scriptRoute.exec);
apiRoute(POST, '/api/script/run/:noteId', scriptRoute.run); apiRoute(POST, '/api/script/run/:noteId', scriptRoute.run);
apiRoute(GET, '/api/script/startup', scriptRoute.getStartupBundles); apiRoute(GET, '/api/script/startup', scriptRoute.getStartupBundles);

532
src/services/consistency_checks.js
View File

@ -11,46 +11,47 @@ const syncTableService = require('./sync_table');
const optionsService = require('./options'); const optionsService = require('./options');
const Branch = require('../entities/branch'); const Branch = require('../entities/branch');
let unrecoveredConsistencyErrors = false; class ConsistencyChecks {
let fixedIssues = false; constructor(autoFix) {
this.autoFix = autoFix;
this.unrecoveredConsistencyErrors = false;
this.fixedIssues = false;
}
async function findIssues(query, errorCb) { async findIssues(query, errorCb) {
const results = await sql.getRows(query); const results = await sql.getRows(query);
for (const res of results) { for (const res of results) {
logError(errorCb(res)); logError(errorCb(res));
unrecoveredConsistencyErrors = true; this.unrecoveredConsistencyErrors = true;
} }
return results; return results;
} }
async function findAndFixIssues(query, fixerCb) { async findAndFixIssues(query, fixerCb) {
const results = await sql.getRows(query); const results = await sql.getRows(query);
for (const res of results) { for (const res of results) {
const autoFix = await optionsService.getOptionBool('autoFixConsistencyIssues');
try { try {
await fixerCb(res, autoFix); await fixerCb(res);
if (autoFix) { if (this.autoFix) {
fixedIssues = true; this.fixedIssues = true;
} else { } else {
unrecoveredConsistencyErrors = true; this.unrecoveredConsistencyErrors = true;
} }
} } catch (e) {
catch (e) {
logError(`Fixer failed with ${e.message} ${e.stack}`); logError(`Fixer failed with ${e.message} ${e.stack}`);
unrecoveredConsistencyErrors = true; this.unrecoveredConsistencyErrors = true;
} }
} }
return results; return results;
} }
async function checkTreeCycles() { async checkTreeCycles() {
const childToParents = {}; const childToParents = {};
const rows = await sql.getRows("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0"); const rows = await sql.getRows("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0");
@ -70,7 +71,7 @@ async function checkTreeCycles() {
if (!childToParents[noteId] || childToParents[noteId].length === 0) { if (!childToParents[noteId] || childToParents[noteId].length === 0) {
logError(`No parents found for note ${noteId}`); logError(`No parents found for note ${noteId}`);
unrecoveredConsistencyErrors = true; this.unrecoveredConsistencyErrors = true;
return; return;
} }
@ -78,9 +79,8 @@ async function checkTreeCycles() {
if (path.includes(parentNoteId)) { if (path.includes(parentNoteId)) {
logError(`Tree cycle detected at parent-child relationship: ${parentNoteId} - ${noteId}, whole path: ${path}`); logError(`Tree cycle detected at parent-child relationship: ${parentNoteId} - ${noteId}, whole path: ${path}`);
unrecoveredConsistencyErrors = true; this.unrecoveredConsistencyErrors = true;
} } else {
else {
const newPath = path.slice(); const newPath = path.slice();
newPath.push(noteId); newPath.push(noteId);
@ -97,148 +97,145 @@ async function checkTreeCycles() {
if (childToParents['root'].length !== 1 || childToParents['root'][0] !== 'none') { if (childToParents['root'].length !== 1 || childToParents['root'][0] !== 'none') {
logError('Incorrect root parent: ' + JSON.stringify(childToParents['root'])); logError('Incorrect root parent: ' + JSON.stringify(childToParents['root']));
unrecoveredConsistencyErrors = true; this.unrecoveredConsistencyErrors = true;
}
} }
}
async function findBrokenReferenceIssues() { async findBrokenReferenceIssues() {
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT branchId, branches.noteId SELECT branchId, branches.noteId
FROM branches LEFT JOIN notes USING(noteId) FROM branches
WHERE branches.isDeleted = 0 AND notes.noteId IS NULL`, LEFT JOIN notes USING (noteId)
async ({branchId, noteId}, autoFix) => { WHERE branches.isDeleted = 0
if (autoFix) { AND notes.noteId IS NULL`,
async ({branchId, noteId}) => {
if (this.autoFix) {
const branch = await repository.getBranch(branchId); const branch = await repository.getBranch(branchId);
branch.isDeleted = true; branch.isDeleted = true;
await branch.save(); await branch.save();
logFix(`Branch ${branchId} has been deleted since it references missing note ${noteId}`); logFix(`Branch ${branchId} has been deleted since it references missing note ${noteId}`);
} } else {
else {
logError(`Branch ${branchId} references missing note ${noteId}`); logError(`Branch ${branchId} references missing note ${noteId}`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT branchId, branches.noteId AS parentNoteId SELECT branchId, branches.noteId AS parentNoteId
FROM branches LEFT JOIN notes ON notes.noteId = branches.parentNoteId FROM branches
WHERE branches.isDeleted = 0 AND branches.branchId != 'root' AND notes.noteId IS NULL`, LEFT JOIN notes ON notes.noteId = branches.parentNoteId
async ({branchId, parentNoteId}, autoFix) => { WHERE branches.isDeleted = 0
if (autoFix) { AND branches.branchId != 'root'
AND notes.noteId IS NULL`,
async ({branchId, parentNoteId}) => {
if (this.autoFix) {
const branch = await repository.getBranch(branchId); const branch = await repository.getBranch(branchId);
branch.parentNoteId = 'root'; branch.parentNoteId = 'root';
await branch.save(); await branch.save();
logFix(`Branch ${branchId} was set to root parent since it was referencing missing parent note ${parentNoteId}`); logFix(`Branch ${branchId} was set to root parent since it was referencing missing parent note ${parentNoteId}`);
} } else {
else {
logError(`Branch ${branchId} references missing parent note ${parentNoteId}`); logError(`Branch ${branchId} references missing parent note ${parentNoteId}`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT attributeId, attributes.noteId SELECT attributeId, attributes.noteId
FROM attributes LEFT JOIN notes USING(noteId) FROM attributes
WHERE attributes.isDeleted = 0 AND notes.noteId IS NULL`, LEFT JOIN notes USING (noteId)
async ({attributeId, noteId}, autoFix) => { WHERE attributes.isDeleted = 0
if (autoFix) { AND notes.noteId IS NULL`,
async ({attributeId, noteId}) => {
if (this.autoFix) {
const attribute = await repository.getAttribute(attributeId); const attribute = await repository.getAttribute(attributeId);
attribute.isDeleted = true; attribute.isDeleted = true;
await attribute.save(); await attribute.save();
logFix(`Attribute ${attributeId} has been deleted since it references missing source note ${noteId}`); logFix(`Attribute ${attributeId} has been deleted since it references missing source note ${noteId}`);
} } else {
else {
logError(`Attribute ${attributeId} references missing source note ${noteId}`); logError(`Attribute ${attributeId} references missing source note ${noteId}`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT attributeId, attributes.value AS noteId SELECT attributeId, attributes.value AS noteId
FROM attributes LEFT JOIN notes ON notes.noteId = attributes.value FROM attributes
WHERE attributes.isDeleted = 0 AND attributes.type = 'relation' LEFT JOIN notes ON notes.noteId = attributes.value
WHERE attributes.isDeleted = 0
AND attributes.type = 'relation'
AND notes.noteId IS NULL`, AND notes.noteId IS NULL`,
async ({attributeId, noteId}, autoFix) => { async ({attributeId, noteId}) => {
if (autoFix) { if (this.autoFix) {
const attribute = await repository.getAttribute(attributeId); const attribute = await repository.getAttribute(attributeId);
attribute.isDeleted = true; attribute.isDeleted = true;
await attribute.save(); await attribute.save();
logFix(`Relation ${attributeId} has been deleted since it references missing note ${noteId}`) logFix(`Relation ${attributeId} has been deleted since it references missing note ${noteId}`)
} } else {
else {
logError(`Relation ${attributeId} references missing note ${noteId}`) logError(`Relation ${attributeId} references missing note ${noteId}`)
} }
}); });
await findIssues(` await this.findIssues(`
SELECT noteRevisionId, note_revisions.noteId SELECT noteRevisionId, note_revisions.noteId
FROM note_revisions LEFT JOIN notes USING(noteId) FROM note_revisions
LEFT JOIN notes USING (noteId)
WHERE notes.noteId IS NULL`, WHERE notes.noteId IS NULL`,
({noteRevisionId, noteId}) => `Note revision ${noteRevisionId} references missing note ${noteId}`); ({noteRevisionId, noteId}) => `Note revision ${noteRevisionId} references missing note ${noteId}`);
} }
async function findExistencyIssues() { async findExistencyIssues() {
// principle for fixing inconsistencies is that if the note itself is deleted (isDeleted=true) then all related entities should be also deleted (branches, attributes) // principle for fixing inconsistencies is that if the note itself is deleted (isDeleted=true) then all related entities should be also deleted (branches, attributes)
// but if note is not deleted, then at least one branch should exist. // but if note is not deleted, then at least one branch should exist.
// the order here is important - first we might need to delete inconsistent branches and after that // the order here is important - first we might need to delete inconsistent branches and after that
// another check might create missing branch // another check might create missing branch
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT branchId,
branchId, noteId noteId
FROM FROM branches
branches JOIN notes USING (noteId)
JOIN notes USING(noteId) WHERE notes.isDeleted = 1
WHERE
notes.isDeleted = 1
AND branches.isDeleted = 0`, AND branches.isDeleted = 0`,
async ({branchId, noteId}, autoFix) => { async ({branchId, noteId}) => {
if (autoFix) { if (this.autoFix) {
const branch = await repository.getBranch(branchId); const branch = await repository.getBranch(branchId);
branch.isDeleted = true; branch.isDeleted = true;
await branch.save(); await branch.save();
logFix(`Branch ${branchId} has been deleted since associated note ${noteId} is deleted.`); logFix(`Branch ${branchId} has been deleted since associated note ${noteId} is deleted.`);
} } else {
else {
logError(`Branch ${branchId} is not deleted even though associated note ${noteId} is deleted.`) logError(`Branch ${branchId} is not deleted even though associated note ${noteId} is deleted.`)
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT branchId,
branchId, parentNoteId parentNoteId
FROM FROM branches
branches
JOIN notes AS parentNote ON parentNote.noteId = branches.parentNoteId JOIN notes AS parentNote ON parentNote.noteId = branches.parentNoteId
WHERE WHERE parentNote.isDeleted = 1
parentNote.isDeleted = 1
AND branches.isDeleted = 0 AND branches.isDeleted = 0
`, async ({branchId, parentNoteId}, autoFix) => { `, async ({branchId, parentNoteId}) => {
if (autoFix) { if (this.autoFix) {
const branch = await repository.getBranch(branchId); const branch = await repository.getBranch(branchId);
branch.isDeleted = true; branch.isDeleted = true;
await branch.save(); await branch.save();
logFix(`Branch ${branchId} has been deleted since associated parent note ${parentNoteId} is deleted.`); logFix(`Branch ${branchId} has been deleted since associated parent note ${parentNoteId} is deleted.`);
} } else {
else {
logError(`Branch ${branchId} is not deleted even though associated parent note ${parentNoteId} is deleted.`) logError(`Branch ${branchId} is not deleted even though associated parent note ${parentNoteId} is deleted.`)
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT DISTINCT notes.noteId
DISTINCT notes.noteId FROM notes
FROM
notes
LEFT JOIN branches ON notes.noteId = branches.noteId AND branches.isDeleted = 0 LEFT JOIN branches ON notes.noteId = branches.noteId AND branches.isDeleted = 0
WHERE WHERE notes.isDeleted = 0
notes.isDeleted = 0
AND branches.branchId IS NULL AND branches.branchId IS NULL
`, async ({noteId}, autoFix) => { `, async ({noteId}) => {
if (autoFix) { if (this.autoFix) {
const branch = await new Branch({ const branch = await new Branch({
parentNoteId: 'root', parentNoteId: 'root',
noteId: noteId, noteId: noteId,
@ -246,27 +243,22 @@ async function findExistencyIssues() {
}).save(); }).save();
logFix(`Created missing branch ${branch.branchId} for note ${noteId}`); logFix(`Created missing branch ${branch.branchId} for note ${noteId}`);
} } else {
else {
logError(`No undeleted branch found for note ${noteId}`); logError(`No undeleted branch found for note ${noteId}`);
} }
}); });
// there should be a unique relationship between note and its parent // there should be a unique relationship between note and its parent
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT noteId,
noteId, parentNoteId parentNoteId
FROM FROM branches
branches WHERE branches.isDeleted = 0
WHERE GROUP BY branches.parentNoteId,
branches.isDeleted = 0
GROUP BY
branches.parentNoteId,
branches.noteId branches.noteId
HAVING HAVING COUNT(1) > 1`,
COUNT(1) > 1`, async ({noteId, parentNoteId}) => {
async ({noteId, parentNoteId}, autoFix) => { if (this.autoFix) {
if (autoFix) {
const branches = await repository.getEntities( const branches = await repository.getEntities(
`SELECT * `SELECT *
FROM branches FROM branches
@ -284,170 +276,156 @@ async function findExistencyIssues() {
logFix(`Removing branch ${branch.branchId} since it's parent-child duplicate of branch ${origBranch.branchId}`); logFix(`Removing branch ${branch.branchId} since it's parent-child duplicate of branch ${origBranch.branchId}`);
} }
} } else {
else {
logError(`Duplicate branches for note ${noteId} and parent ${parentNoteId}`); logError(`Duplicate branches for note ${noteId} and parent ${parentNoteId}`);
} }
}); });
} }
async function findLogicIssues() { async findLogicIssues() {
await findAndFixIssues( ` await this.findAndFixIssues(`
SELECT noteId, type SELECT noteId, type
FROM notes FROM notes
WHERE WHERE isDeleted = 0
isDeleted = 0
AND type NOT IN ('text', 'code', 'render', 'file', 'image', 'search', 'relation-map', 'book')`, AND type NOT IN ('text', 'code', 'render', 'file', 'image', 'search', 'relation-map', 'book')`,
async ({noteId, type}, autoFix) => { async ({noteId, type}) => {
if (autoFix) { if (this.autoFix) {
const note = await repository.getNote(noteId); const note = await repository.getNote(noteId);
note.type = 'file'; // file is a safe option to recover notes if type is not known note.type = 'file'; // file is a safe option to recover notes if type is not known
await note.save(); await note.save();
logFix(`Note ${noteId} type has been change to file since it had invalid type=${type}`) logFix(`Note ${noteId} type has been change to file since it had invalid type=${type}`)
} } else {
else {
logError(`Note ${noteId} has invalid type=${type}`); logError(`Note ${noteId} has invalid type=${type}`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT notes.noteId SELECT notes.noteId
FROM notes FROM notes
LEFT JOIN note_contents USING(noteId) LEFT JOIN note_contents USING (noteId)
WHERE WHERE note_contents.noteId IS NULL`,
note_contents.noteId IS NULL`, async ({noteId}) => {
async ({noteId}, autoFix) => { if (this.autoFix) {
if (autoFix) {
const note = await repository.getNote(noteId); const note = await repository.getNote(noteId);
// empty string might be wrong choice for some note types (and protected notes) but it's a best guess // empty string might be wrong choice for some note types (and protected notes) but it's a best guess
await note.setContent(note.isErased ? null : ''); await note.setContent(note.isErased ? null : '');
logFix(`Note ${noteId} content was set to empty string since there was no corresponding row`); logFix(`Note ${noteId} content was set to empty string since there was no corresponding row`);
} } else {
else {
logError(`Note ${noteId} content row does not exist`); logError(`Note ${noteId} content row does not exist`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT noteId SELECT noteId
FROM notes FROM notes
JOIN note_contents USING(noteId) JOIN note_contents USING (noteId)
WHERE WHERE isDeleted = 0
isDeleted = 0
AND content IS NULL`, AND content IS NULL`,
async ({noteId}, autoFix) => { async ({noteId}) => {
if (autoFix) { if (this.autoFix) {
const note = await repository.getNote(noteId); const note = await repository.getNote(noteId);
// empty string might be wrong choice for some note types (and protected notes) but it's a best guess // empty string might be wrong choice for some note types (and protected notes) but it's a best guess
await note.setContent(''); await note.setContent('');
logFix(`Note ${noteId} content was set to empty string since it was null even though it is not deleted`); logFix(`Note ${noteId} content was set to empty string since it was null even though it is not deleted`);
} } else {
else {
logError(`Note ${noteId} content is null even though it is not deleted`); logError(`Note ${noteId} content is null even though it is not deleted`);
} }
}); });
await findIssues(` await this.findIssues(`
SELECT noteId SELECT noteId
FROM notes FROM notes
JOIN note_contents USING(noteId) JOIN note_contents USING (noteId)
WHERE WHERE isErased = 1
isErased = 1
AND content IS NOT NULL`, AND content IS NOT NULL`,
({noteId}) => `Note ${noteId} content is not null even though the note is erased`); ({noteId}) => `Note ${noteId} content is not null even though the note is erased`);
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT noteId, noteRevisionId SELECT noteId, noteRevisionId
FROM notes FROM notes
JOIN note_revisions USING(noteId) JOIN note_revisions USING (noteId)
WHERE WHERE notes.isErased = 1
notes.isErased = 1
AND note_revisions.isErased = 0`, AND note_revisions.isErased = 0`,
async ({noteId, noteRevisionId}, autoFix) => { async ({noteId, noteRevisionId}) => {
if (autoFix) { if (this.autoFix) {
const noteRevision = await repository.getNoteRevision(noteRevisionId); const noteRevision = await repository.getNoteRevision(noteRevisionId);
noteRevision.isErased = true; noteRevision.isErased = true;
await noteRevision.setContent(null); await noteRevision.setContent(null);
await noteRevision.save(); await noteRevision.save();
logFix(`Note revision ${noteRevisionId} has been erased since its note ${noteId} is also erased.`); logFix(`Note revision ${noteRevisionId} has been erased since its note ${noteId} is also erased.`);
} } else {
else {
logError(`Note revision ${noteRevisionId} is not erased even though note ${noteId} is erased.`); logError(`Note revision ${noteRevisionId} is not erased even though note ${noteId} is erased.`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT note_revisions.noteRevisionId SELECT note_revisions.noteRevisionId
FROM note_revisions FROM note_revisions
LEFT JOIN note_revision_contents USING(noteRevisionId) LEFT JOIN note_revision_contents USING (noteRevisionId)
WHERE note_revision_contents.noteRevisionId IS NULL`, WHERE note_revision_contents.noteRevisionId IS NULL`,
async ({noteRevisionId}, autoFix) => { async ({noteRevisionId}) => {
if (autoFix) { if (this.autoFix) {
const noteRevision = await repository.getNoteRevision(noteRevisionId); const noteRevision = await repository.getNoteRevision(noteRevisionId);
await noteRevision.setContent(null); await noteRevision.setContent(null);
noteRevision.isErased = true; noteRevision.isErased = true;
await noteRevision.save(); await noteRevision.save();
logFix(`Note revision content ${noteRevisionId} was created and set to erased since it did not exist.`); logFix(`Note revision content ${noteRevisionId} was created and set to erased since it did not exist.`);
} } else {
else {
logError(`Note revision content ${noteRevisionId} does not exist`); logError(`Note revision content ${noteRevisionId} does not exist`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT noteRevisionId SELECT noteRevisionId
FROM note_revisions FROM note_revisions
JOIN note_revision_contents USING(noteRevisionId) JOIN note_revision_contents USING (noteRevisionId)
WHERE WHERE isErased = 0
isErased = 0
AND content IS NULL`, AND content IS NULL`,
async ({noteRevisionId}, autoFix) => { async ({noteRevisionId}) => {
if (autoFix) { if (this.autoFix) {
const noteRevision = await repository.getNoteRevision(noteRevisionId); const noteRevision = await repository.getNoteRevision(noteRevisionId);
noteRevision.isErased = true; noteRevision.isErased = true;
await noteRevision.save(); await noteRevision.save();
logFix(`Note revision ${noteRevisionId} content was set to empty string since it was null even though it is not erased`); logFix(`Note revision ${noteRevisionId} content was set to empty string since it was null even though it is not erased`);
} } else {
else {
logError(`Note revision ${noteRevisionId} content is null even though it is not erased`); logError(`Note revision ${noteRevisionId} content is null even though it is not erased`);
} }
}); });
await findIssues(` await this.findIssues(`
SELECT noteRevisionId SELECT noteRevisionId
FROM note_revisions FROM note_revisions
JOIN note_revision_contents USING(noteRevisionId) JOIN note_revision_contents USING (noteRevisionId)
WHERE WHERE isErased = 1
isErased = 1
AND content IS NOT NULL`, AND content IS NOT NULL`,
({noteRevisionId}) => `Note revision ${noteRevisionId} content is not null even though the note revision is erased`); ({noteRevisionId}) => `Note revision ${noteRevisionId} content is not null even though the note revision is erased`);
await findIssues(` await this.findIssues(`
SELECT noteId SELECT noteId
FROM notes FROM notes
WHERE WHERE isErased = 1
isErased = 1
AND isDeleted = 0`, AND isDeleted = 0`,
({noteId}) => `Note ${noteId} is not deleted even though it is erased`); ({noteId}) => `Note ${noteId} is not deleted even though it is erased`);
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT parentNoteId SELECT parentNoteId
FROM FROM branches
branches
JOIN notes ON notes.noteId = branches.parentNoteId JOIN notes ON notes.noteId = branches.parentNoteId
WHERE WHERE notes.isDeleted = 0
notes.isDeleted = 0
AND notes.type == 'search' AND notes.type == 'search'
AND branches.isDeleted = 0`, AND branches.isDeleted = 0`,
async ({parentNoteId}, autoFix) => { async ({parentNoteId}) => {
if (autoFix) { if (this.autoFix) {
const branches = await repository.getEntities(`SELECT * FROM branches WHERE isDeleted = 0 AND parentNoteId = ?`, [parentNoteId]); const branches = await repository.getEntities(`SELECT *
FROM branches
WHERE isDeleted = 0
AND parentNoteId = ?`, [parentNoteId]);
for (const branch of branches) { for (const branch of branches) {
branch.parentNoteId = 'root'; branch.parentNoteId = 'root';
@ -455,106 +433,92 @@ async function findLogicIssues() {
logFix(`Child branch ${branch.branchId} has been moved to root since it was a child of a search note ${parentNoteId}`) logFix(`Child branch ${branch.branchId} has been moved to root since it was a child of a search note ${parentNoteId}`)
} }
} } else {
else {
logError(`Search note ${parentNoteId} has children`); logError(`Search note ${parentNoteId} has children`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT attributeId SELECT attributeId
FROM attributes FROM attributes
WHERE WHERE isDeleted = 0
isDeleted = 0
AND type = 'relation' AND type = 'relation'
AND value = ''`, AND value = ''`,
async ({attributeId}, autoFix) => { async ({attributeId}) => {
if (autoFix) { if (this.autoFix) {
const relation = await repository.getAttribute(attributeId); const relation = await repository.getAttribute(attributeId);
relation.isDeleted = true; relation.isDeleted = true;
await relation.save(); await relation.save();
logFix(`Removed relation ${relation.attributeId} of name "${relation.name} with empty target.`); logFix(`Removed relation ${relation.attributeId} of name "${relation.name} with empty target.`);
} } else {
else {
logError(`Relation ${attributeId} has empty target.`); logError(`Relation ${attributeId} has empty target.`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT attributeId,
attributeId,
type type
FROM attributes FROM attributes
WHERE WHERE isDeleted = 0
isDeleted = 0
AND type != 'label' AND type != 'label'
AND type != 'label-definition' AND type != 'label-definition'
AND type != 'relation' AND type != 'relation'
AND type != 'relation-definition'`, AND type != 'relation-definition'`,
async ({attributeId, type}, autoFix) => { async ({attributeId, type}) => {
if (autoFix) { if (this.autoFix) {
const attribute = await repository.getAttribute(attributeId); const attribute = await repository.getAttribute(attributeId);
attribute.type = 'label'; attribute.type = 'label';
await attribute.save(); await attribute.save();
logFix(`Attribute ${attributeId} type was changed to label since it had invalid type '${type}'`); logFix(`Attribute ${attributeId} type was changed to label since it had invalid type '${type}'`);
} } else {
else {
logError(`Attribute ${attributeId} has invalid type '${type}'`); logError(`Attribute ${attributeId} has invalid type '${type}'`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT attributeId,
attributeId,
attributes.noteId attributes.noteId
FROM FROM attributes
attributes
JOIN notes ON attributes.noteId = notes.noteId JOIN notes ON attributes.noteId = notes.noteId
WHERE WHERE attributes.isDeleted = 0
attributes.isDeleted = 0
AND notes.isDeleted = 1`, AND notes.isDeleted = 1`,
async ({attributeId, noteId}, autoFix) => { async ({attributeId, noteId}) => {
if (autoFix) { if (this.autoFix) {
const attribute = await repository.getAttribute(attributeId); const attribute = await repository.getAttribute(attributeId);
attribute.isDeleted = true; attribute.isDeleted = true;
await attribute.save(); await attribute.save();
logFix(`Removed attribute ${attributeId} because owning note ${noteId} is also deleted.`); logFix(`Removed attribute ${attributeId} because owning note ${noteId} is also deleted.`);
} } else {
else {
logError(`Attribute ${attributeId} is not deleted even though owning note ${noteId} is deleted.`); logError(`Attribute ${attributeId} is not deleted even though owning note ${noteId} is deleted.`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT attributeId,
attributeId,
attributes.value AS targetNoteId attributes.value AS targetNoteId
FROM FROM attributes
attributes
JOIN notes ON attributes.value = notes.noteId JOIN notes ON attributes.value = notes.noteId
WHERE WHERE attributes.type = 'relation'
attributes.type = 'relation'
AND attributes.isDeleted = 0 AND attributes.isDeleted = 0
AND notes.isDeleted = 1`, AND notes.isDeleted = 1`,
async ({attributeId, targetNoteId}, autoFix) => { async ({attributeId, targetNoteId}) => {
if (autoFix) { if (this.autoFix) {
const attribute = await repository.getAttribute(attributeId); const attribute = await repository.getAttribute(attributeId);
attribute.isDeleted = true; attribute.isDeleted = true;
await attribute.save(); await attribute.save();
logFix(`Removed attribute ${attributeId} because target note ${targetNoteId} is also deleted.`); logFix(`Removed attribute ${attributeId} because target note ${targetNoteId} is also deleted.`);
} } else {
else {
logError(`Attribute ${attributeId} is not deleted even though target note ${targetNoteId} is deleted.`); logError(`Attribute ${attributeId} is not deleted even though target note ${targetNoteId} is deleted.`);
} }
}); });
} }
async function runSyncRowChecks(entityName, key) { async runSyncRowChecks(entityName, key) {
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT
${key} as entityId ${key} as entityId
FROM FROM
@ -562,18 +526,17 @@ async function runSyncRowChecks(entityName, key) {
LEFT JOIN sync ON sync.entityName = '${entityName}' AND entityId = ${key} LEFT JOIN sync ON sync.entityName = '${entityName}' AND entityId = ${key}
WHERE WHERE
sync.id IS NULL AND ` + (entityName === 'options' ? 'isSynced = 1' : '1'), sync.id IS NULL AND ` + (entityName === 'options' ? 'isSynced = 1' : '1'),
async ({entityId}, autoFix) => { async ({entityId}) => {
if (autoFix) { if (this.autoFix) {
await syncTableService.addEntitySync(entityName, entityId); await syncTableService.addEntitySync(entityName, entityId);
logFix(`Created missing sync record for entityName=${entityName}, entityId=${entityId}`); logFix(`Created missing sync record for entityName=${entityName}, entityId=${entityId}`);
} } else {
else {
logError(`Missing sync record for entityName=${entityName}, entityId=${entityId}`); logError(`Missing sync record for entityName=${entityName}, entityId=${entityId}`);
} }
}); });
await findAndFixIssues(` await this.findAndFixIssues(`
SELECT SELECT
id, entityId id, entityId
FROM FROM
@ -582,92 +545,101 @@ async function runSyncRowChecks(entityName, key) {
WHERE WHERE
sync.entityName = '${entityName}' sync.entityName = '${entityName}'
AND ${key} IS NULL`, AND ${key} IS NULL`,
async ({id, entityId}, autoFix) => { async ({id, entityId}) => {
if (autoFix) { if (this.autoFix) {
await sql.execute("DELETE FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]); await sql.execute("DELETE FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
logFix(`Deleted extra sync record id=${id}, entityName=${entityName}, entityId=${entityId}`); logFix(`Deleted extra sync record id=${id}, entityName=${entityName}, entityId=${entityId}`);
} } else {
else {
logError(`Unrecognized sync record id=${id}, entityName=${entityName}, entityId=${entityId}`); logError(`Unrecognized sync record id=${id}, entityName=${entityName}, entityId=${entityId}`);
} }
}); });
}
async function findSyncRowsIssues() {
await runSyncRowChecks("notes", "noteId");
await runSyncRowChecks("note_contents", "noteId");
await runSyncRowChecks("note_revisions", "noteRevisionId");
await runSyncRowChecks("branches", "branchId");
await runSyncRowChecks("recent_notes", "noteId");
await runSyncRowChecks("attributes", "attributeId");
await runSyncRowChecks("api_tokens", "apiTokenId");
await runSyncRowChecks("options", "name");
}
async function runAllChecks() {
unrecoveredConsistencyErrors = false;
fixedIssues = false;
await findBrokenReferenceIssues();
await findExistencyIssues();
await findLogicIssues();
await findSyncRowsIssues();
if (unrecoveredConsistencyErrors) {
// we run this only if basic checks passed since this assumes basic data consistency
await checkTreeCycles();
} }
return !unrecoveredConsistencyErrors; async findSyncRowsIssues() {
} await this.runSyncRowChecks("notes", "noteId");
await this.runSyncRowChecks("note_contents", "noteId");
await this.runSyncRowChecks("note_revisions", "noteRevisionId");
await this.runSyncRowChecks("branches", "branchId");
await this.runSyncRowChecks("recent_notes", "noteId");
await this.runSyncRowChecks("attributes", "attributeId");
await this.runSyncRowChecks("api_tokens", "apiTokenId");
await this.runSyncRowChecks("options", "name");
}
async function showEntityStat(name, query) { async runAllChecks() {
this.unrecoveredConsistencyErrors = false;
this.fixedIssues = false;
await this.findBrokenReferenceIssues();
await this.findExistencyIssues();
await this.findLogicIssues();
await this.findSyncRowsIssues();
if (this.unrecoveredConsistencyErrors) {
// we run this only if basic checks passed since this assumes basic data consistency
await this.checkTreeCycles();
}
return !this.unrecoveredConsistencyErrors;
}
async showEntityStat(name, query) {
const map = await sql.getMap(query); const map = await sql.getMap(query);
map[0] = map[0] || 0; map[0] = map[0] || 0;
map[1] = map[1] || 0; map[1] = map[1] || 0;
log.info(`${name} deleted: ${map[1]}, not deleted ${map[0]}`); log.info(`${name} deleted: ${map[1]}, not deleted ${map[0]}`);
} }
async function runDbDiagnostics() { async runDbDiagnostics() {
await showEntityStat("Notes", `SELECT isDeleted, count(1) FROM notes GROUP BY isDeleted`); await this.showEntityStat("Notes", `SELECT isDeleted, count(1)
await showEntityStat("Note revisions", `SELECT isErased, count(1) FROM note_revisions GROUP BY isErased`); FROM notes
await showEntityStat("Branches", `SELECT isDeleted, count(1) FROM branches GROUP BY isDeleted`); GROUP BY isDeleted`);
await showEntityStat("Attributes", `SELECT isDeleted, count(1) FROM attributes GROUP BY isDeleted`); await this.showEntityStat("Note revisions", `SELECT isErased, count(1)
await showEntityStat("API tokens", `SELECT isDeleted, count(1) FROM api_tokens GROUP BY isDeleted`); FROM note_revisions
} GROUP BY isErased`);
await this.showEntityStat("Branches", `SELECT isDeleted, count(1)
FROM branches
GROUP BY isDeleted`);
await this.showEntityStat("Attributes", `SELECT isDeleted, count(1)
FROM attributes
GROUP BY isDeleted`);
await this.showEntityStat("API tokens", `SELECT isDeleted, count(1)
FROM api_tokens
GROUP BY isDeleted`);
}
async function runChecks() { async runChecks() {
let elapsedTimeMs; let elapsedTimeMs;
await syncMutexService.doExclusively(async () => { await syncMutexService.doExclusively(async () => {
const startTime = new Date(); const startTime = new Date();
await runDbDiagnostics(); await this.runDbDiagnostics();
await runAllChecks(); await this.runAllChecks();
elapsedTimeMs = Date.now() - startTime.getTime(); elapsedTimeMs = Date.now() - startTime.getTime();
}); });
if (fixedIssues) { if (this.fixedIssues) {
ws.refreshTree(); ws.refreshTree();
} }
if (unrecoveredConsistencyErrors) { if (this.unrecoveredConsistencyErrors) {
log.info(`Consistency checks failed (took ${elapsedTimeMs}ms)`); log.info(`Consistency checks failed (took ${elapsedTimeMs}ms)`);
ws.sendMessageToAllClients({type: 'consistency-checks-failed'}); ws.sendMessageToAllClients({type: 'consistency-checks-failed'});
} } else {
else {
log.info(`All consistency checks passed (took ${elapsedTimeMs}ms)`); log.info(`All consistency checks passed (took ${elapsedTimeMs}ms)`);
} }
}
} }
function logFix(message) { function logFix(message) {
@ -678,11 +650,25 @@ function logError(message) {
log.info("Consistency error: " + message); log.info("Consistency error: " + message);
} }
async function runPeriodicChecks() {
const autoFix = await optionsService.getOptionBool('autoFixConsistencyIssues');
const consistencyChecks = new ConsistencyChecks(autoFix);
await consistencyChecks.runChecks();
}
async function runOnDemandChecks(autoFix) {
const consistencyChecks = new ConsistencyChecks(autoFix);
await consistencyChecks.runChecks();
}
sqlInit.dbReady.then(() => { sqlInit.dbReady.then(() => {
setInterval(cls.wrap(runChecks), 60 * 60 * 1000); setInterval(cls.wrap(runPeriodicChecks), 60 * 60 * 1000);
// kickoff checks soon after startup (to not block the initial load) // kickoff checks soon after startup (to not block the initial load)
setTimeout(cls.wrap(runChecks), 20 * 1000); setTimeout(cls.wrap(runPeriodicChecks), 20 * 1000);
}); });
module.exports = {}; module.exports = {
runOnDemandChecks
};