From aa233b8adbb51d4b0e56f6b0db4cfa4284b42b7f Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Wed, 3 Apr 2024 20:47:41 +0300 Subject: [PATCH] server-ts: Port consistency_checks --- src/app.js | 10 +- src/routes/api/database.js | 2 +- src/routes/api/sync.js | 6 +- ...stency_checks.js => consistency_checks.ts} | 136 +++++++++++++----- src/services/entity_changes_interface.ts | 2 + src/services/sql_init.ts | 2 +- src/services/sync.ts | 2 +- 7 files changed, 111 insertions(+), 49 deletions(-) rename src/services/{consistency_checks.js => consistency_checks.ts} (88%) diff --git a/src/app.js b/src/app.js index 2dad38c7d..49373545e 100644 --- a/src/app.js +++ b/src/app.js @@ -26,10 +26,10 @@ app.use(helmet({ crossOriginEmbedderPolicy: false })); -app.use(express.text({limit: '500mb'})); -app.use(express.json({limit: '500mb'})); -app.use(express.raw({limit: '500mb'})); -app.use(express.urlencoded({extended: false})); +app.use(express.text({ limit: '500mb' })); +app.use(express.json({ limit: '500mb' })); +app.use(express.raw({ limit: '500mb' })); +app.use(express.urlencoded({ extended: false })); app.use(cookieParser()); app.use(express.static(path.join(__dirname, 'public/root'))); app.use(`/manifest.webmanifest`, express.static(path.join(__dirname, 'public/manifest.webmanifest'))); @@ -49,7 +49,7 @@ require('./services/sync'); require('./services/backup'); // trigger consistency checks timer -require('./services/consistency_checks.js'); +require('./services/consistency_checks'); require('./services/scheduler.js'); diff --git a/src/routes/api/database.js b/src/routes/api/database.js index 4ea27e8ad..d8d8cfa9a 100644 --- a/src/routes/api/database.js +++ b/src/routes/api/database.js @@ -4,7 +4,7 @@ const sql = require('../../services/sql'); const log = require('../../services/log'); const backupService = require('../../services/backup'); const anonymizationService = require('../../services/anonymization'); -const consistencyChecksService = require('../../services/consistency_checks.js'); +const consistencyChecksService = require('../../services/consistency_checks'); function getExistingBackups() { return backupService.getExistingBackups(); diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js index c1939552e..bd38e2905 100644 --- a/src/routes/api/sync.js +++ b/src/routes/api/sync.js @@ -132,7 +132,7 @@ function getChanged(req) { const partialRequests = {}; function update(req) { - let {body} = req; + let { body } = req; const pageCount = parseInt(req.get('pageCount')); const pageIndex = parseInt(req.get('pageIndex')); @@ -164,7 +164,7 @@ function update(req) { } } - const {entities, instanceId} = body; + const { entities, instanceId } = body; sql.transactional(() => syncUpdateService.updateEntities(entities, instanceId)); } @@ -193,7 +193,7 @@ function queueSector(req) { } function checkEntityChanges() { - require('../../services/consistency_checks.js').runEntityChangesChecks(); + require('../../services/consistency_checks').runEntityChangesChecks(); } module.exports = { diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.ts similarity index 88% rename from src/services/consistency_checks.js rename to src/services/consistency_checks.ts index 2e2d6385b..81ce810ca 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.ts @@ -1,33 +1,42 @@ "use strict"; -const sql = require('./sql'); -const sqlInit = require('./sql_init'); -const log = require('./log'); -const ws = require('./ws'); -const syncMutexService = require('./sync_mutex'); -const cls = require('./cls'); -const entityChangesService = require('./entity_changes'); -const optionsService = require('./options'); -const BBranch = require('../becca/entities/bbranch'); -const revisionService = require('./revisions'); -const becca = require('../becca/becca'); -const utils = require('../services/utils'); -const eraseService = require('../services/erase'); -const {sanitizeAttributeName} = require('./sanitize_attribute_name'); -const noteTypes = require('../services/note_types').getNoteTypeNames(); +import sql = require('./sql'); +import sqlInit = require('./sql_init'); +import log = require('./log'); +import ws = require('./ws'); +import syncMutexService = require('./sync_mutex'); +import cls = require('./cls'); +import entityChangesService = require('./entity_changes'); +import optionsService = require('./options'); +import BBranch = require('../becca/entities/bbranch'); +import revisionService = require('./revisions'); +import becca = require('../becca/becca'); +import utils = require('../services/utils'); +import eraseService = require('../services/erase'); +import sanitizeAttributeName = require('./sanitize_attribute_name'); +import noteTypesService = require('../services/note_types'); +import { BranchRow, NoteRow } from '../becca/entities/rows'; +import { EntityChange, EntityRow } from './entity_changes_interface'; +const noteTypes = noteTypesService.getNoteTypeNames(); class ConsistencyChecks { + + private autoFix: boolean; + private unrecoveredConsistencyErrors: boolean; + private fixedIssues: boolean; + private reloadNeeded: boolean; + /** * @param autoFix - automatically fix all encountered problems. False is only for debugging during development (fail fast) */ - constructor(autoFix) { + constructor(autoFix: boolean) { this.autoFix = autoFix; this.unrecoveredConsistencyErrors = false; this.fixedIssues = false; this.reloadNeeded = false; } - findAndFixIssues(query, fixerCb) { + findAndFixIssues(query: string, fixerCb: (res: any) => void) { const results = sql.getRows(query); for (const res of results) { @@ -39,7 +48,7 @@ class ConsistencyChecks { } else { this.unrecoveredConsistencyErrors = true; } - } catch (e) { + } catch (e: any) { logError(`Fixer failed with ${e.message} ${e.stack}`); this.unrecoveredConsistencyErrors = true; } @@ -49,8 +58,8 @@ class ConsistencyChecks { } checkTreeCycles() { - const childToParents = {}; - const rows = sql.getRows("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0"); + const childToParents: Record = {}; + const rows = sql.getRows("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0"); for (const row of rows) { const childNoteId = row.noteId; @@ -61,7 +70,7 @@ class ConsistencyChecks { } /** @returns {boolean} true if cycle was found and we should try again */ - const checkTreeCycle = (noteId, path) => { + const checkTreeCycle = (noteId: string, path: string[]) => { if (noteId === 'root') { return false; } @@ -70,8 +79,10 @@ class ConsistencyChecks { if (path.includes(parentNoteId)) { if (this.autoFix) { const branch = becca.getBranchFromChildAndParent(noteId, parentNoteId); - branch.markAsDeleted('cycle-autofix'); - logFix(`Branch '${branch.branchId}' between child '${noteId}' and parent '${parentNoteId}' has been deleted since it was causing a tree cycle.`); + if (branch) { + branch.markAsDeleted('cycle-autofix'); + logFix(`Branch '${branch.branchId}' between child '${noteId}' and parent '${parentNoteId}' has been deleted since it was causing a tree cycle.`); + } return true; } @@ -133,6 +144,9 @@ class ConsistencyChecks { ({branchId, noteId}) => { if (this.autoFix) { const branch = becca.getBranch(branchId); + if (!branch) { + return; + } branch.markAsDeleted(); this.reloadNeeded = true; @@ -154,12 +168,21 @@ class ConsistencyChecks { if (this.autoFix) { // Delete the old branch and recreate it with root as parent. const oldBranch = becca.getBranch(branchId); + if (!oldBranch) { + return; + } + const noteId = oldBranch.noteId; oldBranch.markAsDeleted("missing-parent"); let message = `Branch '${branchId}' was missing parent note '${parentNoteId}', so it was deleted. `; - if (becca.getNote(noteId).getParentBranches().length === 0) { + const note = becca.getNote(noteId); + if (!note) { + return; + } + + if (note.getParentBranches().length === 0) { const newBranch = new BBranch({ parentNoteId: 'root', noteId: noteId, @@ -188,6 +211,9 @@ class ConsistencyChecks { ({attributeId, noteId}) => { if (this.autoFix) { const attribute = becca.getAttribute(attributeId); + if (!attribute) { + return; + } attribute.markAsDeleted(); this.reloadNeeded = true; @@ -208,6 +234,9 @@ class ConsistencyChecks { ({attributeId, noteId}) => { if (this.autoFix) { const attribute = becca.getAttribute(attributeId); + if (!attribute) { + return; + } attribute.markAsDeleted(); this.reloadNeeded = true; @@ -230,6 +259,9 @@ class ConsistencyChecks { ({attachmentId, ownerId}) => { if (this.autoFix) { const attachment = becca.getAttachment(attachmentId); + if (!attachment) { + return; + } attachment.markAsDeleted(); this.reloadNeeded = false; @@ -258,6 +290,7 @@ class ConsistencyChecks { ({branchId, noteId}) => { if (this.autoFix) { const branch = becca.getBranch(branchId); + if (!branch) return; branch.markAsDeleted(); this.reloadNeeded = true; @@ -278,6 +311,9 @@ class ConsistencyChecks { `, ({branchId, parentNoteId}) => { if (this.autoFix) { const branch = becca.getBranch(branchId); + if (!branch) { + return; + } branch.markAsDeleted(); this.reloadNeeded = true; @@ -321,7 +357,7 @@ class ConsistencyChecks { HAVING COUNT(1) > 1`, ({noteId, parentNoteId}) => { if (this.autoFix) { - const branchIds = sql.getColumn( + const branchIds = sql.getColumn( `SELECT branchId FROM branches WHERE noteId = ? @@ -333,9 +369,17 @@ class ConsistencyChecks { // it's not necessarily "original" branch, it's just the only one which will survive const origBranch = branches[0]; + if (!origBranch) { + logError(`Unable to find original branch.`); + return; + } // delete all but the first branch for (const branch of branches.slice(1)) { + if (!branch) { + continue; + } + branch.markAsDeleted(); logFix(`Removing branch '${branch.branchId}' since it's a parent-child duplicate of branch '${origBranch.branchId}'`); @@ -357,6 +401,7 @@ class ConsistencyChecks { ({attachmentId, noteId}) => { if (this.autoFix) { const attachment = becca.getAttachment(attachmentId); + if (!attachment) return; attachment.markAsDeleted(); this.reloadNeeded = false; @@ -379,6 +424,7 @@ class ConsistencyChecks { ({noteId, type}) => { if (this.autoFix) { const note = becca.getNote(noteId); + if (!note) return; note.type = 'file'; // file is a safe option to recover notes if the type is not known note.save(); @@ -404,6 +450,10 @@ class ConsistencyChecks { const fakeDate = "2000-01-01 00:00:00Z"; const blankContent = getBlankContent(isProtected, type, mime); + if (!blankContent) { + logError(`Unable to recover note ${noteId} since it's content could not be retrieved (might be protected note).`); + return; + } const blobId = utils.hashedBlobId(blankContent); const blobAlreadyExists = !!sql.getValue("SELECT 1 FROM blobs WHERE blobId = ?", [blobId]); @@ -452,7 +502,11 @@ class ConsistencyChecks { if (this.autoFix) { const note = becca.getNote(noteId); const blankContent = getBlankContent(false, type, mime); - note.setContent(blankContent); + if (!note) return; + + if (blankContent) { + note.setContent(blankContent); + } this.reloadNeeded = true; @@ -506,7 +560,7 @@ class ConsistencyChecks { AND branches.isDeleted = 0`, ({parentNoteId}) => { if (this.autoFix) { - const branchIds = sql.getColumn(` + const branchIds = sql.getColumn(` SELECT branchId FROM branches WHERE isDeleted = 0 @@ -515,6 +569,8 @@ class ConsistencyChecks { const branches = branchIds.map(branchId => becca.getBranch(branchId)); for (const branch of branches) { + if (!branch) continue; + // delete the old wrong branch branch.markAsDeleted("parent-is-search"); @@ -543,6 +599,7 @@ class ConsistencyChecks { ({attributeId}) => { if (this.autoFix) { const relation = becca.getAttribute(attributeId); + if (!relation) return; relation.markAsDeleted(); this.reloadNeeded = true; @@ -563,6 +620,7 @@ class ConsistencyChecks { ({attributeId, type}) => { if (this.autoFix) { const attribute = becca.getAttribute(attributeId); + if (!attribute) return; attribute.type = 'label'; attribute.save(); @@ -584,6 +642,7 @@ class ConsistencyChecks { ({attributeId, noteId}) => { if (this.autoFix) { const attribute = becca.getAttribute(attributeId); + if (!attribute) return; attribute.markAsDeleted(); this.reloadNeeded = true; @@ -605,6 +664,7 @@ class ConsistencyChecks { ({attributeId, targetNoteId}) => { if (this.autoFix) { const attribute = becca.getAttribute(attributeId); + if (!attribute) return; attribute.markAsDeleted(); this.reloadNeeded = true; @@ -616,14 +676,14 @@ class ConsistencyChecks { }); } - runEntityChangeChecks(entityName, key) { + runEntityChangeChecks(entityName: string, key: string) { this.findAndFixIssues(` SELECT ${key} as entityId FROM ${entityName} LEFT JOIN entity_changes ec ON ec.entityName = '${entityName}' AND ec.entityId = ${entityName}.${key} WHERE ec.id IS NULL`, ({entityId}) => { - const entityRow = sql.getRow(`SELECT * FROM ${entityName} WHERE ${key} = ?`, [entityId]); + const entityRow = sql.getRow(`SELECT * FROM ${entityName} WHERE ${key} = ?`, [entityId]); if (this.autoFix) { entityChangesService.putEntityChange({ @@ -691,10 +751,10 @@ class ConsistencyChecks { } findWronglyNamedAttributes() { - const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`); + const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`); for (const origName of attrNames) { - const fixedName = sanitizeAttributeName(origName); + const fixedName = sanitizeAttributeName.sanitizeAttributeName(origName); if (fixedName !== origName) { if (this.autoFix) { @@ -721,7 +781,7 @@ class ConsistencyChecks { findSyncIssues() { const lastSyncedPush = parseInt(sql.getValue("SELECT value FROM options WHERE name = 'lastSyncedPush'")); - const maxEntityChangeId = sql.getValue("SELECT MAX(id) FROM entity_changes"); + const maxEntityChangeId = sql.getValue("SELECT MAX(id) FROM entity_changes"); if (lastSyncedPush > maxEntityChangeId) { if (this.autoFix) { @@ -773,8 +833,8 @@ class ConsistencyChecks { } runDbDiagnostics() { - function getTableRowCount(tableName) { - const count = sql.getValue(`SELECT COUNT(1) FROM ${tableName}`); + function getTableRowCount(tableName: string) { + const count = sql.getValue(`SELECT COUNT(1) FROM ${tableName}`); return `${tableName}: ${count}`; } @@ -810,7 +870,7 @@ class ConsistencyChecks { } } -function getBlankContent(isProtected, type, mime) { +function getBlankContent(isProtected: boolean, type: string, mime: string) { if (isProtected) { return null; // this is wrong for protected non-erased notes, but we cannot create a valid value without a password } @@ -822,11 +882,11 @@ function getBlankContent(isProtected, type, mime) { return ''; // empty string might be a wrong choice for some note types, but it's the best guess } -function logFix(message) { +function logFix(message: string) { log.info(`Consistency issue fixed: ${message}`); } -function logError(message) { +function logError(message: string) { log.info(`Consistency error: ${message}`); } @@ -837,7 +897,7 @@ function runPeriodicChecks() { consistencyChecks.runChecks(); } -async function runOnDemandChecks(autoFix) { +async function runOnDemandChecks(autoFix: boolean) { const consistencyChecks = new ConsistencyChecks(autoFix); await consistencyChecks.runChecks(); } diff --git a/src/services/entity_changes_interface.ts b/src/services/entity_changes_interface.ts index 0276e24a0..11eb69ccb 100644 --- a/src/services/entity_changes_interface.ts +++ b/src/services/entity_changes_interface.ts @@ -7,6 +7,8 @@ export interface EntityChange { positions?: Record; hash: string; utcDateChanged?: string; + utcDateModified?: string; + utcDateCreated?: string; isSynced: boolean | 1 | 0; isErased: boolean | 1 | 0; componentId?: string | null; diff --git a/src/services/sql_init.ts b/src/services/sql_init.ts index dbdde89c5..5c9937daa 100644 --- a/src/services/sql_init.ts +++ b/src/services/sql_init.ts @@ -179,7 +179,7 @@ dbReady.then(() => { }); function getDbSize() { - return sql.getValue("SELECT page_count * page_size / 1000 as size FROM pragma_page_count(), pragma_page_size()"); + return sql.getValue("SELECT page_count * page_size / 1000 as size FROM pragma_page_count(), pragma_page_size()"); } log.info(`DB size: ${getDbSize()} KB`); diff --git a/src/services/sync.ts b/src/services/sync.ts index da559e36a..11978cade 100644 --- a/src/services/sync.ts +++ b/src/services/sync.ts @@ -282,7 +282,7 @@ async function checkContentHash(syncContext: SyncContext) { if (failedChecks.length > 0) { // before re-queuing sectors, make sure the entity changes are correct - const consistencyChecks = require('./consistency_checks.js'); + const consistencyChecks = require('./consistency_checks'); consistencyChecks.runEntityChangesChecks(); await syncRequest(syncContext, 'POST', `/api/sync/check-entity-changes`);