server-ts: Port consistency_checks

src/app.js

@@ -26,10 +26,10 @@ app.use(helmet({
     crossOriginEmbedderPolicy: false
 }));
 
-app.use(express.text({limit: '500mb'}));
+app.use(express.text({ limit: '500mb' }));
-app.use(express.json({limit: '500mb'}));
+app.use(express.json({ limit: '500mb' }));
-app.use(express.raw({limit: '500mb'}));
+app.use(express.raw({ limit: '500mb' }));
-app.use(express.urlencoded({extended: false}));
+app.use(express.urlencoded({ extended: false }));
 app.use(cookieParser());
 app.use(express.static(path.join(__dirname, 'public/root')));
 app.use(`/manifest.webmanifest`, express.static(path.join(__dirname, 'public/manifest.webmanifest')));
@@ -49,7 +49,7 @@ require('./services/sync');
 require('./services/backup');
 
 // trigger consistency checks timer
-require('./services/consistency_checks.js');
+require('./services/consistency_checks');
 
 require('./services/scheduler.js');
 

src/routes/api/database.js

@@ -4,7 +4,7 @@ const sql = require('../../services/sql');
 const log = require('../../services/log');
 const backupService = require('../../services/backup');
 const anonymizationService = require('../../services/anonymization');
-const consistencyChecksService = require('../../services/consistency_checks.js');
+const consistencyChecksService = require('../../services/consistency_checks');
 
 function getExistingBackups() {
     return backupService.getExistingBackups();

src/routes/api/sync.js

@@ -132,7 +132,7 @@ function getChanged(req) {
 const partialRequests = {};
 
 function update(req) {
-    let {body} = req;
+    let { body } = req;
 
     const pageCount = parseInt(req.get('pageCount'));
     const pageIndex = parseInt(req.get('pageIndex'));
@@ -164,7 +164,7 @@ function update(req) {
         }
     }
 
-    const {entities, instanceId} = body;
+    const { entities, instanceId } = body;
 
     sql.transactional(() => syncUpdateService.updateEntities(entities, instanceId));
 }
@@ -193,7 +193,7 @@ function queueSector(req) {
 }
 
 function checkEntityChanges() {
-    require('../../services/consistency_checks.js').runEntityChangesChecks();
+    require('../../services/consistency_checks').runEntityChangesChecks();
 }
 
 module.exports = {

src/services/consistency_checks.ts

@@ -1,33 +1,42 @@
 "use strict";
 
-const sql = require('./sql');
+import sql = require('./sql');
-const sqlInit = require('./sql_init');
+import sqlInit = require('./sql_init');
-const log = require('./log');
+import log = require('./log');
-const ws = require('./ws');
+import ws = require('./ws');
-const syncMutexService = require('./sync_mutex');
+import syncMutexService = require('./sync_mutex');
-const cls = require('./cls');
+import cls = require('./cls');
-const entityChangesService = require('./entity_changes');
+import entityChangesService = require('./entity_changes');
-const optionsService = require('./options');
+import optionsService = require('./options');
-const BBranch = require('../becca/entities/bbranch');
+import BBranch = require('../becca/entities/bbranch');
-const revisionService = require('./revisions');
+import revisionService = require('./revisions');
-const becca = require('../becca/becca');
+import becca = require('../becca/becca');
-const utils = require('../services/utils');
+import utils = require('../services/utils');
-const eraseService = require('../services/erase');
+import eraseService = require('../services/erase');
-const {sanitizeAttributeName} = require('./sanitize_attribute_name');
+import sanitizeAttributeName = require('./sanitize_attribute_name');
-const noteTypes = require('../services/note_types').getNoteTypeNames();
+import noteTypesService = require('../services/note_types');
+import { BranchRow, NoteRow } from '../becca/entities/rows';
+import { EntityChange, EntityRow } from './entity_changes_interface';
+const noteTypes = noteTypesService.getNoteTypeNames();
 
 class ConsistencyChecks {
+
+    private autoFix: boolean;
+    private unrecoveredConsistencyErrors: boolean;
+    private fixedIssues: boolean;    
+    private reloadNeeded: boolean;
+
     /**
      * @param autoFix - automatically fix all encountered problems. False is only for debugging during development (fail fast)
      */
-    constructor(autoFix) {
+    constructor(autoFix: boolean) {
         this.autoFix = autoFix;
         this.unrecoveredConsistencyErrors = false;
         this.fixedIssues = false;
         this.reloadNeeded = false;
     }
 
-    findAndFixIssues(query, fixerCb) {
+    findAndFixIssues(query: string, fixerCb: (res: any) => void) {
         const results = sql.getRows(query);
 
         for (const res of results) {
@@ -39,7 +48,7 @@ class ConsistencyChecks {
                 } else {
                     this.unrecoveredConsistencyErrors = true;
                 }
-            } catch (e) {
+            } catch (e: any) {
                 logError(`Fixer failed with ${e.message} ${e.stack}`);
                 this.unrecoveredConsistencyErrors = true;
             }
@@ -49,8 +58,8 @@ class ConsistencyChecks {
     }
 
     checkTreeCycles() {
-        const childToParents = {};
+        const childToParents: Record<string, string[]> = {};
-        const rows = sql.getRows("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0");
+        const rows = sql.getRows<BranchRow>("SELECT noteId, parentNoteId FROM branches WHERE isDeleted = 0");
 
         for (const row of rows) {
             const childNoteId = row.noteId;
@@ -61,7 +70,7 @@ class ConsistencyChecks {
         }
 
         /** @returns {boolean} true if cycle was found and we should try again */
-        const checkTreeCycle = (noteId, path) => {
+        const checkTreeCycle = (noteId: string, path: string[]) => {
             if (noteId === 'root') {
                 return false;
             }
@@ -70,8 +79,10 @@ class ConsistencyChecks {
                 if (path.includes(parentNoteId)) {
                     if (this.autoFix) {
                         const branch = becca.getBranchFromChildAndParent(noteId, parentNoteId);
-                        branch.markAsDeleted('cycle-autofix');
+                        if (branch) {
-                        logFix(`Branch '${branch.branchId}' between child '${noteId}' and parent '${parentNoteId}' has been deleted since it was causing a tree cycle.`);
+                            branch.markAsDeleted('cycle-autofix');
+                            logFix(`Branch '${branch.branchId}' between child '${noteId}' and parent '${parentNoteId}' has been deleted since it was causing a tree cycle.`);
+                        }
 
                         return true;
                     }
@@ -133,6 +144,9 @@ class ConsistencyChecks {
             ({branchId, noteId}) => {
                 if (this.autoFix) {
                     const branch = becca.getBranch(branchId);
+                    if (!branch) {
+                        return;
+                    }
                     branch.markAsDeleted();
 
                     this.reloadNeeded = true;
@@ -154,12 +168,21 @@ class ConsistencyChecks {
                 if (this.autoFix) {
                     // Delete the old branch and recreate it with root as parent.
                     const oldBranch = becca.getBranch(branchId);
+                    if (!oldBranch) {
+                        return;
+                    }
+
                     const noteId = oldBranch.noteId;
                     oldBranch.markAsDeleted("missing-parent");
 
                     let message = `Branch '${branchId}' was missing parent note '${parentNoteId}', so it was deleted. `;
 
-                    if (becca.getNote(noteId).getParentBranches().length === 0) {
+                    const note = becca.getNote(noteId);
+                    if (!note) {
+                        return;
+                    }
+
+                    if (note.getParentBranches().length === 0) {
                         const newBranch = new BBranch({
                             parentNoteId: 'root',
                             noteId: noteId,
@@ -188,6 +211,9 @@ class ConsistencyChecks {
             ({attributeId, noteId}) => {
                 if (this.autoFix) {
                     const attribute = becca.getAttribute(attributeId);
+                    if (!attribute) {
+                        return;
+                    }
                     attribute.markAsDeleted();
 
                     this.reloadNeeded = true;
@@ -208,6 +234,9 @@ class ConsistencyChecks {
             ({attributeId, noteId}) => {
                 if (this.autoFix) {
                     const attribute = becca.getAttribute(attributeId);
+                    if (!attribute) {
+                        return;
+                    }
                     attribute.markAsDeleted();
 
                     this.reloadNeeded = true;
@@ -230,6 +259,9 @@ class ConsistencyChecks {
             ({attachmentId, ownerId}) => {
                 if (this.autoFix) {
                     const attachment = becca.getAttachment(attachmentId);
+                    if (!attachment) {
+                        return;
+                    }
                     attachment.markAsDeleted();
 
                     this.reloadNeeded = false;
@@ -258,6 +290,7 @@ class ConsistencyChecks {
             ({branchId, noteId}) => {
                 if (this.autoFix) {
                     const branch = becca.getBranch(branchId);
+                    if (!branch) return;
                     branch.markAsDeleted();
 
                     this.reloadNeeded = true;
@@ -278,6 +311,9 @@ class ConsistencyChecks {
         `, ({branchId, parentNoteId}) => {
             if (this.autoFix) {
                 const branch = becca.getBranch(branchId);
+                if (!branch) {
+                    return;
+                }
                 branch.markAsDeleted();
 
                 this.reloadNeeded = true;
@@ -321,7 +357,7 @@ class ConsistencyChecks {
                     HAVING COUNT(1) > 1`,
             ({noteId, parentNoteId}) => {
                 if (this.autoFix) {
-                    const branchIds = sql.getColumn(
+                    const branchIds = sql.getColumn<string>(
                             `SELECT branchId
                              FROM branches
                              WHERE noteId = ?
@@ -333,9 +369,17 @@ class ConsistencyChecks {
 
                     // it's not necessarily "original" branch, it's just the only one which will survive
                     const origBranch = branches[0];
+                    if (!origBranch) {
+                        logError(`Unable to find original branch.`);
+                        return;
+                    }
 
                     // delete all but the first branch
                     for (const branch of branches.slice(1)) {
+                        if (!branch) {
+                            continue;
+                        }
+
                         branch.markAsDeleted();
 
                         logFix(`Removing branch '${branch.branchId}' since it's a parent-child duplicate of branch '${origBranch.branchId}'`);
@@ -357,6 +401,7 @@ class ConsistencyChecks {
             ({attachmentId, noteId}) => {
                 if (this.autoFix) {
                     const attachment = becca.getAttachment(attachmentId);
+                    if (!attachment) return;
                     attachment.markAsDeleted();
 
                     this.reloadNeeded = false;
@@ -379,6 +424,7 @@ class ConsistencyChecks {
             ({noteId, type}) => {
                 if (this.autoFix) {
                     const note = becca.getNote(noteId);
+                    if (!note) return;
                     note.type = 'file'; // file is a safe option to recover notes if the type is not known
                     note.save();
 
@@ -404,6 +450,10 @@ class ConsistencyChecks {
                     const fakeDate = "2000-01-01 00:00:00Z";
 
                     const blankContent = getBlankContent(isProtected, type, mime);
+                    if (!blankContent) {
+                        logError(`Unable to recover note ${noteId} since it's content could not be retrieved (might be protected note).`);
+                        return;
+                    }
                     const blobId = utils.hashedBlobId(blankContent);
                     const blobAlreadyExists = !!sql.getValue("SELECT 1 FROM blobs WHERE blobId = ?", [blobId]);
 
@@ -452,7 +502,11 @@ class ConsistencyChecks {
                     if (this.autoFix) {
                         const note = becca.getNote(noteId);
                         const blankContent = getBlankContent(false, type, mime);
-                        note.setContent(blankContent);
+                        if (!note) return;
+
+                        if (blankContent) {
+                            note.setContent(blankContent);
+                        }
 
                         this.reloadNeeded = true;
 
@@ -506,7 +560,7 @@ class ConsistencyChecks {
                       AND branches.isDeleted = 0`,
             ({parentNoteId}) => {
                 if (this.autoFix) {
-                    const branchIds = sql.getColumn(`
+                    const branchIds = sql.getColumn<string>(`
                         SELECT branchId
                         FROM branches
                         WHERE isDeleted = 0
@@ -515,6 +569,8 @@ class ConsistencyChecks {
                     const branches = branchIds.map(branchId => becca.getBranch(branchId));
 
                     for (const branch of branches) {
+                        if (!branch) continue;
+
                         // delete the old wrong branch
                         branch.markAsDeleted("parent-is-search");
 
@@ -543,6 +599,7 @@ class ConsistencyChecks {
             ({attributeId}) => {
                 if (this.autoFix) {
                     const relation = becca.getAttribute(attributeId);
+                    if (!relation) return;
                     relation.markAsDeleted();
 
                     this.reloadNeeded = true;
@@ -563,6 +620,7 @@ class ConsistencyChecks {
             ({attributeId, type}) => {
                 if (this.autoFix) {
                     const attribute = becca.getAttribute(attributeId);
+                    if (!attribute) return;
                     attribute.type = 'label';
                     attribute.save();
 
@@ -584,6 +642,7 @@ class ConsistencyChecks {
             ({attributeId, noteId}) => {
                 if (this.autoFix) {
                     const attribute = becca.getAttribute(attributeId);
+                    if (!attribute) return;
                     attribute.markAsDeleted();
 
                     this.reloadNeeded = true;
@@ -605,6 +664,7 @@ class ConsistencyChecks {
             ({attributeId, targetNoteId}) => {
                 if (this.autoFix) {
                     const attribute = becca.getAttribute(attributeId);
+                    if (!attribute) return;
                     attribute.markAsDeleted();
 
                     this.reloadNeeded = true;
@@ -616,14 +676,14 @@ class ConsistencyChecks {
             });
     }
 
-    runEntityChangeChecks(entityName, key) {
+    runEntityChangeChecks(entityName: string, key: string) {
         this.findAndFixIssues(`
             SELECT ${key} as entityId
             FROM ${entityName}
             LEFT JOIN entity_changes ec ON ec.entityName = '${entityName}' AND ec.entityId = ${entityName}.${key} 
             WHERE ec.id IS NULL`,
             ({entityId}) => {
-                const entityRow = sql.getRow(`SELECT * FROM ${entityName} WHERE ${key} = ?`, [entityId]);
+                const entityRow = sql.getRow<EntityChange>(`SELECT * FROM ${entityName} WHERE ${key} = ?`, [entityId]);
 
                 if (this.autoFix) {
                     entityChangesService.putEntityChange({
@@ -691,10 +751,10 @@ class ConsistencyChecks {
     }
 
     findWronglyNamedAttributes() {
-        const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`);
+        const attrNames = sql.getColumn<string>(`SELECT DISTINCT name FROM attributes`);
 
         for (const origName of attrNames) {
-            const fixedName = sanitizeAttributeName(origName);
+            const fixedName = sanitizeAttributeName.sanitizeAttributeName(origName);
 
             if (fixedName !== origName) {
                 if (this.autoFix) {
@@ -721,7 +781,7 @@ class ConsistencyChecks {
 
     findSyncIssues() {
         const lastSyncedPush = parseInt(sql.getValue("SELECT value FROM options WHERE name = 'lastSyncedPush'"));
-        const maxEntityChangeId = sql.getValue("SELECT MAX(id) FROM entity_changes");
+        const maxEntityChangeId = sql.getValue<number>("SELECT MAX(id) FROM entity_changes");
 
         if (lastSyncedPush > maxEntityChangeId) {
             if (this.autoFix) {
@@ -773,8 +833,8 @@ class ConsistencyChecks {
     }
 
     runDbDiagnostics() {
-        function getTableRowCount(tableName) {
+        function getTableRowCount(tableName: string) {
-            const count = sql.getValue(`SELECT COUNT(1) FROM ${tableName}`);
+            const count = sql.getValue<number>(`SELECT COUNT(1) FROM ${tableName}`);
 
             return `${tableName}: ${count}`;
         }
@@ -810,7 +870,7 @@ class ConsistencyChecks {
     }
 }
 
-function getBlankContent(isProtected, type, mime) {
+function getBlankContent(isProtected: boolean, type: string, mime: string) {
     if (isProtected) {
         return null; // this is wrong for protected non-erased notes, but we cannot create a valid value without a password
     }
@@ -822,11 +882,11 @@ function getBlankContent(isProtected, type, mime) {
     return ''; // empty string might be a wrong choice for some note types, but it's the best guess
 }
 
-function logFix(message) {
+function logFix(message: string) {
     log.info(`Consistency issue fixed: ${message}`);
 }
 
-function logError(message) {
+function logError(message: string) {
     log.info(`Consistency error: ${message}`);
 }
 
@@ -837,7 +897,7 @@ function runPeriodicChecks() {
     consistencyChecks.runChecks();
 }
 
-async function runOnDemandChecks(autoFix) {
+async function runOnDemandChecks(autoFix: boolean) {
     const consistencyChecks = new ConsistencyChecks(autoFix);
     await consistencyChecks.runChecks();
 }

src/services/entity_changes_interface.ts

@@ -7,6 +7,8 @@ export interface EntityChange {
 	positions?: Record<string, number>;
 	hash: string;
 	utcDateChanged?: string;
+	utcDateModified?: string;
+	utcDateCreated?: string;
 	isSynced: boolean | 1 | 0;
 	isErased: boolean | 1 | 0;
 	componentId?: string | null;

src/services/sql_init.ts

@@ -179,7 +179,7 @@ dbReady.then(() => {
 });
 
 function getDbSize() {
-    return sql.getValue("SELECT page_count * page_size / 1000 as size FROM pragma_page_count(), pragma_page_size()");
+    return sql.getValue<number>("SELECT page_count * page_size / 1000 as size FROM pragma_page_count(), pragma_page_size()");
 }
 
 log.info(`DB size: ${getDbSize()} KB`);

src/services/sync.ts

@@ -282,7 +282,7 @@ async function checkContentHash(syncContext: SyncContext) {
 
     if (failedChecks.length > 0) {
         // before re-queuing sectors, make sure the entity changes are correct
-        const consistencyChecks = require('./consistency_checks.js');
+        const consistencyChecks = require('./consistency_checks');
         consistencyChecks.runEntityChangesChecks();
 
         await syncRequest(syncContext, 'POST', `/api/sync/check-entity-changes`);