diff --git a/src/errors/not_found_error.ts b/src/errors/not_found_error.ts index 7ceb105a9..f765e11f9 100644 --- a/src/errors/not_found_error.ts +++ b/src/errors/not_found_error.ts @@ -6,4 +6,4 @@ class NotFoundError { } } -module.exports = NotFoundError; \ No newline at end of file +export = NotFoundError; \ No newline at end of file diff --git a/src/services/blob-interface.ts b/src/services/blob-interface.ts new file mode 100644 index 000000000..5905e9d99 --- /dev/null +++ b/src/services/blob-interface.ts @@ -0,0 +1,5 @@ +export interface Blob { + blobId: string; + content: Buffer; + utcDateModified: string; +} \ No newline at end of file diff --git a/src/services/blob.ts b/src/services/blob.ts index 85917f9bb..39b979fee 100644 --- a/src/services/blob.ts +++ b/src/services/blob.ts @@ -2,6 +2,7 @@ import becca = require('../becca/becca.js'); import NotFoundError = require('../errors/not_found_error'); import protectedSessionService = require('./protected_session'); import utils = require('./utils'); +import type { Blob } from "./blob-interface"; function getBlobPojo(entityName: string, entityId: string) { const entity = becca.getEntity(entityName, entityId); @@ -48,11 +49,11 @@ function processContent(content: Buffer | string | null, isProtected: boolean, i } } -function calculateContentHash({blobId, content}: { blobId: string, content: Buffer }) { +function calculateContentHash({blobId, content}: Blob) { return utils.hash(`${blobId}|${content.toString()}`); } -module.exports = { +export = { getBlobPojo, processContent, calculateContentHash diff --git a/src/services/date_utils.ts b/src/services/date_utils.ts index a4150c89e..31c58c2e4 100644 --- a/src/services/date_utils.ts +++ b/src/services/date_utils.ts @@ -95,7 +95,7 @@ function validateUtcDateTime(str: string) { } } -module.exports = { +export = { utcNowDateTime, localNowDateTime, localNowDate, diff --git a/src/services/entity_changes.js b/src/services/entity_changes.js deleted file mode 100644 index 22b5875a0..000000000 --- a/src/services/entity_changes.js +++ /dev/null @@ -1,202 +0,0 @@ -const sql = require('./sql'); -const dateUtils = require('./date_utils'); -const log = require('./log'); -const cls = require('./cls'); -const utils = require('./utils'); -const instanceId = require('./instance_id'); -const becca = require('../becca/becca.js'); -const blobService = require('../services/blob.js'); - -let maxEntityChangeId = 0; - -function putEntityChangeWithInstanceId(origEntityChange, instanceId) { - const ec = {...origEntityChange, instanceId}; - - putEntityChange(ec); -} - -function putEntityChangeWithForcedChange(origEntityChange) { - const ec = {...origEntityChange, changeId: null}; - - putEntityChange(ec); -} - -function putEntityChange(origEntityChange) { - const ec = {...origEntityChange}; - - delete ec.id; - - if (!ec.changeId) { - ec.changeId = utils.randomString(12); - } - - ec.componentId = ec.componentId || cls.getComponentId() || "NA"; // NA = not available - ec.instanceId = ec.instanceId || instanceId; - ec.isSynced = ec.isSynced ? 1 : 0; - ec.isErased = ec.isErased ? 1 : 0; - ec.id = sql.replace("entity_changes", ec); - - maxEntityChangeId = Math.max(maxEntityChangeId, ec.id); - - cls.putEntityChange(ec); -} - -function putNoteReorderingEntityChange(parentNoteId, componentId) { - putEntityChange({ - entityName: "note_reordering", - entityId: parentNoteId, - hash: 'N/A', - isErased: false, - utcDateChanged: dateUtils.utcNowDateTime(), - isSynced: true, - componentId, - instanceId - }); - - const eventService = require('./events'); - - eventService.emit(eventService.ENTITY_CHANGED, { - entityName: 'note_reordering', - entity: sql.getMap(`SELECT branchId, notePosition FROM branches WHERE isDeleted = 0 AND parentNoteId = ?`, [parentNoteId]) - }); -} - -function putEntityChangeForOtherInstances(ec) { - putEntityChange({ - ...ec, - changeId: null, - instanceId: null - }); -} - -function addEntityChangesForSector(entityName, sector) { - const entityChanges = sql.getRows(`SELECT * FROM entity_changes WHERE entityName = ? AND SUBSTR(entityId, 1, 1) = ?`, [entityName, sector]); - - let entitiesInserted = entityChanges.length; - - sql.transactional(() => { - if (entityName === 'blobs') { - entitiesInserted += addEntityChangesForDependingEntity(sector, 'notes', 'noteId'); - entitiesInserted += addEntityChangesForDependingEntity(sector, 'attachments', 'attachmentId'); - entitiesInserted += addEntityChangesForDependingEntity(sector, 'revisions', 'revisionId'); - } - - for (const ec of entityChanges) { - putEntityChangeWithForcedChange(ec); - } - }); - - log.info(`Added sector ${sector} of '${entityName}' (${entitiesInserted} entities) to the sync queue.`); -} - -function addEntityChangesForDependingEntity(sector, tableName, primaryKeyColumn) { - // problem in blobs might be caused by problem in entity referencing the blob - const dependingEntityChanges = sql.getRows(` - SELECT dep_change.* - FROM entity_changes orig_sector - JOIN ${tableName} ON ${tableName}.blobId = orig_sector.entityId - JOIN entity_changes dep_change ON dep_change.entityName = '${tableName}' AND dep_change.entityId = ${tableName}.${primaryKeyColumn} - WHERE orig_sector.entityName = 'blobs' AND SUBSTR(orig_sector.entityId, 1, 1) = ?`, [sector]); - - for (const ec of dependingEntityChanges) { - putEntityChangeWithForcedChange(ec); - } - - return dependingEntityChanges.length; -} - -function cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey) { - sql.execute(` - DELETE - FROM entity_changes - WHERE - isErased = 0 - AND entityName = '${entityName}' - AND entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`); -} - -function fillEntityChanges(entityName, entityPrimaryKey, condition = '') { - cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey); - - sql.transactional(() => { - const entityIds = sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} ${condition}`); - - let createdCount = 0; - - for (const entityId of entityIds) { - const existingRows = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]); - - if (existingRows !== 0) { - // we don't want to replace existing entities (which would effectively cause full resync) - continue; - } - - createdCount++; - - const ec = { - entityName, - entityId, - isErased: false - }; - - if (entityName === 'blobs') { - const blob = sql.getRow("SELECT blobId, content, utcDateModified FROM blobs WHERE blobId = ?", [entityId]); - ec.hash = blobService.calculateContentHash(blob); - ec.utcDateChanged = blob.utcDateModified; - ec.isSynced = true; // blobs are always synced - } else { - const entity = becca.getEntity(entityName, entityId); - - if (entity) { - ec.hash = entity.generateHash(); - ec.utcDateChanged = entity.getUtcDateChanged() || dateUtils.utcNowDateTime(); - ec.isSynced = entityName !== 'options' || !!entity.isSynced; - } else { - // entity might be null (not present in becca) when it's deleted - // this will produce different hash value than when entity is being deleted since then - // all normal hashed attributes are being used. Sync should recover from that, though. - ec.hash = "deleted"; - ec.utcDateChanged = dateUtils.utcNowDateTime(); - ec.isSynced = true; // deletable (the ones with isDeleted) entities are synced - } - } - - putEntityChange(ec); - } - - if (createdCount > 0) { - log.info(`Created ${createdCount} missing entity changes for entity '${entityName}'.`); - } - }); -} - -function fillAllEntityChanges() { - sql.transactional(() => { - sql.execute("DELETE FROM entity_changes WHERE isErased = 0"); - - fillEntityChanges("notes", "noteId"); - fillEntityChanges("branches", "branchId"); - fillEntityChanges("revisions", "revisionId"); - fillEntityChanges("attachments", "attachmentId"); - fillEntityChanges("blobs", "blobId"); - fillEntityChanges("attributes", "attributeId"); - fillEntityChanges("etapi_tokens", "etapiTokenId"); - fillEntityChanges("options", "name", 'WHERE isSynced = 1'); - }); -} - -function recalculateMaxEntityChangeId() { - maxEntityChangeId = sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes"); -} - -module.exports = { - putNoteReorderingEntityChange, - putEntityChangeForOtherInstances, - putEntityChangeWithForcedChange, - putEntityChange, - putEntityChangeWithInstanceId, - fillAllEntityChanges, - addEntityChangesForSector, - getMaxEntityChangeId: () => maxEntityChangeId, - recalculateMaxEntityChangeId -}; diff --git a/src/services/entity_changes.ts b/src/services/entity_changes.ts index 73f4ff259..ff2754851 100644 --- a/src/services/entity_changes.ts +++ b/src/services/entity_changes.ts @@ -1,12 +1,206 @@ -export interface EntityChange { - id?: string; - entityName: string; - entityId: string; - hash: string; - utcDateChanged: string; - isSynced: boolean | 1 | 0; - isErased: boolean | 1 | 0; - componentId?: string | null; - changeId?: string | null; - instanceId?: string | null; +import sql = require('./sql'); +import dateUtils = require('./date_utils'); +import log = require('./log'); +import cls = require('./cls'); +import utils = require('./utils'); +import instanceId = require('./instance_id'); +import becca = require('../becca/becca.js'); +import blobService = require('../services/blob.js'); +import { EntityChange } from './entity_changes_interface'; +import type { Blob } from "./blob-interface"; + +let maxEntityChangeId = 0; + +function putEntityChangeWithInstanceId(origEntityChange: EntityChange, instanceId: string) { + const ec = {...origEntityChange, instanceId}; + + putEntityChange(ec); } + +function putEntityChangeWithForcedChange(origEntityChange: EntityChange) { + const ec = {...origEntityChange, changeId: null}; + + putEntityChange(ec); +} + +function putEntityChange(origEntityChange: EntityChange) { + const ec = {...origEntityChange}; + + delete ec.id; + + if (!ec.changeId) { + ec.changeId = utils.randomString(12); + } + + ec.componentId = ec.componentId || cls.getComponentId() || "NA"; // NA = not available + ec.instanceId = ec.instanceId || instanceId; + ec.isSynced = ec.isSynced ? 1 : 0; + ec.isErased = ec.isErased ? 1 : 0; + ec.id = sql.replace("entity_changes", ec); + + if (ec.id) { + maxEntityChangeId = Math.max(maxEntityChangeId, ec.id); + } + + cls.putEntityChange(ec); +} + +function putNoteReorderingEntityChange(parentNoteId: string, componentId: string) { + putEntityChange({ + entityName: "note_reordering", + entityId: parentNoteId, + hash: 'N/A', + isErased: false, + utcDateChanged: dateUtils.utcNowDateTime(), + isSynced: true, + componentId, + instanceId + }); + + const eventService = require('./events'); + + eventService.emit(eventService.ENTITY_CHANGED, { + entityName: 'note_reordering', + entity: sql.getMap(`SELECT branchId, notePosition FROM branches WHERE isDeleted = 0 AND parentNoteId = ?`, [parentNoteId]) + }); +} + +function putEntityChangeForOtherInstances(ec: EntityChange) { + putEntityChange({ + ...ec, + changeId: null, + instanceId: null + }); +} + +function addEntityChangesForSector(entityName: string, sector: string) { + const entityChanges = sql.getRows(`SELECT * FROM entity_changes WHERE entityName = ? AND SUBSTR(entityId, 1, 1) = ?`, [entityName, sector]); + + let entitiesInserted = entityChanges.length; + + sql.transactional(() => { + if (entityName === 'blobs') { + entitiesInserted += addEntityChangesForDependingEntity(sector, 'notes', 'noteId'); + entitiesInserted += addEntityChangesForDependingEntity(sector, 'attachments', 'attachmentId'); + entitiesInserted += addEntityChangesForDependingEntity(sector, 'revisions', 'revisionId'); + } + + for (const ec of entityChanges) { + putEntityChangeWithForcedChange(ec); + } + }); + + log.info(`Added sector ${sector} of '${entityName}' (${entitiesInserted} entities) to the sync queue.`); +} + +function addEntityChangesForDependingEntity(sector: string, tableName: string, primaryKeyColumn: string) { + // problem in blobs might be caused by problem in entity referencing the blob + const dependingEntityChanges = sql.getRows(` + SELECT dep_change.* + FROM entity_changes orig_sector + JOIN ${tableName} ON ${tableName}.blobId = orig_sector.entityId + JOIN entity_changes dep_change ON dep_change.entityName = '${tableName}' AND dep_change.entityId = ${tableName}.${primaryKeyColumn} + WHERE orig_sector.entityName = 'blobs' AND SUBSTR(orig_sector.entityId, 1, 1) = ?`, [sector]); + + for (const ec of dependingEntityChanges) { + putEntityChangeWithForcedChange(ec); + } + + return dependingEntityChanges.length; +} + +function cleanupEntityChangesForMissingEntities(entityName: string, entityPrimaryKey: string) { + sql.execute(` + DELETE + FROM entity_changes + WHERE + isErased = 0 + AND entityName = '${entityName}' + AND entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`); +} + +function fillEntityChanges(entityName: string, entityPrimaryKey: string, condition = '') { + cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey); + + sql.transactional(() => { + const entityIds = sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} ${condition}`); + + let createdCount = 0; + + for (const entityId of entityIds) { + const existingRows = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]); + + if (existingRows !== 0) { + // we don't want to replace existing entities (which would effectively cause full resync) + continue; + } + + createdCount++; + + const ec: Partial = { + entityName, + entityId, + isErased: false + }; + + if (entityName === 'blobs') { + const blob = sql.getRow("SELECT blobId, content, utcDateModified FROM blobs WHERE blobId = ?", [entityId]); + ec.hash = blobService.calculateContentHash(blob); + ec.utcDateChanged = blob.utcDateModified; + ec.isSynced = true; // blobs are always synced + } else { + const entity = becca.getEntity(entityName, entityId); + + if (entity) { + ec.hash = entity.generateHash(); + ec.utcDateChanged = entity.getUtcDateChanged() || dateUtils.utcNowDateTime(); + ec.isSynced = entityName !== 'options' || !!entity.isSynced; + } else { + // entity might be null (not present in becca) when it's deleted + // this will produce different hash value than when entity is being deleted since then + // all normal hashed attributes are being used. Sync should recover from that, though. + ec.hash = "deleted"; + ec.utcDateChanged = dateUtils.utcNowDateTime(); + ec.isSynced = true; // deletable (the ones with isDeleted) entities are synced + } + } + + putEntityChange(ec as EntityChange); + } + + if (createdCount > 0) { + log.info(`Created ${createdCount} missing entity changes for entity '${entityName}'.`); + } + }); +} + +function fillAllEntityChanges() { + sql.transactional(() => { + sql.execute("DELETE FROM entity_changes WHERE isErased = 0"); + + fillEntityChanges("notes", "noteId"); + fillEntityChanges("branches", "branchId"); + fillEntityChanges("revisions", "revisionId"); + fillEntityChanges("attachments", "attachmentId"); + fillEntityChanges("blobs", "blobId"); + fillEntityChanges("attributes", "attributeId"); + fillEntityChanges("etapi_tokens", "etapiTokenId"); + fillEntityChanges("options", "name", 'WHERE isSynced = 1'); + }); +} + +function recalculateMaxEntityChangeId() { + maxEntityChangeId = sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes"); +} + +export = { + putNoteReorderingEntityChange, + putEntityChangeForOtherInstances, + putEntityChangeWithForcedChange, + putEntityChange, + putEntityChangeWithInstanceId, + fillAllEntityChanges, + addEntityChangesForSector, + getMaxEntityChangeId: () => maxEntityChangeId, + recalculateMaxEntityChangeId +}; diff --git a/src/services/entity_changes_interface.ts b/src/services/entity_changes_interface.ts new file mode 100644 index 000000000..0a6409a80 --- /dev/null +++ b/src/services/entity_changes_interface.ts @@ -0,0 +1,12 @@ +export interface EntityChange { + id?: number | null; + entityName: string; + entityId: string; + hash: string; + utcDateChanged: string; + isSynced: boolean | 1 | 0; + isErased: boolean | 1 | 0; + componentId?: string | null; + changeId?: string | null; + instanceId?: string | null; +} \ No newline at end of file diff --git a/src/services/instance_id.ts b/src/services/instance_id.ts index 704da863c..6e0eb503c 100644 --- a/src/services/instance_id.ts +++ b/src/services/instance_id.ts @@ -2,4 +2,4 @@ import utils = require('./utils'); const instanceId = utils.randomString(12); -module.exports = instanceId; +export = instanceId; diff --git a/src/services/protected_session.ts b/src/services/protected_session.ts index 060705280..19d15d542 100644 --- a/src/services/protected_session.ts +++ b/src/services/protected_session.ts @@ -30,16 +30,16 @@ function encrypt(plainText: string | Buffer) { return dataEncryptionService.encrypt(dataKey, plainText); } -function decrypt(cipherText: string | Buffer) { +function decrypt(cipherText: string | Buffer): Buffer | null { const dataKey = getDataKey(); if (cipherText === null || dataKey === null) { return null; } - return dataEncryptionService.decrypt(dataKey, cipherText); + return dataEncryptionService.decrypt(dataKey, cipherText) || null; } -function decryptString(cipherText: string) { +function decryptString(cipherText: string): string | null { const dataKey = getDataKey(); if (dataKey === null) { return null; @@ -70,7 +70,7 @@ function checkProtectedSessionExpiration() { } } -module.exports = { +export = { setDataKey, resetDataKey, isProtectedSessionAvailable, diff --git a/src/services/sql.ts b/src/services/sql.ts index bdf490d92..f31d51728 100644 --- a/src/services/sql.ts +++ b/src/services/sql.ts @@ -103,8 +103,8 @@ function getRowOrNull(query: string, params: Params = []): T | null { return (all.length > 0 ? all[0] : null) as (T | null); } -function getValue(query: string, params: Params = []) { - return wrap(query, s => s.pluck().get(params)); +function getValue(query: string, params: Params = []): T { + return wrap(query, s => s.pluck().get(params)) as T; } // smaller values can result in better performance due to better usage of statement cache @@ -139,8 +139,8 @@ function getManyRows(query: string, params: Params) { return results; } -function getRows(query: string, params: Params = []): unknown[] { - return wrap(query, s => s.all(params)) as unknown[]; +function getRows(query: string, params: Params = []): T[] { + return wrap(query, s => s.all(params)) as T[]; } function getRawRows(query: string, params: Params = []): T[] | null { @@ -166,8 +166,8 @@ function getMap(query: string, params: Pa return map; } -function getColumn(query: string, params: Params = []) { - return wrap(query, s => s.pluck().all(params)); +function getColumn(query: string, params: Params = []): T[] { + return wrap(query, s => s.pluck().all(params)) as T[]; } function execute(query: string, params: Params = []): RunResult {