From 64b212b93e30985f747e33f24bd6d3aa5cf7c45d Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Tue, 6 Jan 2026 13:42:29 +0200 Subject: [PATCH] chore(core): integrate entity_changes --- apps/server/src/services/cls.ts | 41 ++-- apps/server/src/services/entity_changes.ts | 209 +---------------- apps/server/src/services/instance_id.ts | 7 +- packages/trilium-core/src/index.ts | 3 + packages/trilium-core/src/services/context.ts | 19 ++ .../src/services/entity_changes.ts | 211 ++++++++++++++++++ .../trilium-core/src/services/instance_id.ts | 5 + 7 files changed, 258 insertions(+), 237 deletions(-) create mode 100644 packages/trilium-core/src/services/entity_changes.ts create mode 100644 packages/trilium-core/src/services/instance_id.ts diff --git a/apps/server/src/services/cls.ts b/apps/server/src/services/cls.ts index ecbef741e..ebf693559 100644 --- a/apps/server/src/services/cls.ts +++ b/apps/server/src/services/cls.ts @@ -1,10 +1,10 @@ import type { EntityChange } from "@triliumnext/commons"; -import { getContext, getHoistedNoteId as getHoistedNoteIdInternal, isEntityEventsDisabled as isEntityEventsDisabledInternal } from "@triliumnext/core/src/services/context"; +import { cls } from "@triliumnext/core"; type Callback = (...args: any[]) => any; function init(callback: () => T) { - return getContext().init(callback); + return cls.getContext().init(callback); } function wrap(callback: Callback) { @@ -18,68 +18,59 @@ function wrap(callback: Callback) { } function getHoistedNoteId() { - return getHoistedNoteIdInternal(); + return cls.getHoistedNoteId(); } function getComponentId() { - return getContext().get("componentId"); + return cls.getComponentId(); } function disableEntityEvents() { - getContext().set("disableEntityEvents", true); + cls.getContext().set("disableEntityEvents", true); } function enableEntityEvents() { - getContext().set("disableEntityEvents", false); + cls.getContext().set("disableEntityEvents", false); } function isEntityEventsDisabled() { - return isEntityEventsDisabledInternal(); + return cls.isEntityEventsDisabled(); } function setMigrationRunning(running: boolean) { - getContext().set("migrationRunning", !!running); + cls.getContext().set("migrationRunning", !!running); } function isMigrationRunning() { - return !!getContext().get("migrationRunning"); + return !!cls.getContext().get("migrationRunning"); } function getAndClearEntityChangeIds() { - const entityChangeIds = getContext().get("entityChangeIds") || []; + const entityChangeIds = cls.getContext().get("entityChangeIds") || []; - getContext().set("entityChangeIds", []); + cls.getContext().set("entityChangeIds", []); return entityChangeIds; } function putEntityChange(entityChange: EntityChange) { - if (getContext().get("ignoreEntityChangeIds")) { - return; - } - - const entityChangeIds = getContext().get("entityChangeIds") || []; - - // store only ID since the record can be modified (e.g., in erase) - entityChangeIds.push(entityChange.id); - - getContext().set("entityChangeIds", entityChangeIds); + cls.putEntityChange(entityChange); } function ignoreEntityChangeIds() { - getContext().set("ignoreEntityChangeIds", true); + cls.getContext().set("ignoreEntityChangeIds", true); } function get(key: string) { - return getContext().get(key); + return cls.getContext().get(key); } function set(key: string, value: unknown) { - getContext().set(key, value); + cls.getContext().set(key, value); } function reset() { - getContext().reset(); + cls.getContext().reset(); } export default { diff --git a/apps/server/src/services/entity_changes.ts b/apps/server/src/services/entity_changes.ts index 65c941e1c..24b210d95 100644 --- a/apps/server/src/services/entity_changes.ts +++ b/apps/server/src/services/entity_changes.ts @@ -1,207 +1,2 @@ -import type { BlobRow, EntityChange } from "@triliumnext/commons"; -import { blob as blobService, events as eventService } from "@triliumnext/core"; - -import becca from "../becca/becca.js"; -import cls from "./cls.js"; -import dateUtils from "./date_utils.js"; -import instanceId from "./instance_id.js"; -import log from "./log.js"; -import sql from "./sql.js"; -import { randomString } from "./utils.js"; - -let maxEntityChangeId = 0; - -function putEntityChangeWithInstanceId(origEntityChange: EntityChange, instanceId: string) { - const ec = { ...origEntityChange, instanceId }; - - putEntityChange(ec); -} - -function putEntityChangeWithForcedChange(origEntityChange: EntityChange) { - const ec = { ...origEntityChange, changeId: null }; - - putEntityChange(ec); -} - -function putEntityChange(origEntityChange: EntityChange) { - const ec = { ...origEntityChange }; - - delete ec.id; - - if (!ec.changeId) { - ec.changeId = randomString(12); - } - - ec.componentId = ec.componentId || cls.getComponentId() || "NA"; // NA = not available - ec.instanceId = ec.instanceId || instanceId; - ec.isSynced = ec.isSynced ? 1 : 0; - ec.isErased = ec.isErased ? 1 : 0; - ec.id = sql.replace("entity_changes", ec); - - if (ec.id) { - maxEntityChangeId = Math.max(maxEntityChangeId, ec.id); - } - - cls.putEntityChange(ec); -} - -function putNoteReorderingEntityChange(parentNoteId: string, componentId?: string) { - putEntityChange({ - entityName: "note_reordering", - entityId: parentNoteId, - hash: "N/A", - isErased: false, - utcDateChanged: dateUtils.utcNowDateTime(), - isSynced: true, - componentId, - instanceId - }); - - eventService.emit(eventService.ENTITY_CHANGED, { - entityName: "note_reordering", - entity: sql.getMap(/*sql*/`SELECT branchId, notePosition FROM branches WHERE isDeleted = 0 AND parentNoteId = ?`, [parentNoteId]) - }); -} - -function putEntityChangeForOtherInstances(ec: EntityChange) { - putEntityChange({ - ...ec, - changeId: null, - instanceId: null - }); -} - -function addEntityChangesForSector(entityName: string, sector: string) { - const entityChanges = sql.getRows(/*sql*/`SELECT * FROM entity_changes WHERE entityName = ? AND SUBSTR(entityId, 1, 1) = ?`, [entityName, sector]); - - let entitiesInserted = entityChanges.length; - - sql.transactional(() => { - if (entityName === "blobs") { - entitiesInserted += addEntityChangesForDependingEntity(sector, "notes", "noteId"); - entitiesInserted += addEntityChangesForDependingEntity(sector, "attachments", "attachmentId"); - entitiesInserted += addEntityChangesForDependingEntity(sector, "revisions", "revisionId"); - } - - for (const ec of entityChanges) { - putEntityChangeWithForcedChange(ec); - } - }); - - log.info(`Added sector ${sector} of '${entityName}' (${entitiesInserted} entities) to the sync queue.`); -} - -function addEntityChangesForDependingEntity(sector: string, tableName: string, primaryKeyColumn: string) { - // problem in blobs might be caused by problem in entity referencing the blob - const dependingEntityChanges = sql.getRows( - ` - SELECT dep_change.* - FROM entity_changes orig_sector - JOIN ${tableName} ON ${tableName}.blobId = orig_sector.entityId - JOIN entity_changes dep_change ON dep_change.entityName = '${tableName}' AND dep_change.entityId = ${tableName}.${primaryKeyColumn} - WHERE orig_sector.entityName = 'blobs' AND SUBSTR(orig_sector.entityId, 1, 1) = ?`, - [sector] - ); - - for (const ec of dependingEntityChanges) { - putEntityChangeWithForcedChange(ec); - } - - return dependingEntityChanges.length; -} - -function cleanupEntityChangesForMissingEntities(entityName: string, entityPrimaryKey: string) { - sql.execute(` - DELETE - FROM entity_changes - WHERE - isErased = 0 - AND entityName = '${entityName}' - AND entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`); -} - -function fillEntityChanges(entityName: string, entityPrimaryKey: string, condition = "") { - cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey); - - sql.transactional(() => { - const entityIds = sql.getColumn(/*sql*/`SELECT ${entityPrimaryKey} FROM ${entityName} ${condition}`); - - let createdCount = 0; - - for (const entityId of entityIds) { - const existingRows = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]); - - if (existingRows !== 0) { - // we don't want to replace existing entities (which would effectively cause full resync) - continue; - } - - createdCount++; - - const ec: Partial = { - entityName, - entityId, - isErased: false - }; - - if (entityName === "blobs") { - const blob = sql.getRow>("SELECT blobId, content, utcDateModified FROM blobs WHERE blobId = ?", [entityId]); - ec.hash = blobService.calculateContentHash(blob); - ec.utcDateChanged = blob.utcDateModified; - ec.isSynced = true; // blobs are always synced - } else { - const entity = becca.getEntity(entityName, entityId); - - if (entity) { - ec.hash = entity.generateHash(); - ec.utcDateChanged = entity.getUtcDateChanged() || dateUtils.utcNowDateTime(); - ec.isSynced = entityName !== "options" || !!entity.isSynced; - } else { - // entity might be null (not present in becca) when it's deleted - // this will produce different hash value than when entity is being deleted since then - // all normal hashed attributes are being used. Sync should recover from that, though. - ec.hash = "deleted"; - ec.utcDateChanged = dateUtils.utcNowDateTime(); - ec.isSynced = true; // deletable (the ones with isDeleted) entities are synced - } - } - - putEntityChange(ec as EntityChange); - } - - if (createdCount > 0) { - log.info(`Created ${createdCount} missing entity changes for entity '${entityName}'.`); - } - }); -} - -function fillAllEntityChanges() { - sql.transactional(() => { - sql.execute("DELETE FROM entity_changes WHERE isErased = 0"); - - fillEntityChanges("notes", "noteId"); - fillEntityChanges("branches", "branchId"); - fillEntityChanges("revisions", "revisionId"); - fillEntityChanges("attachments", "attachmentId"); - fillEntityChanges("blobs", "blobId"); - fillEntityChanges("attributes", "attributeId"); - fillEntityChanges("etapi_tokens", "etapiTokenId"); - fillEntityChanges("options", "name", "WHERE isSynced = 1"); - }); -} - -function recalculateMaxEntityChangeId() { - maxEntityChangeId = sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes"); -} - -export default { - putNoteReorderingEntityChange, - putEntityChangeForOtherInstances, - putEntityChangeWithForcedChange, - putEntityChange, - putEntityChangeWithInstanceId, - fillAllEntityChanges, - addEntityChangesForSector, - getMaxEntityChangeId: () => maxEntityChangeId, - recalculateMaxEntityChangeId -}; +import { entity_changes } from "@triliumnext/core"; +export default entity_changes; diff --git a/apps/server/src/services/instance_id.ts b/apps/server/src/services/instance_id.ts index 31e620a8f..a61218020 100644 --- a/apps/server/src/services/instance_id.ts +++ b/apps/server/src/services/instance_id.ts @@ -1,5 +1,2 @@ -import { randomString } from "./utils.js"; - -const instanceId = randomString(12); - -export default instanceId; +import { instance_id } from "@triliumnext/core"; +export default instance_id; diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index 7a0d5ac69..5c10216df 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -18,8 +18,11 @@ export { default as options } from "./services/options"; export { default as options_init } from "./services/options_init"; export { default as app_info } from "./services/app_info"; export { default as keyboard_actions } from "./services/keyboard_actions"; +export { default as entity_changes } from "./services/entity_changes"; export { getContext, type ExecutionContext } from "./services/context"; +export * as cls from "./services/context"; export * from "./errors"; +export { default as instance_id } from "./services/instance_id"; export type { CryptoProvider } from "./services/encryption/crypto"; export { default as becca } from "./becca/becca"; diff --git a/packages/trilium-core/src/services/context.ts b/packages/trilium-core/src/services/context.ts index 870596ae3..f1276bc95 100644 --- a/packages/trilium-core/src/services/context.ts +++ b/packages/trilium-core/src/services/context.ts @@ -1,3 +1,5 @@ +import { EntityChange } from "@triliumnext/commons"; + export interface ExecutionContext { init(fn: () => T): T; get(key: string): T | undefined; @@ -24,3 +26,20 @@ export function getHoistedNoteId() { export function isEntityEventsDisabled() { return !!getContext().get("disableEntityEvents"); } + +export function getComponentId() { + return getContext().get("componentId"); +} + +export function putEntityChange(entityChange: EntityChange) { + if (getContext().get("ignoreEntityChangeIds")) { + return; + } + + const entityChangeIds = getContext().get("entityChangeIds") || []; + + // store only ID since the record can be modified (e.g., in erase) + entityChangeIds.push(entityChange.id); + + getContext().set("entityChangeIds", entityChangeIds); +} diff --git a/packages/trilium-core/src/services/entity_changes.ts b/packages/trilium-core/src/services/entity_changes.ts new file mode 100644 index 000000000..b389806d4 --- /dev/null +++ b/packages/trilium-core/src/services/entity_changes.ts @@ -0,0 +1,211 @@ +import type { BlobRow, EntityChange } from "@triliumnext/commons"; + +import becca from "../becca/becca.js"; +import dateUtils from "./utils/date.js"; +import instanceId from "./instance_id.js"; +import log, { getLog } from "./log.js"; +import { randomString } from "./utils/index.js"; +import { getSql } from "./sql/index.js"; +import { getComponentId } from "./context.js"; +import events from "./events.js"; +import blobService from "./blob.js"; + +let maxEntityChangeId = 0; + +function putEntityChangeWithInstanceId(origEntityChange: EntityChange, instanceId: string) { + const ec = { ...origEntityChange, instanceId }; + + putEntityChange(ec); +} + +function putEntityChangeWithForcedChange(origEntityChange: EntityChange) { + const ec = { ...origEntityChange, changeId: null }; + + putEntityChange(ec); +} + +function putEntityChange(origEntityChange: EntityChange) { + const ec = { ...origEntityChange }; + + delete ec.id; + + if (!ec.changeId) { + ec.changeId = randomString(12); + } + + ec.componentId = ec.componentId || getComponentId() || "NA"; // NA = not available + ec.instanceId = ec.instanceId || instanceId; + ec.isSynced = ec.isSynced ? 1 : 0; + ec.isErased = ec.isErased ? 1 : 0; + ec.id = getSql().replace("entity_changes", ec); + + if (ec.id) { + maxEntityChangeId = Math.max(maxEntityChangeId, ec.id); + } + + cls.putEntityChange(ec); +} + +function putNoteReorderingEntityChange(parentNoteId: string, componentId?: string) { + putEntityChange({ + entityName: "note_reordering", + entityId: parentNoteId, + hash: "N/A", + isErased: false, + utcDateChanged: dateUtils.utcNowDateTime(), + isSynced: true, + componentId, + instanceId + }); + + events.emit(events.ENTITY_CHANGED, { + entityName: "note_reordering", + entity: getSql().getMap(/*sql*/`SELECT branchId, notePosition FROM branches WHERE isDeleted = 0 AND parentNoteId = ?`, [parentNoteId]) + }); +} + +function putEntityChangeForOtherInstances(ec: EntityChange) { + putEntityChange({ + ...ec, + changeId: null, + instanceId: null + }); +} + +function addEntityChangesForSector(entityName: string, sector: string) { + const sql = getSql(); + const entityChanges = sql.getRows(/*sql*/`SELECT * FROM entity_changes WHERE entityName = ? AND SUBSTR(entityId, 1, 1) = ?`, [entityName, sector]); + + let entitiesInserted = entityChanges.length; + + sql.transactional(() => { + if (entityName === "blobs") { + entitiesInserted += addEntityChangesForDependingEntity(sector, "notes", "noteId"); + entitiesInserted += addEntityChangesForDependingEntity(sector, "attachments", "attachmentId"); + entitiesInserted += addEntityChangesForDependingEntity(sector, "revisions", "revisionId"); + } + + for (const ec of entityChanges) { + putEntityChangeWithForcedChange(ec); + } + }); + + getLog().info(`Added sector ${sector} of '${entityName}' (${entitiesInserted} entities) to the sync queue.`); +} + +function addEntityChangesForDependingEntity(sector: string, tableName: string, primaryKeyColumn: string) { + // problem in blobs might be caused by problem in entity referencing the blob + const dependingEntityChanges = getSql().getRows( + ` + SELECT dep_change.* + FROM entity_changes orig_sector + JOIN ${tableName} ON ${tableName}.blobId = orig_sector.entityId + JOIN entity_changes dep_change ON dep_change.entityName = '${tableName}' AND dep_change.entityId = ${tableName}.${primaryKeyColumn} + WHERE orig_sector.entityName = 'blobs' AND SUBSTR(orig_sector.entityId, 1, 1) = ?`, + [sector] + ); + + for (const ec of dependingEntityChanges) { + putEntityChangeWithForcedChange(ec); + } + + return dependingEntityChanges.length; +} + +function cleanupEntityChangesForMissingEntities(entityName: string, entityPrimaryKey: string) { + getSql().execute(` + DELETE + FROM entity_changes + WHERE + isErased = 0 + AND entityName = '${entityName}' + AND entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`); +} + +function fillEntityChanges(entityName: string, entityPrimaryKey: string, condition = "") { + cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey); + + const sql = getSql(); + sql.transactional(() => { + const entityIds = sql.getColumn(/*sql*/`SELECT ${entityPrimaryKey} FROM ${entityName} ${condition}`); + + let createdCount = 0; + + for (const entityId of entityIds) { + const existingRows = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]); + + if (existingRows !== 0) { + // we don't want to replace existing entities (which would effectively cause full resync) + continue; + } + + createdCount++; + + const ec: Partial = { + entityName, + entityId, + isErased: false + }; + + if (entityName === "blobs") { + const blob = sql.getRow>("SELECT blobId, content, utcDateModified FROM blobs WHERE blobId = ?", [entityId]); + ec.hash = blobService.calculateContentHash(blob); + ec.utcDateChanged = blob.utcDateModified; + ec.isSynced = true; // blobs are always synced + } else { + const entity = becca.getEntity(entityName, entityId); + + if (entity) { + ec.hash = entity.generateHash(); + ec.utcDateChanged = entity.getUtcDateChanged() || dateUtils.utcNowDateTime(); + ec.isSynced = entityName !== "options" || !!entity.isSynced; + } else { + // entity might be null (not present in becca) when it's deleted + // this will produce different hash value than when entity is being deleted since then + // all normal hashed attributes are being used. Sync should recover from that, though. + ec.hash = "deleted"; + ec.utcDateChanged = dateUtils.utcNowDateTime(); + ec.isSynced = true; // deletable (the ones with isDeleted) entities are synced + } + } + + putEntityChange(ec as EntityChange); + } + + if (createdCount > 0) { + getLog().info(`Created ${createdCount} missing entity changes for entity '${entityName}'.`); + } + }); +} + +function fillAllEntityChanges() { + const sql = getSql(); + sql.transactional(() => { + sql.execute("DELETE FROM entity_changes WHERE isErased = 0"); + + fillEntityChanges("notes", "noteId"); + fillEntityChanges("branches", "branchId"); + fillEntityChanges("revisions", "revisionId"); + fillEntityChanges("attachments", "attachmentId"); + fillEntityChanges("blobs", "blobId"); + fillEntityChanges("attributes", "attributeId"); + fillEntityChanges("etapi_tokens", "etapiTokenId"); + fillEntityChanges("options", "name", "WHERE isSynced = 1"); + }); +} + +function recalculateMaxEntityChangeId() { + maxEntityChangeId = getSql().getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes"); +} + +export default { + putNoteReorderingEntityChange, + putEntityChangeForOtherInstances, + putEntityChangeWithForcedChange, + putEntityChange, + putEntityChangeWithInstanceId, + fillAllEntityChanges, + addEntityChangesForSector, + getMaxEntityChangeId: () => maxEntityChangeId, + recalculateMaxEntityChangeId +}; diff --git a/packages/trilium-core/src/services/instance_id.ts b/packages/trilium-core/src/services/instance_id.ts new file mode 100644 index 000000000..cd8257edf --- /dev/null +++ b/packages/trilium-core/src/services/instance_id.ts @@ -0,0 +1,5 @@ +import { randomString } from "./utils"; + +const instanceId = randomString(12); + +export default instanceId;