chore(core): integrate entity_changes

This commit is contained in:
Elian Doran 2026-01-06 13:42:29 +02:00
parent 60cb8d950e
commit 64b212b93e
No known key found for this signature in database
7 changed files with 258 additions and 237 deletions

View File

@ -1,10 +1,10 @@
import type { EntityChange } from "@triliumnext/commons";
import { getContext, getHoistedNoteId as getHoistedNoteIdInternal, isEntityEventsDisabled as isEntityEventsDisabledInternal } from "@triliumnext/core/src/services/context";
import { cls } from "@triliumnext/core";
type Callback = (...args: any[]) => any;
function init<T>(callback: () => T) {
return getContext().init(callback);
return cls.getContext().init(callback);
}
function wrap(callback: Callback) {
@ -18,68 +18,59 @@ function wrap(callback: Callback) {
}
function getHoistedNoteId() {
return getHoistedNoteIdInternal();
return cls.getHoistedNoteId();
}
function getComponentId() {
return getContext().get("componentId");
return cls.getComponentId();
}
function disableEntityEvents() {
getContext().set("disableEntityEvents", true);
cls.getContext().set("disableEntityEvents", true);
}
function enableEntityEvents() {
getContext().set("disableEntityEvents", false);
cls.getContext().set("disableEntityEvents", false);
}
function isEntityEventsDisabled() {
return isEntityEventsDisabledInternal();
return cls.isEntityEventsDisabled();
}
function setMigrationRunning(running: boolean) {
getContext().set("migrationRunning", !!running);
cls.getContext().set("migrationRunning", !!running);
}
function isMigrationRunning() {
return !!getContext().get("migrationRunning");
return !!cls.getContext().get("migrationRunning");
}
function getAndClearEntityChangeIds() {
const entityChangeIds = getContext().get("entityChangeIds") || [];
const entityChangeIds = cls.getContext().get("entityChangeIds") || [];
getContext().set("entityChangeIds", []);
cls.getContext().set("entityChangeIds", []);
return entityChangeIds;
}
function putEntityChange(entityChange: EntityChange) {
if (getContext().get("ignoreEntityChangeIds")) {
return;
}
const entityChangeIds = getContext().get("entityChangeIds") || [];
// store only ID since the record can be modified (e.g., in erase)
entityChangeIds.push(entityChange.id);
getContext().set("entityChangeIds", entityChangeIds);
cls.putEntityChange(entityChange);
}
function ignoreEntityChangeIds() {
getContext().set("ignoreEntityChangeIds", true);
cls.getContext().set("ignoreEntityChangeIds", true);
}
function get(key: string) {
return getContext().get(key);
return cls.getContext().get(key);
}
function set(key: string, value: unknown) {
getContext().set(key, value);
cls.getContext().set(key, value);
}
function reset() {
getContext().reset();
cls.getContext().reset();
}
export default {

View File

@ -1,207 +1,2 @@
import type { BlobRow, EntityChange } from "@triliumnext/commons";
import { blob as blobService, events as eventService } from "@triliumnext/core";
import becca from "../becca/becca.js";
import cls from "./cls.js";
import dateUtils from "./date_utils.js";
import instanceId from "./instance_id.js";
import log from "./log.js";
import sql from "./sql.js";
import { randomString } from "./utils.js";
let maxEntityChangeId = 0;
function putEntityChangeWithInstanceId(origEntityChange: EntityChange, instanceId: string) {
const ec = { ...origEntityChange, instanceId };
putEntityChange(ec);
}
function putEntityChangeWithForcedChange(origEntityChange: EntityChange) {
const ec = { ...origEntityChange, changeId: null };
putEntityChange(ec);
}
function putEntityChange(origEntityChange: EntityChange) {
const ec = { ...origEntityChange };
delete ec.id;
if (!ec.changeId) {
ec.changeId = randomString(12);
}
ec.componentId = ec.componentId || cls.getComponentId() || "NA"; // NA = not available
ec.instanceId = ec.instanceId || instanceId;
ec.isSynced = ec.isSynced ? 1 : 0;
ec.isErased = ec.isErased ? 1 : 0;
ec.id = sql.replace("entity_changes", ec);
if (ec.id) {
maxEntityChangeId = Math.max(maxEntityChangeId, ec.id);
}
cls.putEntityChange(ec);
}
function putNoteReorderingEntityChange(parentNoteId: string, componentId?: string) {
putEntityChange({
entityName: "note_reordering",
entityId: parentNoteId,
hash: "N/A",
isErased: false,
utcDateChanged: dateUtils.utcNowDateTime(),
isSynced: true,
componentId,
instanceId
});
eventService.emit(eventService.ENTITY_CHANGED, {
entityName: "note_reordering",
entity: sql.getMap(/*sql*/`SELECT branchId, notePosition FROM branches WHERE isDeleted = 0 AND parentNoteId = ?`, [parentNoteId])
});
}
function putEntityChangeForOtherInstances(ec: EntityChange) {
putEntityChange({
...ec,
changeId: null,
instanceId: null
});
}
function addEntityChangesForSector(entityName: string, sector: string) {
const entityChanges = sql.getRows<EntityChange>(/*sql*/`SELECT * FROM entity_changes WHERE entityName = ? AND SUBSTR(entityId, 1, 1) = ?`, [entityName, sector]);
let entitiesInserted = entityChanges.length;
sql.transactional(() => {
if (entityName === "blobs") {
entitiesInserted += addEntityChangesForDependingEntity(sector, "notes", "noteId");
entitiesInserted += addEntityChangesForDependingEntity(sector, "attachments", "attachmentId");
entitiesInserted += addEntityChangesForDependingEntity(sector, "revisions", "revisionId");
}
for (const ec of entityChanges) {
putEntityChangeWithForcedChange(ec);
}
});
log.info(`Added sector ${sector} of '${entityName}' (${entitiesInserted} entities) to the sync queue.`);
}
function addEntityChangesForDependingEntity(sector: string, tableName: string, primaryKeyColumn: string) {
// problem in blobs might be caused by problem in entity referencing the blob
const dependingEntityChanges = sql.getRows<EntityChange>(
`
SELECT dep_change.*
FROM entity_changes orig_sector
JOIN ${tableName} ON ${tableName}.blobId = orig_sector.entityId
JOIN entity_changes dep_change ON dep_change.entityName = '${tableName}' AND dep_change.entityId = ${tableName}.${primaryKeyColumn}
WHERE orig_sector.entityName = 'blobs' AND SUBSTR(orig_sector.entityId, 1, 1) = ?`,
[sector]
);
for (const ec of dependingEntityChanges) {
putEntityChangeWithForcedChange(ec);
}
return dependingEntityChanges.length;
}
function cleanupEntityChangesForMissingEntities(entityName: string, entityPrimaryKey: string) {
sql.execute(`
DELETE
FROM entity_changes
WHERE
isErased = 0
AND entityName = '${entityName}'
AND entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`);
}
function fillEntityChanges(entityName: string, entityPrimaryKey: string, condition = "") {
cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey);
sql.transactional(() => {
const entityIds = sql.getColumn<string>(/*sql*/`SELECT ${entityPrimaryKey} FROM ${entityName} ${condition}`);
let createdCount = 0;
for (const entityId of entityIds) {
const existingRows = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
if (existingRows !== 0) {
// we don't want to replace existing entities (which would effectively cause full resync)
continue;
}
createdCount++;
const ec: Partial<EntityChange> = {
entityName,
entityId,
isErased: false
};
if (entityName === "blobs") {
const blob = sql.getRow<Pick<BlobRow, "blobId" | "content" | "utcDateModified">>("SELECT blobId, content, utcDateModified FROM blobs WHERE blobId = ?", [entityId]);
ec.hash = blobService.calculateContentHash(blob);
ec.utcDateChanged = blob.utcDateModified;
ec.isSynced = true; // blobs are always synced
} else {
const entity = becca.getEntity(entityName, entityId);
if (entity) {
ec.hash = entity.generateHash();
ec.utcDateChanged = entity.getUtcDateChanged() || dateUtils.utcNowDateTime();
ec.isSynced = entityName !== "options" || !!entity.isSynced;
} else {
// entity might be null (not present in becca) when it's deleted
// this will produce different hash value than when entity is being deleted since then
// all normal hashed attributes are being used. Sync should recover from that, though.
ec.hash = "deleted";
ec.utcDateChanged = dateUtils.utcNowDateTime();
ec.isSynced = true; // deletable (the ones with isDeleted) entities are synced
}
}
putEntityChange(ec as EntityChange);
}
if (createdCount > 0) {
log.info(`Created ${createdCount} missing entity changes for entity '${entityName}'.`);
}
});
}
function fillAllEntityChanges() {
sql.transactional(() => {
sql.execute("DELETE FROM entity_changes WHERE isErased = 0");
fillEntityChanges("notes", "noteId");
fillEntityChanges("branches", "branchId");
fillEntityChanges("revisions", "revisionId");
fillEntityChanges("attachments", "attachmentId");
fillEntityChanges("blobs", "blobId");
fillEntityChanges("attributes", "attributeId");
fillEntityChanges("etapi_tokens", "etapiTokenId");
fillEntityChanges("options", "name", "WHERE isSynced = 1");
});
}
function recalculateMaxEntityChangeId() {
maxEntityChangeId = sql.getValue<number>("SELECT COALESCE(MAX(id), 0) FROM entity_changes");
}
export default {
putNoteReorderingEntityChange,
putEntityChangeForOtherInstances,
putEntityChangeWithForcedChange,
putEntityChange,
putEntityChangeWithInstanceId,
fillAllEntityChanges,
addEntityChangesForSector,
getMaxEntityChangeId: () => maxEntityChangeId,
recalculateMaxEntityChangeId
};
import { entity_changes } from "@triliumnext/core";
export default entity_changes;

View File

@ -1,5 +1,2 @@
import { randomString } from "./utils.js";
const instanceId = randomString(12);
export default instanceId;
import { instance_id } from "@triliumnext/core";
export default instance_id;

View File

@ -18,8 +18,11 @@ export { default as options } from "./services/options";
export { default as options_init } from "./services/options_init";
export { default as app_info } from "./services/app_info";
export { default as keyboard_actions } from "./services/keyboard_actions";
export { default as entity_changes } from "./services/entity_changes";
export { getContext, type ExecutionContext } from "./services/context";
export * as cls from "./services/context";
export * from "./errors";
export { default as instance_id } from "./services/instance_id";
export type { CryptoProvider } from "./services/encryption/crypto";
export { default as becca } from "./becca/becca";

View File

@ -1,3 +1,5 @@
import { EntityChange } from "@triliumnext/commons";
export interface ExecutionContext {
init<T>(fn: () => T): T;
get<T = any>(key: string): T | undefined;
@ -24,3 +26,20 @@ export function getHoistedNoteId() {
export function isEntityEventsDisabled() {
return !!getContext().get("disableEntityEvents");
}
export function getComponentId() {
return getContext().get("componentId");
}
export function putEntityChange(entityChange: EntityChange) {
if (getContext().get("ignoreEntityChangeIds")) {
return;
}
const entityChangeIds = getContext().get("entityChangeIds") || [];
// store only ID since the record can be modified (e.g., in erase)
entityChangeIds.push(entityChange.id);
getContext().set("entityChangeIds", entityChangeIds);
}

View File

@ -0,0 +1,211 @@
import type { BlobRow, EntityChange } from "@triliumnext/commons";
import becca from "../becca/becca.js";
import dateUtils from "./utils/date.js";
import instanceId from "./instance_id.js";
import log, { getLog } from "./log.js";
import { randomString } from "./utils/index.js";
import { getSql } from "./sql/index.js";
import { getComponentId } from "./context.js";
import events from "./events.js";
import blobService from "./blob.js";
let maxEntityChangeId = 0;
function putEntityChangeWithInstanceId(origEntityChange: EntityChange, instanceId: string) {
const ec = { ...origEntityChange, instanceId };
putEntityChange(ec);
}
function putEntityChangeWithForcedChange(origEntityChange: EntityChange) {
const ec = { ...origEntityChange, changeId: null };
putEntityChange(ec);
}
function putEntityChange(origEntityChange: EntityChange) {
const ec = { ...origEntityChange };
delete ec.id;
if (!ec.changeId) {
ec.changeId = randomString(12);
}
ec.componentId = ec.componentId || getComponentId() || "NA"; // NA = not available
ec.instanceId = ec.instanceId || instanceId;
ec.isSynced = ec.isSynced ? 1 : 0;
ec.isErased = ec.isErased ? 1 : 0;
ec.id = getSql().replace("entity_changes", ec);
if (ec.id) {
maxEntityChangeId = Math.max(maxEntityChangeId, ec.id);
}
cls.putEntityChange(ec);
}
function putNoteReorderingEntityChange(parentNoteId: string, componentId?: string) {
putEntityChange({
entityName: "note_reordering",
entityId: parentNoteId,
hash: "N/A",
isErased: false,
utcDateChanged: dateUtils.utcNowDateTime(),
isSynced: true,
componentId,
instanceId
});
events.emit(events.ENTITY_CHANGED, {
entityName: "note_reordering",
entity: getSql().getMap(/*sql*/`SELECT branchId, notePosition FROM branches WHERE isDeleted = 0 AND parentNoteId = ?`, [parentNoteId])
});
}
function putEntityChangeForOtherInstances(ec: EntityChange) {
putEntityChange({
...ec,
changeId: null,
instanceId: null
});
}
function addEntityChangesForSector(entityName: string, sector: string) {
const sql = getSql();
const entityChanges = sql.getRows<EntityChange>(/*sql*/`SELECT * FROM entity_changes WHERE entityName = ? AND SUBSTR(entityId, 1, 1) = ?`, [entityName, sector]);
let entitiesInserted = entityChanges.length;
sql.transactional(() => {
if (entityName === "blobs") {
entitiesInserted += addEntityChangesForDependingEntity(sector, "notes", "noteId");
entitiesInserted += addEntityChangesForDependingEntity(sector, "attachments", "attachmentId");
entitiesInserted += addEntityChangesForDependingEntity(sector, "revisions", "revisionId");
}
for (const ec of entityChanges) {
putEntityChangeWithForcedChange(ec);
}
});
getLog().info(`Added sector ${sector} of '${entityName}' (${entitiesInserted} entities) to the sync queue.`);
}
function addEntityChangesForDependingEntity(sector: string, tableName: string, primaryKeyColumn: string) {
// problem in blobs might be caused by problem in entity referencing the blob
const dependingEntityChanges = getSql().getRows<EntityChange>(
`
SELECT dep_change.*
FROM entity_changes orig_sector
JOIN ${tableName} ON ${tableName}.blobId = orig_sector.entityId
JOIN entity_changes dep_change ON dep_change.entityName = '${tableName}' AND dep_change.entityId = ${tableName}.${primaryKeyColumn}
WHERE orig_sector.entityName = 'blobs' AND SUBSTR(orig_sector.entityId, 1, 1) = ?`,
[sector]
);
for (const ec of dependingEntityChanges) {
putEntityChangeWithForcedChange(ec);
}
return dependingEntityChanges.length;
}
function cleanupEntityChangesForMissingEntities(entityName: string, entityPrimaryKey: string) {
getSql().execute(`
DELETE
FROM entity_changes
WHERE
isErased = 0
AND entityName = '${entityName}'
AND entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`);
}
function fillEntityChanges(entityName: string, entityPrimaryKey: string, condition = "") {
cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey);
const sql = getSql();
sql.transactional(() => {
const entityIds = sql.getColumn<string>(/*sql*/`SELECT ${entityPrimaryKey} FROM ${entityName} ${condition}`);
let createdCount = 0;
for (const entityId of entityIds) {
const existingRows = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
if (existingRows !== 0) {
// we don't want to replace existing entities (which would effectively cause full resync)
continue;
}
createdCount++;
const ec: Partial<EntityChange> = {
entityName,
entityId,
isErased: false
};
if (entityName === "blobs") {
const blob = sql.getRow<Pick<BlobRow, "blobId" | "content" | "utcDateModified">>("SELECT blobId, content, utcDateModified FROM blobs WHERE blobId = ?", [entityId]);
ec.hash = blobService.calculateContentHash(blob);
ec.utcDateChanged = blob.utcDateModified;
ec.isSynced = true; // blobs are always synced
} else {
const entity = becca.getEntity(entityName, entityId);
if (entity) {
ec.hash = entity.generateHash();
ec.utcDateChanged = entity.getUtcDateChanged() || dateUtils.utcNowDateTime();
ec.isSynced = entityName !== "options" || !!entity.isSynced;
} else {
// entity might be null (not present in becca) when it's deleted
// this will produce different hash value than when entity is being deleted since then
// all normal hashed attributes are being used. Sync should recover from that, though.
ec.hash = "deleted";
ec.utcDateChanged = dateUtils.utcNowDateTime();
ec.isSynced = true; // deletable (the ones with isDeleted) entities are synced
}
}
putEntityChange(ec as EntityChange);
}
if (createdCount > 0) {
getLog().info(`Created ${createdCount} missing entity changes for entity '${entityName}'.`);
}
});
}
function fillAllEntityChanges() {
const sql = getSql();
sql.transactional(() => {
sql.execute("DELETE FROM entity_changes WHERE isErased = 0");
fillEntityChanges("notes", "noteId");
fillEntityChanges("branches", "branchId");
fillEntityChanges("revisions", "revisionId");
fillEntityChanges("attachments", "attachmentId");
fillEntityChanges("blobs", "blobId");
fillEntityChanges("attributes", "attributeId");
fillEntityChanges("etapi_tokens", "etapiTokenId");
fillEntityChanges("options", "name", "WHERE isSynced = 1");
});
}
function recalculateMaxEntityChangeId() {
maxEntityChangeId = getSql().getValue<number>("SELECT COALESCE(MAX(id), 0) FROM entity_changes");
}
export default {
putNoteReorderingEntityChange,
putEntityChangeForOtherInstances,
putEntityChangeWithForcedChange,
putEntityChange,
putEntityChangeWithInstanceId,
fillAllEntityChanges,
addEntityChangesForSector,
getMaxEntityChangeId: () => maxEntityChangeId,
recalculateMaxEntityChangeId
};

View File

@ -0,0 +1,5 @@
import { randomString } from "./utils";
const instanceId = randomString(12);
export default instanceId;