gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

move entity hash to entity_changes table WIP

Browse Source
This commit is contained in:
zadam 2020-12-07 23:04:17 +01:00
parent 16c4b8aa43
commit 7f3ef2cb8b
14 changed files with 107 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
db/migrations/0173__move_hash_to_entity_changes.sql Normal file
View File

4
src/entities/attribute.js
View File

@ -109,9 +109,7 @@ class Attribute extends Entity {
super.beforeSaving();
if (this.isChanged) {
this.utcDateModified = dateUtils.utcNowDateTime();
}
this.utcDateModified = dateUtils.utcNowDateTime();
}
createClone(type, name, value, isInheritable) {

4
src/entities/branch.js
View File

@ -57,9 +57,7 @@ class Branch extends Entity {
super.beforeSaving();
if (this.isChanged) {
this.utcDateModified = dateUtils.utcNowDateTime();
}
this.utcDateModified = dateUtils.utcNowDateTime();
}
createClone(parentNoteId, notePosition) {

5
src/entities/entity.js
View File

@ -22,11 +22,6 @@ class Entity {
beforeSaving() {
this.generateIdIfNecessary();
const origHash = this.hash;
this.hash = this.generateHash();
this.isChanged = origHash !== this.hash;
}
generateIdIfNecessary() {

13
src/entities/note.js
View File

@ -143,8 +143,7 @@ class Note extends Entity {
noteId: this.noteId,
content: content,
dateModified: dateUtils.localNowDateTime(),
utcDateModified: dateUtils.utcNowDateTime(),
hash: utils.hash(this.noteId + "|" + content.toString())
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) {
@ -158,7 +157,9 @@ class Note extends Entity {
sql.upsert("note_contents", "noteId", pojo);
entityChangesService.addNoteContentEntityChange(this.noteId);
const hash = utils.hash(this.noteId + "|" + content.toString());
entityChangesService.addEntityChange('note_contents', this.noteId, hash);
}
setJsonContent(content) {
@ -904,10 +905,8 @@ class Note extends Entity {
super.beforeSaving();
if (this.isChanged) {
this.dateModified = dateUtils.localNowDateTime();
this.utcDateModified = dateUtils.utcNowDateTime();
}
this.dateModified = dateUtils.localNowDateTime();
this.utcDateModified = dateUtils.utcNowDateTime();
}
// cannot be static!

11
src/entities/note_revision.js
View File

@ -106,8 +106,7 @@ class NoteRevision extends Entity {
const pojo = {
noteRevisionId: this.noteRevisionId,
content: content,
utcDateModified: dateUtils.utcNowDateTime(),
hash: utils.hash(this.noteRevisionId + "|" + content)
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) {
@ -121,15 +120,15 @@ class NoteRevision extends Entity {
sql.upsert("note_revision_contents", "noteRevisionId", pojo);
entityChangesService.addNoteRevisionContentEntityChange(this.noteRevisionId);
const hash = utils.hash(this.noteRevisionId + "|" + content);
entityChangesService.addEntityChange('note_revision_contents', this.noteRevisionId, hash);
}
beforeSaving() {
super.beforeSaving();
if (this.isChanged) {
this.utcDateModified = dateUtils.utcNowDateTime();
}
this.utcDateModified = dateUtils.utcNowDateTime();
}
// cannot be static!

6
src/entities/option.js
View File

@ -32,10 +32,8 @@ class Option extends Entity {
super.beforeSaving();
if (this.isChanged) {
this.utcDateModified = dateUtils.utcNowDateTime();
}
this.utcDateModified = dateUtils.utcNowDateTime();
}
}
module.exports = Option;
module.exports = Option;

14
src/routes/api/sync.js
View File

@ -86,32 +86,32 @@ function forceNoteSync(req) {
const now = dateUtils.utcNowDateTime();
sql.execute(`UPDATE notes SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]);
entityChangesService.addNoteEntityChange(noteId);
entityChangesService.moveEntityChangeToTop('notes', noteId);
sql.execute(`UPDATE note_contents SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]);
entityChangesService.addNoteContentEntityChange(noteId);
entityChangesService.moveEntityChangeToTop('note_contents', noteId);
for (const branchId of sql.getColumn("SELECT branchId FROM branches WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE branches SET utcDateModified = ? WHERE branchId = ?`, [now, branchId]);
entityChangesService.addBranchEntityChange(branchId);
entityChangesService.moveEntityChangeToTop('branches', branchId);
}
for (const attributeId of sql.getColumn("SELECT attributeId FROM attributes WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE attributes SET utcDateModified = ? WHERE attributeId = ?`, [now, attributeId]);
entityChangesService.addAttributeEntityChange(attributeId);
entityChangesService.moveEntityChangeToTop('attributes', attributeId);
}
for (const noteRevisionId of sql.getColumn("SELECT noteRevisionId FROM note_revisions WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE note_revisions SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]);
entityChangesService.addNoteRevisionEntityChange(noteRevisionId);
entityChangesService.moveEntityChangeToTop('note_revisions', noteRevisionId);
sql.execute(`UPDATE note_revision_contents SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]);
entityChangesService.addNoteRevisionContentEntityChange(noteRevisionId);
entityChangesService.moveEntityChangeToTop('note_revision_contents', noteRevisionId);
}
entityChangesService.addRecentNoteEntityChange(noteId);
entityChangesService.moveEntityChangeToTop('recent_changes', noteId);
log.info("Forcing note sync for " + noteId);

4
src/services/app_info.js
View File

@ -4,8 +4,8 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 172;
const SYNC_VERSION = 16;
const APP_DB_VERSION = 173;
const SYNC_VERSION = 17;
const CLIPPER_PROTOCOL_VERSION = "1.0";
module.exports = {

7
src/services/consistency_checks.js
View File

@ -297,11 +297,10 @@ class ConsistencyChecks {
sql.upsert("note_contents", "noteId", {
noteId: noteId,
content: null,
hash: "consistency_checks",
utcDateModified: dateUtils.utcNowDateTime()
});
entityChangesService.addNoteContentEntityChange(noteId);
entityChangesService.addEntityChange('note_contents', noteId, "consistency_checks");
}
else {
// empty string might be wrong choice for some note types but it's a best guess
@ -566,7 +565,9 @@ class ConsistencyChecks {
entity_changes.id IS NULL AND ` + (entityName === 'options' ? 'options.isSynced = 1' : '1'),
({entityId}) => {
if (this.autoFix) {
entityChangesService.addEntityChange(entityName, entityId);
const entity = repository.getEntity(`SELECT * FROM ${entityName} WHERE ${key} = ?`, [entityId]);
entityChangesService.addEntityChange(entityName, entityId, entity.generateHash());
logFix(`Created missing entity change for entityName=${entityName}, entityId=${entityId}`);
} else {

34
src/services/content_hash.js
View File

@ -34,23 +34,33 @@ function getSectorHashes(tableName, primaryKeyName, whereBranch) {
function getEntityHashes() {
const startTime = new Date();
const hashes = {
notes: getSectorHashes(Note.entityName, Note.primaryKeyName),
note_contents: getSectorHashes("note_contents", "noteId"),
branches: getSectorHashes(Branch.entityName, Branch.primaryKeyName),
note_revisions: getSectorHashes(NoteRevision.entityName, NoteRevision.primaryKeyName),
note_revision_contents: getSectorHashes("note_revision_contents", "noteRevisionId"),
recent_notes: getSectorHashes(RecentNote.entityName, RecentNote.primaryKeyName),
options: getSectorHashes(Option.entityName, Option.primaryKeyName, "isSynced = 1"),
attributes: getSectorHashes(Attribute.entityName, Attribute.primaryKeyName),
api_tokens: getSectorHashes(ApiToken.entityName, ApiToken.primaryKeyName),
};
const hashRows = sql.getRows(`SELECT entityName, entityId, hash FROM entity_changes`);
// sorting is faster in memory
// sorting by entityId is enough, hashes will be segmented by entityName later on anyway
hashRows.sort((a, b) => a.entityId < b.entityId ? -1 : 1);
const hashMap = {};
for (const {entityName, entityId, hash} of hashRows) {
const entityHashMap = hashMap[entityName] = hashMap[entityName] || {};
const sector = entityId[0];
entityHashMap[sector] = (entityHashMap[sector] || "") + hash
}
for (const entityHashMap of Object.values(hashMap)) {
for (const key in entityHashMap) {
entityHashMap[key] = utils.hash(entityHashMap[key]);
}
}
const elapsedTimeMs = Date.now() - startTime.getTime();
log.info(`Content hash computation took ${elapsedTimeMs}ms`);
return hashes;
return hashMap;
}
function checkContentHashes(otherHashes) {

35
src/services/entity_changes.js
View File

@ -1,4 +1,5 @@
const sql = require('./sql');
const repository = require('repository');
const sourceIdService = require('./source_id');
const dateUtils = require('./date_utils');
const log = require('./log');
@ -6,10 +7,11 @@ const cls = require('./cls');
let maxEntityChangeId = 0;
function insertEntityChange(entityName, entityId, sourceId = null, isSynced = true) {
function insertEntityChange(entityName, entityId, hash, sourceId = null, isSynced = true) {
const entityChange = {
entityName: entityName,
entityId: entityId,
hash: hash,
utcChangedDate: dateUtils.utcNowDateTime(),
sourceId: sourceId || cls.getSourceId() || sourceIdService.getCurrentSourceId(),
isSynced: isSynced ? 1 : 0
@ -22,12 +24,18 @@ function insertEntityChange(entityName, entityId, sourceId = null, isSynced = tr
return entityChange;
}
function addEntityChange(entityName, entityId, sourceId, isSynced) {
const sync = insertEntityChange(entityName, entityId, sourceId, isSynced);
function addEntityChange(entityName, entityId, hash, sourceId, isSynced) {
const sync = insertEntityChange(entityName, entityId, hash, sourceId, isSynced);
cls.addSyncRow(sync);
}
function moveEntityChangeToTop(entityName, entityId) {
const entityChange = sql.getRow(`SELECT * FROM entity_changes WHERE entityName = ? AND entityId = ?`, [entityName, entityId]);
addEntityChange(entityName, entityId, entityChange.hash, null, entityChange.isSynced);
}
function addEntityChangesForSector(entityName, entityPrimaryKey, sector) {
const startTime = Date.now();
@ -35,15 +43,14 @@ function addEntityChangesForSector(entityName, entityPrimaryKey, sector) {
const entityIds = sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]);
for (const entityId of entityIds) {
if (entityName === 'options') {
const isSynced = sql.getValue(`SELECT isSynced FROM options WHERE name = ?`, [entityId]);
// retrieving entity one by one to avoid memory issues with note_contents
const entity = repository.getEntity(`SELECT * FROM ${entityName} WHERE ${entityPrimaryKey} = ?`, [entityId]);
if (!isSynced) {
continue;
}
if (entityName === 'options' && !entity.isSynced) {
continue
}
insertEntityChange(entityName, entityId, 'content-check', true);
insertEntityChange(entityName, entityId, entity.generateHash(), 'content-check', true);
}
});
@ -112,16 +119,8 @@ function fillAllEntityChanges() {
}
module.exports = {
addNoteEntityChange: (noteId, sourceId) => addEntityChange("notes", noteId, sourceId),
addNoteContentEntityChange: (noteId, sourceId) => addEntityChange("note_contents", noteId, sourceId),
addBranchEntityChange: (branchId, sourceId) => addEntityChange("branches", branchId, sourceId),
addNoteReorderingEntityChange: (parentNoteId, sourceId) => addEntityChange("note_reordering", parentNoteId, sourceId),
addNoteRevisionEntityChange: (noteRevisionId, sourceId) => addEntityChange("note_revisions", noteRevisionId, sourceId),
addNoteRevisionContentEntityChange: (noteRevisionId, sourceId) => addEntityChange("note_revision_contents", noteRevisionId, sourceId),
addOptionEntityChange: (name, sourceId, isSynced) => addEntityChange("options", name, sourceId, isSynced),
addRecentNoteEntityChange: (noteId, sourceId) => addEntityChange("recent_notes", noteId, sourceId),
addAttributeEntityChange: (attributeId, sourceId) => addEntityChange("attributes", attributeId, sourceId),
addApiTokenEntityChange: (apiTokenId, sourceId) => addEntityChange("api_tokens", apiTokenId, sourceId),
moveEntityChangeToTop,
addEntityChange,
fillAllEntityChanges,
addEntityChangesForSector,

27
src/services/repository.js
View File

@ -101,9 +101,6 @@ function updateEntity(entity) {
entity.updatePojo(clone);
}
// indicates whether entity actually changed
delete clone.isChanged;
for (const key in clone) {
// !isBuffer is for images and attachments
if (clone[key] !== null && typeof clone[key] === 'object' && !Buffer.isBuffer(clone[key])) {
@ -116,23 +113,21 @@ function updateEntity(entity) {
const primaryKey = entity[primaryKeyName];
if (entity.isChanged) {
const isSynced = entityName !== 'options' || entity.isSynced;
const isSynced = entityName !== 'options' || entity.isSynced;
entityChangesService.addEntityChange(entityName, primaryKey, null, isSynced);
entityChangesService.addEntityChange(entityName, primaryKey, null, isSynced);
if (!cls.isEntityEventsDisabled()) {
const eventPayload = {
entityName,
entity
};
if (!cls.isEntityEventsDisabled()) {
const eventPayload = {
entityName,
entity
};
if (isNewEntity && !entity.isDeleted) {
eventService.emit(eventService.ENTITY_CREATED, eventPayload);
}
eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload);
if (isNewEntity && !entity.isDeleted) {
eventService.emit(eventService.ENTITY_CREATED, eventPayload);
}
eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload);
}
});
}

58
src/services/sync_update.js
View File

@ -9,38 +9,38 @@ function updateEntity(entityChange, entity, sourceId) {
return false;
}
const {entityName} = entityChange;
const {entityName, hash} = entityChange;
let updated;
if (entityName === 'notes') {
updated = updateNote(entity, sourceId);
updated = updateNote(entity, hash, sourceId);
}
else if (entityName === 'note_contents') {
updated = updateNoteContent(entity, sourceId);
updated = updateNoteContent(entity, hash, sourceId);
}
else if (entityName === 'branches') {
updated = updateBranch(entity, sourceId);
updated = updateBranch(entity, hash, sourceId);
}
else if (entityName === 'note_revisions') {
updated = updateNoteRevision(entity, sourceId);
updated = updateNoteRevision(entity, hash, sourceId);
}
else if (entityName === 'note_revision_contents') {
updated = updateNoteRevisionContent(entity, sourceId);
updated = updateNoteRevisionContent(entity, hash, sourceId);
}
else if (entityName === 'note_reordering') {
updated = updateNoteReordering(entityChange.entityId, entity, sourceId);
}
else if (entityName === 'options') {
updated = updateOptions(entity, sourceId);
updated = updateOptions(entity, hash, sourceId);
}
else if (entityName === 'recent_notes') {
updated = updateRecentNotes(entity, sourceId);
updated = updateRecentNotes(entity, hash, sourceId);
}
else if (entityName === 'attributes') {
updated = updateAttribute(entity, sourceId);
updated = updateAttribute(entity, hash, sourceId);
}
else if (entityName === 'api_tokens') {
updated = updateApiToken(entity, sourceId);
updated = updateApiToken(entity, hash, sourceId);
}
else {
throw new Error(`Unrecognized entity type ${entityName}`);
@ -79,14 +79,14 @@ function shouldWeUpdateEntity(localEntity, remoteEntity) {
return false;
}
function updateNote(remoteEntity, sourceId) {
function updateNote(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace("notes", remoteEntity);
entityChangesService.addNoteEntityChange(remoteEntity.noteId, sourceId);
entityChangesService.addEntityChange('notes', remoteEntity.noteId, hash, sourceId);
});
return true;
@ -95,7 +95,7 @@ function updateNote(remoteEntity, sourceId) {
return false;
}
function updateNoteContent(remoteEntity, sourceId) {
function updateNoteContent(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM note_contents WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
@ -111,7 +111,7 @@ function updateNoteContent(remoteEntity, sourceId) {
sql.transactional(() => {
sql.replace("note_contents", remoteEntity);
entityChangesService.addNoteContentEntityChange(remoteEntity.noteId, sourceId);
entityChangesService.addEntityChange("note_contents", remoteEntity.noteId, hash, sourceId);
});
return true;
@ -120,7 +120,7 @@ function updateNoteContent(remoteEntity, sourceId) {
return false;
}
function updateBranch(remoteEntity, sourceId) {
function updateBranch(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM branches WHERE branchId = ?", [remoteEntity.branchId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
@ -133,7 +133,7 @@ function updateBranch(remoteEntity, sourceId) {
sql.replace('branches', remoteEntity);
entityChangesService.addBranchEntityChange(remoteEntity.branchId, sourceId);
entityChangesService.addEntityChange('branches', remoteEntity.branchId, hash, sourceId);
});
return true;
@ -142,21 +142,21 @@ function updateBranch(remoteEntity, sourceId) {
return false;
}
function updateNoteRevision(remoteEntity, sourceId) {
function updateNoteRevision(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM note_revisions WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
sql.transactional(() => {
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.replace('note_revisions', remoteEntity);
entityChangesService.addNoteRevisionEntityChange(remoteEntity.noteRevisionId, sourceId);
entityChangesService.addEntityChange('note_revisions', remoteEntity.noteRevisionId, hash, sourceId);
log.info("Update/sync note revision " + remoteEntity.noteRevisionId);
}
});
}
function updateNoteRevisionContent(remoteEntity, sourceId) {
function updateNoteRevisionContent(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM note_revision_contents WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
@ -165,7 +165,7 @@ function updateNoteRevisionContent(remoteEntity, sourceId) {
sql.replace('note_revision_contents', remoteEntity);
entityChangesService.addNoteRevisionContentEntityChange(remoteEntity.noteRevisionId, sourceId);
entityChangesService.addEntityChange('note_revision_contents', remoteEntity.noteRevisionId, hash, sourceId);
});
return true;
@ -180,13 +180,13 @@ function updateNoteReordering(entityId, remote, sourceId) {
sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remote[key], key]);
}
entityChangesService.addNoteReorderingEntityChange(entityId, sourceId);
entityChangesService.addEntityChange('note_reordering', entityId, 'none', sourceId);
});
return true;
}
function updateOptions(remoteEntity, sourceId) {
function updateOptions(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [remoteEntity.name]);
if (localEntity && !localEntity.isSynced) {
@ -197,7 +197,7 @@ function updateOptions(remoteEntity, sourceId) {
sql.transactional(() => {
sql.replace('options', remoteEntity);
entityChangesService.addOptionEntityChange(remoteEntity.name, sourceId, true);
entityChangesService.addEntityChange('options', remoteEntity.name, hash, sourceId, true);
});
return true;
@ -206,14 +206,14 @@ function updateOptions(remoteEntity, sourceId) {
return false;
}
function updateRecentNotes(remoteEntity, sourceId) {
function updateRecentNotes(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM recent_notes WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace('recent_notes', remoteEntity);
entityChangesService.addRecentNoteEntityChange(remoteEntity.noteId, sourceId);
entityChangesService.addEntityChange('recent_notes', remoteEntity.noteId, hash, sourceId);
});
return true;
@ -222,14 +222,14 @@ function updateRecentNotes(remoteEntity, sourceId) {
return false;
}
function updateAttribute(remoteEntity, sourceId) {
function updateAttribute(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM attributes WHERE attributeId = ?", [remoteEntity.attributeId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace("attributes", remoteEntity);
entityChangesService.addAttributeEntityChange(remoteEntity.attributeId, sourceId);
entityChangesService.addEntityChange('attributes', remoteEntity.attributeId, hash, sourceId);
});
return true;
@ -238,14 +238,14 @@ function updateAttribute(remoteEntity, sourceId) {
return false;
}
function updateApiToken(entity, sourceId) {
function updateApiToken(entity, hash, sourceId) {
const apiTokenId = sql.getRow("SELECT * FROM api_tokens WHERE apiTokenId = ?", [entity.apiTokenId]);
if (!apiTokenId) {
sql.transactional(() => {
sql.replace("api_tokens", entity);
entityChangesService.addApiTokenEntityChange(entity.apiTokenId, sourceId);
entityChangesService.addEntityChange('api_tokens',entity.apiTokenId, hash, sourceId);
});
return true;