diff --git a/db/migrations/0173__move_hash_to_entity_changes.sql b/db/migrations/0173__move_hash_to_entity_changes.sql new file mode 100644 index 000000000..e69de29bb diff --git a/src/entities/attribute.js b/src/entities/attribute.js index b5aecf6e1..9da32179e 100644 --- a/src/entities/attribute.js +++ b/src/entities/attribute.js @@ -109,9 +109,7 @@ class Attribute extends Entity { super.beforeSaving(); - if (this.isChanged) { - this.utcDateModified = dateUtils.utcNowDateTime(); - } + this.utcDateModified = dateUtils.utcNowDateTime(); } createClone(type, name, value, isInheritable) { diff --git a/src/entities/branch.js b/src/entities/branch.js index 5fb832241..a46b75d17 100644 --- a/src/entities/branch.js +++ b/src/entities/branch.js @@ -57,9 +57,7 @@ class Branch extends Entity { super.beforeSaving(); - if (this.isChanged) { - this.utcDateModified = dateUtils.utcNowDateTime(); - } + this.utcDateModified = dateUtils.utcNowDateTime(); } createClone(parentNoteId, notePosition) { diff --git a/src/entities/entity.js b/src/entities/entity.js index b33fd2826..9d648d4ad 100644 --- a/src/entities/entity.js +++ b/src/entities/entity.js @@ -22,11 +22,6 @@ class Entity { beforeSaving() { this.generateIdIfNecessary(); - - const origHash = this.hash; - - this.hash = this.generateHash(); - this.isChanged = origHash !== this.hash; } generateIdIfNecessary() { diff --git a/src/entities/note.js b/src/entities/note.js index 61c9980ab..840c0d7d7 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -143,8 +143,7 @@ class Note extends Entity { noteId: this.noteId, content: content, dateModified: dateUtils.localNowDateTime(), - utcDateModified: dateUtils.utcNowDateTime(), - hash: utils.hash(this.noteId + "|" + content.toString()) + utcDateModified: dateUtils.utcNowDateTime() }; if (this.isProtected) { @@ -158,7 +157,9 @@ class Note extends Entity { sql.upsert("note_contents", "noteId", pojo); - entityChangesService.addNoteContentEntityChange(this.noteId); + const hash = utils.hash(this.noteId + "|" + content.toString()); + + entityChangesService.addEntityChange('note_contents', this.noteId, hash); } setJsonContent(content) { @@ -904,10 +905,8 @@ class Note extends Entity { super.beforeSaving(); - if (this.isChanged) { - this.dateModified = dateUtils.localNowDateTime(); - this.utcDateModified = dateUtils.utcNowDateTime(); - } + this.dateModified = dateUtils.localNowDateTime(); + this.utcDateModified = dateUtils.utcNowDateTime(); } // cannot be static! diff --git a/src/entities/note_revision.js b/src/entities/note_revision.js index 5b1798f56..f322ed140 100644 --- a/src/entities/note_revision.js +++ b/src/entities/note_revision.js @@ -106,8 +106,7 @@ class NoteRevision extends Entity { const pojo = { noteRevisionId: this.noteRevisionId, content: content, - utcDateModified: dateUtils.utcNowDateTime(), - hash: utils.hash(this.noteRevisionId + "|" + content) + utcDateModified: dateUtils.utcNowDateTime() }; if (this.isProtected) { @@ -121,15 +120,15 @@ class NoteRevision extends Entity { sql.upsert("note_revision_contents", "noteRevisionId", pojo); - entityChangesService.addNoteRevisionContentEntityChange(this.noteRevisionId); + const hash = utils.hash(this.noteRevisionId + "|" + content); + + entityChangesService.addEntityChange('note_revision_contents', this.noteRevisionId, hash); } beforeSaving() { super.beforeSaving(); - if (this.isChanged) { - this.utcDateModified = dateUtils.utcNowDateTime(); - } + this.utcDateModified = dateUtils.utcNowDateTime(); } // cannot be static! diff --git a/src/entities/option.js b/src/entities/option.js index 1b64311d3..9802e0f2d 100644 --- a/src/entities/option.js +++ b/src/entities/option.js @@ -32,10 +32,8 @@ class Option extends Entity { super.beforeSaving(); - if (this.isChanged) { - this.utcDateModified = dateUtils.utcNowDateTime(); - } + this.utcDateModified = dateUtils.utcNowDateTime(); } } -module.exports = Option; \ No newline at end of file +module.exports = Option; diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js index 5cbbda945..9efb24cc9 100644 --- a/src/routes/api/sync.js +++ b/src/routes/api/sync.js @@ -86,32 +86,32 @@ function forceNoteSync(req) { const now = dateUtils.utcNowDateTime(); sql.execute(`UPDATE notes SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]); - entityChangesService.addNoteEntityChange(noteId); + entityChangesService.moveEntityChangeToTop('notes', noteId); sql.execute(`UPDATE note_contents SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]); - entityChangesService.addNoteContentEntityChange(noteId); + entityChangesService.moveEntityChangeToTop('note_contents', noteId); for (const branchId of sql.getColumn("SELECT branchId FROM branches WHERE noteId = ?", [noteId])) { sql.execute(`UPDATE branches SET utcDateModified = ? WHERE branchId = ?`, [now, branchId]); - entityChangesService.addBranchEntityChange(branchId); + entityChangesService.moveEntityChangeToTop('branches', branchId); } for (const attributeId of sql.getColumn("SELECT attributeId FROM attributes WHERE noteId = ?", [noteId])) { sql.execute(`UPDATE attributes SET utcDateModified = ? WHERE attributeId = ?`, [now, attributeId]); - entityChangesService.addAttributeEntityChange(attributeId); + entityChangesService.moveEntityChangeToTop('attributes', attributeId); } for (const noteRevisionId of sql.getColumn("SELECT noteRevisionId FROM note_revisions WHERE noteId = ?", [noteId])) { sql.execute(`UPDATE note_revisions SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]); - entityChangesService.addNoteRevisionEntityChange(noteRevisionId); + entityChangesService.moveEntityChangeToTop('note_revisions', noteRevisionId); sql.execute(`UPDATE note_revision_contents SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]); - entityChangesService.addNoteRevisionContentEntityChange(noteRevisionId); + entityChangesService.moveEntityChangeToTop('note_revision_contents', noteRevisionId); } - entityChangesService.addRecentNoteEntityChange(noteId); + entityChangesService.moveEntityChangeToTop('recent_changes', noteId); log.info("Forcing note sync for " + noteId); diff --git a/src/services/app_info.js b/src/services/app_info.js index 2d1bce314..c706af2e6 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -4,8 +4,8 @@ const build = require('./build'); const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); -const APP_DB_VERSION = 172; -const SYNC_VERSION = 16; +const APP_DB_VERSION = 173; +const SYNC_VERSION = 17; const CLIPPER_PROTOCOL_VERSION = "1.0"; module.exports = { diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index 07c6d9679..8ab13bee7 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -297,11 +297,10 @@ class ConsistencyChecks { sql.upsert("note_contents", "noteId", { noteId: noteId, content: null, - hash: "consistency_checks", utcDateModified: dateUtils.utcNowDateTime() }); - entityChangesService.addNoteContentEntityChange(noteId); + entityChangesService.addEntityChange('note_contents', noteId, "consistency_checks"); } else { // empty string might be wrong choice for some note types but it's a best guess @@ -566,7 +565,9 @@ class ConsistencyChecks { entity_changes.id IS NULL AND ` + (entityName === 'options' ? 'options.isSynced = 1' : '1'), ({entityId}) => { if (this.autoFix) { - entityChangesService.addEntityChange(entityName, entityId); + const entity = repository.getEntity(`SELECT * FROM ${entityName} WHERE ${key} = ?`, [entityId]); + + entityChangesService.addEntityChange(entityName, entityId, entity.generateHash()); logFix(`Created missing entity change for entityName=${entityName}, entityId=${entityId}`); } else { diff --git a/src/services/content_hash.js b/src/services/content_hash.js index 47ae6e535..d29847ea0 100644 --- a/src/services/content_hash.js +++ b/src/services/content_hash.js @@ -34,23 +34,33 @@ function getSectorHashes(tableName, primaryKeyName, whereBranch) { function getEntityHashes() { const startTime = new Date(); - const hashes = { - notes: getSectorHashes(Note.entityName, Note.primaryKeyName), - note_contents: getSectorHashes("note_contents", "noteId"), - branches: getSectorHashes(Branch.entityName, Branch.primaryKeyName), - note_revisions: getSectorHashes(NoteRevision.entityName, NoteRevision.primaryKeyName), - note_revision_contents: getSectorHashes("note_revision_contents", "noteRevisionId"), - recent_notes: getSectorHashes(RecentNote.entityName, RecentNote.primaryKeyName), - options: getSectorHashes(Option.entityName, Option.primaryKeyName, "isSynced = 1"), - attributes: getSectorHashes(Attribute.entityName, Attribute.primaryKeyName), - api_tokens: getSectorHashes(ApiToken.entityName, ApiToken.primaryKeyName), - }; + const hashRows = sql.getRows(`SELECT entityName, entityId, hash FROM entity_changes`); + + // sorting is faster in memory + // sorting by entityId is enough, hashes will be segmented by entityName later on anyway + hashRows.sort((a, b) => a.entityId < b.entityId ? -1 : 1); + + const hashMap = {}; + + for (const {entityName, entityId, hash} of hashRows) { + const entityHashMap = hashMap[entityName] = hashMap[entityName] || {}; + + const sector = entityId[0]; + + entityHashMap[sector] = (entityHashMap[sector] || "") + hash + } + + for (const entityHashMap of Object.values(hashMap)) { + for (const key in entityHashMap) { + entityHashMap[key] = utils.hash(entityHashMap[key]); + } + } const elapsedTimeMs = Date.now() - startTime.getTime(); log.info(`Content hash computation took ${elapsedTimeMs}ms`); - return hashes; + return hashMap; } function checkContentHashes(otherHashes) { diff --git a/src/services/entity_changes.js b/src/services/entity_changes.js index ee6ab270c..3c1f63441 100644 --- a/src/services/entity_changes.js +++ b/src/services/entity_changes.js @@ -1,4 +1,5 @@ const sql = require('./sql'); +const repository = require('repository'); const sourceIdService = require('./source_id'); const dateUtils = require('./date_utils'); const log = require('./log'); @@ -6,10 +7,11 @@ const cls = require('./cls'); let maxEntityChangeId = 0; -function insertEntityChange(entityName, entityId, sourceId = null, isSynced = true) { +function insertEntityChange(entityName, entityId, hash, sourceId = null, isSynced = true) { const entityChange = { entityName: entityName, entityId: entityId, + hash: hash, utcChangedDate: dateUtils.utcNowDateTime(), sourceId: sourceId || cls.getSourceId() || sourceIdService.getCurrentSourceId(), isSynced: isSynced ? 1 : 0 @@ -22,12 +24,18 @@ function insertEntityChange(entityName, entityId, sourceId = null, isSynced = tr return entityChange; } -function addEntityChange(entityName, entityId, sourceId, isSynced) { - const sync = insertEntityChange(entityName, entityId, sourceId, isSynced); +function addEntityChange(entityName, entityId, hash, sourceId, isSynced) { + const sync = insertEntityChange(entityName, entityId, hash, sourceId, isSynced); cls.addSyncRow(sync); } +function moveEntityChangeToTop(entityName, entityId) { + const entityChange = sql.getRow(`SELECT * FROM entity_changes WHERE entityName = ? AND entityId = ?`, [entityName, entityId]); + + addEntityChange(entityName, entityId, entityChange.hash, null, entityChange.isSynced); +} + function addEntityChangesForSector(entityName, entityPrimaryKey, sector) { const startTime = Date.now(); @@ -35,15 +43,14 @@ function addEntityChangesForSector(entityName, entityPrimaryKey, sector) { const entityIds = sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]); for (const entityId of entityIds) { - if (entityName === 'options') { - const isSynced = sql.getValue(`SELECT isSynced FROM options WHERE name = ?`, [entityId]); + // retrieving entity one by one to avoid memory issues with note_contents + const entity = repository.getEntity(`SELECT * FROM ${entityName} WHERE ${entityPrimaryKey} = ?`, [entityId]); - if (!isSynced) { - continue; - } + if (entityName === 'options' && !entity.isSynced) { + continue } - insertEntityChange(entityName, entityId, 'content-check', true); + insertEntityChange(entityName, entityId, entity.generateHash(), 'content-check', true); } }); @@ -112,16 +119,8 @@ function fillAllEntityChanges() { } module.exports = { - addNoteEntityChange: (noteId, sourceId) => addEntityChange("notes", noteId, sourceId), - addNoteContentEntityChange: (noteId, sourceId) => addEntityChange("note_contents", noteId, sourceId), - addBranchEntityChange: (branchId, sourceId) => addEntityChange("branches", branchId, sourceId), addNoteReorderingEntityChange: (parentNoteId, sourceId) => addEntityChange("note_reordering", parentNoteId, sourceId), - addNoteRevisionEntityChange: (noteRevisionId, sourceId) => addEntityChange("note_revisions", noteRevisionId, sourceId), - addNoteRevisionContentEntityChange: (noteRevisionId, sourceId) => addEntityChange("note_revision_contents", noteRevisionId, sourceId), - addOptionEntityChange: (name, sourceId, isSynced) => addEntityChange("options", name, sourceId, isSynced), - addRecentNoteEntityChange: (noteId, sourceId) => addEntityChange("recent_notes", noteId, sourceId), - addAttributeEntityChange: (attributeId, sourceId) => addEntityChange("attributes", attributeId, sourceId), - addApiTokenEntityChange: (apiTokenId, sourceId) => addEntityChange("api_tokens", apiTokenId, sourceId), + moveEntityChangeToTop, addEntityChange, fillAllEntityChanges, addEntityChangesForSector, diff --git a/src/services/repository.js b/src/services/repository.js index 93befda04..7c9fc4335 100644 --- a/src/services/repository.js +++ b/src/services/repository.js @@ -101,9 +101,6 @@ function updateEntity(entity) { entity.updatePojo(clone); } - // indicates whether entity actually changed - delete clone.isChanged; - for (const key in clone) { // !isBuffer is for images and attachments if (clone[key] !== null && typeof clone[key] === 'object' && !Buffer.isBuffer(clone[key])) { @@ -116,23 +113,21 @@ function updateEntity(entity) { const primaryKey = entity[primaryKeyName]; - if (entity.isChanged) { - const isSynced = entityName !== 'options' || entity.isSynced; + const isSynced = entityName !== 'options' || entity.isSynced; - entityChangesService.addEntityChange(entityName, primaryKey, null, isSynced); + entityChangesService.addEntityChange(entityName, primaryKey, null, isSynced); - if (!cls.isEntityEventsDisabled()) { - const eventPayload = { - entityName, - entity - }; + if (!cls.isEntityEventsDisabled()) { + const eventPayload = { + entityName, + entity + }; - if (isNewEntity && !entity.isDeleted) { - eventService.emit(eventService.ENTITY_CREATED, eventPayload); - } - - eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload); + if (isNewEntity && !entity.isDeleted) { + eventService.emit(eventService.ENTITY_CREATED, eventPayload); } + + eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload); } }); } diff --git a/src/services/sync_update.js b/src/services/sync_update.js index d00e1c47f..267887353 100644 --- a/src/services/sync_update.js +++ b/src/services/sync_update.js @@ -9,38 +9,38 @@ function updateEntity(entityChange, entity, sourceId) { return false; } - const {entityName} = entityChange; + const {entityName, hash} = entityChange; let updated; if (entityName === 'notes') { - updated = updateNote(entity, sourceId); + updated = updateNote(entity, hash, sourceId); } else if (entityName === 'note_contents') { - updated = updateNoteContent(entity, sourceId); + updated = updateNoteContent(entity, hash, sourceId); } else if (entityName === 'branches') { - updated = updateBranch(entity, sourceId); + updated = updateBranch(entity, hash, sourceId); } else if (entityName === 'note_revisions') { - updated = updateNoteRevision(entity, sourceId); + updated = updateNoteRevision(entity, hash, sourceId); } else if (entityName === 'note_revision_contents') { - updated = updateNoteRevisionContent(entity, sourceId); + updated = updateNoteRevisionContent(entity, hash, sourceId); } else if (entityName === 'note_reordering') { updated = updateNoteReordering(entityChange.entityId, entity, sourceId); } else if (entityName === 'options') { - updated = updateOptions(entity, sourceId); + updated = updateOptions(entity, hash, sourceId); } else if (entityName === 'recent_notes') { - updated = updateRecentNotes(entity, sourceId); + updated = updateRecentNotes(entity, hash, sourceId); } else if (entityName === 'attributes') { - updated = updateAttribute(entity, sourceId); + updated = updateAttribute(entity, hash, sourceId); } else if (entityName === 'api_tokens') { - updated = updateApiToken(entity, sourceId); + updated = updateApiToken(entity, hash, sourceId); } else { throw new Error(`Unrecognized entity type ${entityName}`); @@ -79,14 +79,14 @@ function shouldWeUpdateEntity(localEntity, remoteEntity) { return false; } -function updateNote(remoteEntity, sourceId) { +function updateNote(remoteEntity, hash, sourceId) { const localEntity = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [remoteEntity.noteId]); if (shouldWeUpdateEntity(localEntity, remoteEntity)) { sql.transactional(() => { sql.replace("notes", remoteEntity); - entityChangesService.addNoteEntityChange(remoteEntity.noteId, sourceId); + entityChangesService.addEntityChange('notes', remoteEntity.noteId, hash, sourceId); }); return true; @@ -95,7 +95,7 @@ function updateNote(remoteEntity, sourceId) { return false; } -function updateNoteContent(remoteEntity, sourceId) { +function updateNoteContent(remoteEntity, hash, sourceId) { const localEntity = sql.getRow("SELECT * FROM note_contents WHERE noteId = ?", [remoteEntity.noteId]); if (shouldWeUpdateEntity(localEntity, remoteEntity)) { @@ -111,7 +111,7 @@ function updateNoteContent(remoteEntity, sourceId) { sql.transactional(() => { sql.replace("note_contents", remoteEntity); - entityChangesService.addNoteContentEntityChange(remoteEntity.noteId, sourceId); + entityChangesService.addEntityChange("note_contents", remoteEntity.noteId, hash, sourceId); }); return true; @@ -120,7 +120,7 @@ function updateNoteContent(remoteEntity, sourceId) { return false; } -function updateBranch(remoteEntity, sourceId) { +function updateBranch(remoteEntity, hash, sourceId) { const localEntity = sql.getRowOrNull("SELECT * FROM branches WHERE branchId = ?", [remoteEntity.branchId]); if (shouldWeUpdateEntity(localEntity, remoteEntity)) { @@ -133,7 +133,7 @@ function updateBranch(remoteEntity, sourceId) { sql.replace('branches', remoteEntity); - entityChangesService.addBranchEntityChange(remoteEntity.branchId, sourceId); + entityChangesService.addEntityChange('branches', remoteEntity.branchId, hash, sourceId); }); return true; @@ -142,21 +142,21 @@ function updateBranch(remoteEntity, sourceId) { return false; } -function updateNoteRevision(remoteEntity, sourceId) { +function updateNoteRevision(remoteEntity, hash, sourceId) { const localEntity = sql.getRowOrNull("SELECT * FROM note_revisions WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]); sql.transactional(() => { if (shouldWeUpdateEntity(localEntity, remoteEntity)) { sql.replace('note_revisions', remoteEntity); - entityChangesService.addNoteRevisionEntityChange(remoteEntity.noteRevisionId, sourceId); + entityChangesService.addEntityChange('note_revisions', remoteEntity.noteRevisionId, hash, sourceId); log.info("Update/sync note revision " + remoteEntity.noteRevisionId); } }); } -function updateNoteRevisionContent(remoteEntity, sourceId) { +function updateNoteRevisionContent(remoteEntity, hash, sourceId) { const localEntity = sql.getRowOrNull("SELECT * FROM note_revision_contents WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]); if (shouldWeUpdateEntity(localEntity, remoteEntity)) { @@ -165,7 +165,7 @@ function updateNoteRevisionContent(remoteEntity, sourceId) { sql.replace('note_revision_contents', remoteEntity); - entityChangesService.addNoteRevisionContentEntityChange(remoteEntity.noteRevisionId, sourceId); + entityChangesService.addEntityChange('note_revision_contents', remoteEntity.noteRevisionId, hash, sourceId); }); return true; @@ -180,13 +180,13 @@ function updateNoteReordering(entityId, remote, sourceId) { sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remote[key], key]); } - entityChangesService.addNoteReorderingEntityChange(entityId, sourceId); + entityChangesService.addEntityChange('note_reordering', entityId, 'none', sourceId); }); return true; } -function updateOptions(remoteEntity, sourceId) { +function updateOptions(remoteEntity, hash, sourceId) { const localEntity = sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [remoteEntity.name]); if (localEntity && !localEntity.isSynced) { @@ -197,7 +197,7 @@ function updateOptions(remoteEntity, sourceId) { sql.transactional(() => { sql.replace('options', remoteEntity); - entityChangesService.addOptionEntityChange(remoteEntity.name, sourceId, true); + entityChangesService.addEntityChange('options', remoteEntity.name, hash, sourceId, true); }); return true; @@ -206,14 +206,14 @@ function updateOptions(remoteEntity, sourceId) { return false; } -function updateRecentNotes(remoteEntity, sourceId) { +function updateRecentNotes(remoteEntity, hash, sourceId) { const localEntity = sql.getRowOrNull("SELECT * FROM recent_notes WHERE noteId = ?", [remoteEntity.noteId]); if (shouldWeUpdateEntity(localEntity, remoteEntity)) { sql.transactional(() => { sql.replace('recent_notes', remoteEntity); - entityChangesService.addRecentNoteEntityChange(remoteEntity.noteId, sourceId); + entityChangesService.addEntityChange('recent_notes', remoteEntity.noteId, hash, sourceId); }); return true; @@ -222,14 +222,14 @@ function updateRecentNotes(remoteEntity, sourceId) { return false; } -function updateAttribute(remoteEntity, sourceId) { +function updateAttribute(remoteEntity, hash, sourceId) { const localEntity = sql.getRow("SELECT * FROM attributes WHERE attributeId = ?", [remoteEntity.attributeId]); if (shouldWeUpdateEntity(localEntity, remoteEntity)) { sql.transactional(() => { sql.replace("attributes", remoteEntity); - entityChangesService.addAttributeEntityChange(remoteEntity.attributeId, sourceId); + entityChangesService.addEntityChange('attributes', remoteEntity.attributeId, hash, sourceId); }); return true; @@ -238,14 +238,14 @@ function updateAttribute(remoteEntity, sourceId) { return false; } -function updateApiToken(entity, sourceId) { +function updateApiToken(entity, hash, sourceId) { const apiTokenId = sql.getRow("SELECT * FROM api_tokens WHERE apiTokenId = ?", [entity.apiTokenId]); if (!apiTokenId) { sql.transactional(() => { sql.replace("api_tokens", entity); - entityChangesService.addApiTokenEntityChange(entity.apiTokenId, sourceId); + entityChangesService.addEntityChange('api_tokens',entity.apiTokenId, hash, sourceId); }); return true;