From 20c7c657da74ec7ea9a0b3bfdd3e419913a71da2 Mon Sep 17 00:00:00 2001 From: zadam Date: Mon, 14 Dec 2020 13:15:32 +0100 Subject: [PATCH] moving isErased into entity_changes --- .../0173__move_hash_to_entity_changes.sql | 3 + .../0174__isErased_in_entity_changes.sql | 65 +++++ src/services/consistency_checks.js | 5 +- src/services/entity_changes.js | 10 +- src/services/sync_update.js | 226 ++---------------- 5 files changed, 101 insertions(+), 208 deletions(-) create mode 100644 db/migrations/0174__isErased_in_entity_changes.sql diff --git a/db/migrations/0173__move_hash_to_entity_changes.sql b/db/migrations/0173__move_hash_to_entity_changes.sql index 11c4f1065..32d351cb2 100644 --- a/db/migrations/0173__move_hash_to_entity_changes.sql +++ b/db/migrations/0173__move_hash_to_entity_changes.sql @@ -37,6 +37,9 @@ CREATE TABLE IF NOT EXISTS "mig_api_tokens" INSERT INTO mig_api_tokens (apiTokenId, token, utcDateCreated, isDeleted) SELECT apiTokenId, token, utcDateCreated, isDeleted FROM api_tokens; +DROP TABLE api_tokens; +ALTER TABLE mig_api_tokens RENAME TO api_tokens; + CREATE TABLE IF NOT EXISTS "mig_attributes" ( attributeId TEXT not null primary key, diff --git a/db/migrations/0174__isErased_in_entity_changes.sql b/db/migrations/0174__isErased_in_entity_changes.sql new file mode 100644 index 000000000..be4e8e343 --- /dev/null +++ b/db/migrations/0174__isErased_in_entity_changes.sql @@ -0,0 +1,65 @@ +CREATE TABLE IF NOT EXISTS "mig_entity_changes" ( + `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + `entityName` TEXT NOT NULL, + `entityId` TEXT NOT NULL, + `hash` TEXT NOT NULL, + `sourceId` TEXT NOT NULL, + `isErased` INT NOT NULL, + `utcDateChanged` TEXT NOT NULL, + `isSynced` INTEGER NOT NULL); + +INSERT INTO mig_entity_changes (entityName, entityId, hash, sourceId, isSynced, utcDateChanged, isErased) +SELECT entityName, entityId, '', sourceId, isSynced, 'now', 0 FROM entity_changes; + +UPDATE mig_entity_changes SET isErased = (SELECT isErased FROM notes WHERE noteId = entityId) WHERE entityName = 'notes'; +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM notes WHERE noteId = entityId) WHERE entityName = 'notes'; + +UPDATE mig_entity_changes SET isErased = ( + SELECT isErased + FROM attributes + JOIN notes USING(noteId) + WHERE attributeId = entityId +) WHERE entityName = 'attributes'; +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM attributes WHERE attributeId = entityId) WHERE entityName = 'attributes'; + +UPDATE mig_entity_changes SET isErased = ( + SELECT isErased + FROM branches + JOIN notes USING(noteId) + WHERE branchId = entityId +) WHERE entityName = 'branches'; +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM branches WHERE branchId = entityId) WHERE entityName = 'branches'; + +UPDATE mig_entity_changes SET isErased = ( + SELECT isErased + FROM note_revisions + WHERE noteRevisionId = entityId +) WHERE entityName = 'note_revisions'; +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_revisions WHERE noteRevisionId = entityId) WHERE entityName = 'note_revisions'; + +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateCreated FROM api_tokens WHERE apiTokenId = entityId) WHERE entityName = 'api_tokens'; + +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_contents WHERE noteId = entityId) WHERE entityName = 'note_contents'; + +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_revision_contents WHERE noteRevisionId = entityId) WHERE entityName = 'note_revision_contents'; + +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM options WHERE name = entityId) WHERE entityName = 'options'; + +UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateCreated FROM recent_notes WHERE noteId = entityId) WHERE entityName = 'options'; + +DROP TABLE entity_changes; +ALTER TABLE mig_entity_changes RENAME TO entity_changes; + +CREATE UNIQUE INDEX `IDX_entityChanges_entityName_entityId` ON "entity_changes" ( + `entityName`, + `entityId` + ); + +DELETE FROM attributes WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1); +DELETE FROM branches WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1); +DELETE FROM note_contents WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1); +DELETE FROM note_revision_contents WHERE noteRevisionId IN ( + SELECT noteRevisionId FROM note_revisions WHERE isErased = 1 +); +DELETE FROM note_revisions WHERE isErased = 1; +DELETE FROM notes WHERE isErased = 1; diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index 32b358bff..575cdcccd 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -581,8 +581,9 @@ class ConsistencyChecks { FROM entity_changes LEFT JOIN ${entityName} ON entityId = ${key} - WHERE - entity_changes.entityName = '${entityName}' + WHERE + entity_changes.isErased = 0 + AND entity_changes.entityName = '${entityName}' AND ${key} IS NULL`, ({id, entityId}) => { if (this.autoFix) { diff --git a/src/services/entity_changes.js b/src/services/entity_changes.js index 576a0a40f..9ade3a532 100644 --- a/src/services/entity_changes.js +++ b/src/services/entity_changes.js @@ -22,8 +22,8 @@ function insertEntityChange(entityName, entityId, hash, sourceId = null, isSynce return entityChange; } -function addEntityChange(entityName, entityId, hash, sourceId, isSynced) { - const sync = insertEntityChange(entityName, entityId, hash, sourceId, isSynced); +function addEntityChange(entityChange, sourceId, isSynced) { + const sync = insertEntityChange(entityChange.entityName, entityChange.entityId, entityChange.hash, sourceId, isSynced); cls.addSyncRow(sync); } @@ -59,7 +59,9 @@ function cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey) { sql.execute(` DELETE FROM entity_changes - WHERE sync.entityName = '${entityName}' + WHERE + isErased = 0 + AND sync.entityName = '${entityName}' AND sync.entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`); } @@ -101,7 +103,7 @@ function fillEntityChanges(entityName, entityPrimaryKey, condition = '') { function fillAllEntityChanges() { sql.transactional(() => { - sql.execute("DELETE FROM entity_changes"); + sql.execute("DELETE FROM entity_changes WHERE isErased = 0"); fillEntityChanges("notes", "noteId"); fillEntityChanges("note_contents", "noteId"); diff --git a/src/services/sync_update.js b/src/services/sync_update.js index 267887353..9f5c1bd81 100644 --- a/src/services/sync_update.js +++ b/src/services/sync_update.js @@ -12,38 +12,11 @@ function updateEntity(entityChange, entity, sourceId) { const {entityName, hash} = entityChange; let updated; - if (entityName === 'notes') { - updated = updateNote(entity, hash, sourceId); - } - else if (entityName === 'note_contents') { - updated = updateNoteContent(entity, hash, sourceId); - } - else if (entityName === 'branches') { - updated = updateBranch(entity, hash, sourceId); - } - else if (entityName === 'note_revisions') { - updated = updateNoteRevision(entity, hash, sourceId); - } - else if (entityName === 'note_revision_contents') { - updated = updateNoteRevisionContent(entity, hash, sourceId); - } - else if (entityName === 'note_reordering') { - updated = updateNoteReordering(entityChange.entityId, entity, sourceId); - } - else if (entityName === 'options') { - updated = updateOptions(entity, hash, sourceId); - } - else if (entityName === 'recent_notes') { - updated = updateRecentNotes(entity, hash, sourceId); - } - else if (entityName === 'attributes') { - updated = updateAttribute(entity, hash, sourceId); - } - else if (entityName === 'api_tokens') { - updated = updateApiToken(entity, hash, sourceId); + if (entityName === 'note_reordering') { + updated = updateNoteReordering(entityChange, entity, sourceId); } else { - throw new Error(`Unrecognized entity type ${entityName}`); + updated = updateNormalEntity(entityChange, entity, sourceId); } // currently making exception for protected notes and note revisions because here @@ -59,113 +32,30 @@ function updateEntity(entityChange, entity, sourceId) { return updated; } -function shouldWeUpdateEntity(localEntity, remoteEntity) { - if (!localEntity) { - return true; - } +function updateNormalEntity(entityChange, entity, sourceId) { + const {utcDateChanged, hash} = sql.getRow(` + SELECT utcDateChanged, hash + FROM entity_changes + WHERE entityName = ? AND entityId = ?`, [entityChange.entityName, entityChange.entityId]); - const localDate = localEntity.utcDateModified || localEntity.utcDateCreated; - const remoteDate = remoteEntity.utcDateModified || remoteEntity.utcDateCreated; + if (utcDateChanged < entityChange.utcDateChanged + || hash !== entityChange.hash // sync error, we should still update + ) { + if (['note_contents', 'note_revision_contents'].includes(entityChange.entityName)) { + // we always use Buffer object which is different from normal saving - there we use simple string type for "string notes" + // the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order) + entity.content = entity.content === null ? null : Buffer.from(entity.content, 'base64'); - if (localDate < remoteDate) { - return true; - } - - // this can happen in case of sync error when hashes are different but dates are the same - we should still update - if (localEntity.hash !== remoteEntity.hash && localDate === remoteDate) { - return true; - } - - return false; -} - -function updateNote(remoteEntity, hash, sourceId) { - const localEntity = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [remoteEntity.noteId]); - - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { - sql.transactional(() => { - sql.replace("notes", remoteEntity); - - entityChangesService.addEntityChange('notes', remoteEntity.noteId, hash, sourceId); - }); - - return true; - } - - return false; -} - -function updateNoteContent(remoteEntity, hash, sourceId) { - const localEntity = sql.getRow("SELECT * FROM note_contents WHERE noteId = ?", [remoteEntity.noteId]); - - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { - // we always use Buffer object which is different from normal saving - there we use simple string type for "string notes" - // the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order) - remoteEntity.content = remoteEntity.content === null ? null : Buffer.from(remoteEntity.content, 'base64'); - - if (remoteEntity.content && remoteEntity.content.byteLength === 0) { - // there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency - remoteEntity.content = ""; - } - - sql.transactional(() => { - sql.replace("note_contents", remoteEntity); - - entityChangesService.addEntityChange("note_contents", remoteEntity.noteId, hash, sourceId); - }); - - return true; - } - - return false; -} - -function updateBranch(remoteEntity, hash, sourceId) { - const localEntity = sql.getRowOrNull("SELECT * FROM branches WHERE branchId = ?", [remoteEntity.branchId]); - - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { - sql.transactional(() => { - // isExpanded is not synced unless it's a new branch instance - // otherwise in case of full new sync we'll get all branches (even root) collapsed. - if (localEntity) { - delete remoteEntity.isExpanded; + if (entity.content && entity.content.byteLength === 0) { + // there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency + entity.content = ""; } - - sql.replace('branches', remoteEntity); - - entityChangesService.addEntityChange('branches', remoteEntity.branchId, hash, sourceId); - }); - - return true; - } - - return false; -} - -function updateNoteRevision(remoteEntity, hash, sourceId) { - const localEntity = sql.getRowOrNull("SELECT * FROM note_revisions WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]); - - sql.transactional(() => { - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { - sql.replace('note_revisions', remoteEntity); - - entityChangesService.addEntityChange('note_revisions', remoteEntity.noteRevisionId, hash, sourceId); - - log.info("Update/sync note revision " + remoteEntity.noteRevisionId); } - }); -} -function updateNoteRevisionContent(remoteEntity, hash, sourceId) { - const localEntity = sql.getRowOrNull("SELECT * FROM note_revision_contents WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]); - - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { sql.transactional(() => { - remoteEntity.content = remoteEntity.content === null ? null : Buffer.from(remoteEntity.content, 'base64'); + sql.replace(entityChange.entityName, entity); - sql.replace('note_revision_contents', remoteEntity); - - entityChangesService.addEntityChange('note_revision_contents', remoteEntity.noteRevisionId, hash, sourceId); + entityChangesService.addEntityChange(entityChange, sourceId); }); return true; @@ -174,86 +64,18 @@ function updateNoteRevisionContent(remoteEntity, hash, sourceId) { return false; } -function updateNoteReordering(entityId, remote, sourceId) { +function updateNoteReordering(entityChange, entity, sourceId) { sql.transactional(() => { - for (const key in remote) { - sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remote[key], key]); + for (const key in entity) { + sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]); } - entityChangesService.addEntityChange('note_reordering', entityId, 'none', sourceId); + entityChangesService.addEntityChange(entityChange, sourceId); }); return true; } -function updateOptions(remoteEntity, hash, sourceId) { - const localEntity = sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [remoteEntity.name]); - - if (localEntity && !localEntity.isSynced) { - return; - } - - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { - sql.transactional(() => { - sql.replace('options', remoteEntity); - - entityChangesService.addEntityChange('options', remoteEntity.name, hash, sourceId, true); - }); - - return true; - } - - return false; -} - -function updateRecentNotes(remoteEntity, hash, sourceId) { - const localEntity = sql.getRowOrNull("SELECT * FROM recent_notes WHERE noteId = ?", [remoteEntity.noteId]); - - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { - sql.transactional(() => { - sql.replace('recent_notes', remoteEntity); - - entityChangesService.addEntityChange('recent_notes', remoteEntity.noteId, hash, sourceId); - }); - - return true; - } - - return false; -} - -function updateAttribute(remoteEntity, hash, sourceId) { - const localEntity = sql.getRow("SELECT * FROM attributes WHERE attributeId = ?", [remoteEntity.attributeId]); - - if (shouldWeUpdateEntity(localEntity, remoteEntity)) { - sql.transactional(() => { - sql.replace("attributes", remoteEntity); - - entityChangesService.addEntityChange('attributes', remoteEntity.attributeId, hash, sourceId); - }); - - return true; - } - - return false; -} - -function updateApiToken(entity, hash, sourceId) { - const apiTokenId = sql.getRow("SELECT * FROM api_tokens WHERE apiTokenId = ?", [entity.apiTokenId]); - - if (!apiTokenId) { - sql.transactional(() => { - sql.replace("api_tokens", entity); - - entityChangesService.addEntityChange('api_tokens',entity.apiTokenId, hash, sourceId); - }); - - return true; - } - - return false; -} - module.exports = { updateEntity };