gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-30 19:19:03 +01:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

moving isErased into entity_changes

Browse Source
This commit is contained in:
zadam 2020-12-14 13:15:32 +01:00
parent 0a896cc19c
commit 20c7c657da
5 changed files with 101 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
db/migrations/0173__move_hash_to_entity_changes.sql
View File

@ -37,6 +37,9 @@ CREATE TABLE IF NOT EXISTS "mig_api_tokens"
INSERT INTO mig_api_tokens (apiTokenId, token, utcDateCreated, isDeleted) INSERT INTO mig_api_tokens (apiTokenId, token, utcDateCreated, isDeleted)
SELECT apiTokenId, token, utcDateCreated, isDeleted FROM api_tokens; SELECT apiTokenId, token, utcDateCreated, isDeleted FROM api_tokens;
DROP TABLE api_tokens;
ALTER TABLE mig_api_tokens RENAME TO api_tokens;
CREATE TABLE IF NOT EXISTS "mig_attributes" CREATE TABLE IF NOT EXISTS "mig_attributes"
( (
attributeId TEXT not null primary key, attributeId TEXT not null primary key,

65
db/migrations/0174__isErased_in_entity_changes.sql Normal file
View File

@ -0,0 +1,65 @@
CREATE TABLE IF NOT EXISTS "mig_entity_changes" (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`entityName` TEXT NOT NULL,
`entityId` TEXT NOT NULL,
`hash` TEXT NOT NULL,
`sourceId` TEXT NOT NULL,
`isErased` INT NOT NULL,
`utcDateChanged` TEXT NOT NULL,
`isSynced` INTEGER NOT NULL);
INSERT INTO mig_entity_changes (entityName, entityId, hash, sourceId, isSynced, utcDateChanged, isErased)
SELECT entityName, entityId, '', sourceId, isSynced, 'now', 0 FROM entity_changes;
UPDATE mig_entity_changes SET isErased = (SELECT isErased FROM notes WHERE noteId = entityId) WHERE entityName = 'notes';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM notes WHERE noteId = entityId) WHERE entityName = 'notes';
UPDATE mig_entity_changes SET isErased = (
SELECT isErased
FROM attributes
JOIN notes USING(noteId)
WHERE attributeId = entityId
) WHERE entityName = 'attributes';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM attributes WHERE attributeId = entityId) WHERE entityName = 'attributes';
UPDATE mig_entity_changes SET isErased = (
SELECT isErased
FROM branches
JOIN notes USING(noteId)
WHERE branchId = entityId
) WHERE entityName = 'branches';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM branches WHERE branchId = entityId) WHERE entityName = 'branches';
UPDATE mig_entity_changes SET isErased = (
SELECT isErased
FROM note_revisions
WHERE noteRevisionId = entityId
) WHERE entityName = 'note_revisions';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_revisions WHERE noteRevisionId = entityId) WHERE entityName = 'note_revisions';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateCreated FROM api_tokens WHERE apiTokenId = entityId) WHERE entityName = 'api_tokens';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_contents WHERE noteId = entityId) WHERE entityName = 'note_contents';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM note_revision_contents WHERE noteRevisionId = entityId) WHERE entityName = 'note_revision_contents';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateModified FROM options WHERE name = entityId) WHERE entityName = 'options';
UPDATE mig_entity_changes SET utcDateChanged = (SELECT utcDateCreated FROM recent_notes WHERE noteId = entityId) WHERE entityName = 'options';
DROP TABLE entity_changes;
ALTER TABLE mig_entity_changes RENAME TO entity_changes;
CREATE UNIQUE INDEX `IDX_entityChanges_entityName_entityId` ON "entity_changes" (
`entityName`,
`entityId`
);
DELETE FROM attributes WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1);
DELETE FROM branches WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1);
DELETE FROM note_contents WHERE noteId IN (SELECT noteId FROM notes WHERE isErased = 1);
DELETE FROM note_revision_contents WHERE noteRevisionId IN (
SELECT noteRevisionId FROM note_revisions WHERE isErased = 1
);
DELETE FROM note_revisions WHERE isErased = 1;
DELETE FROM notes WHERE isErased = 1;

5
src/services/consistency_checks.js
View File

@ -581,8 +581,9 @@ class ConsistencyChecks {
FROM FROM
entity_changes entity_changes
LEFT JOIN ${entityName} ON entityId = ${key} LEFT JOIN ${entityName} ON entityId = ${key}
WHERE WHERE
entity_changes.entityName = '${entityName}' entity_changes.isErased = 0
AND entity_changes.entityName = '${entityName}'
AND ${key} IS NULL`, AND ${key} IS NULL`,
({id, entityId}) => { ({id, entityId}) => {
if (this.autoFix) { if (this.autoFix) {

10
src/services/entity_changes.js
View File

@ -22,8 +22,8 @@ function insertEntityChange(entityName, entityId, hash, sourceId = null, isSynce
return entityChange; return entityChange;
} }
function addEntityChange(entityName, entityId, hash, sourceId, isSynced) { function addEntityChange(entityChange, sourceId, isSynced) {
const sync = insertEntityChange(entityName, entityId, hash, sourceId, isSynced); const sync = insertEntityChange(entityChange.entityName, entityChange.entityId, entityChange.hash, sourceId, isSynced);
cls.addSyncRow(sync); cls.addSyncRow(sync);
} }
@ -59,7 +59,9 @@ function cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey) {
sql.execute(` sql.execute(`
DELETE DELETE
FROM entity_changes FROM entity_changes
WHERE sync.entityName = '${entityName}' WHERE
isErased = 0
AND sync.entityName = '${entityName}'
AND sync.entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`); AND sync.entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`);
} }
@ -101,7 +103,7 @@ function fillEntityChanges(entityName, entityPrimaryKey, condition = '') {
function fillAllEntityChanges() { function fillAllEntityChanges() {
sql.transactional(() => { sql.transactional(() => {
sql.execute("DELETE FROM entity_changes"); sql.execute("DELETE FROM entity_changes WHERE isErased = 0");
fillEntityChanges("notes", "noteId"); fillEntityChanges("notes", "noteId");
fillEntityChanges("note_contents", "noteId"); fillEntityChanges("note_contents", "noteId");

226
src/services/sync_update.js
View File

@ -12,38 +12,11 @@ function updateEntity(entityChange, entity, sourceId) {
const {entityName, hash} = entityChange; const {entityName, hash} = entityChange;
let updated; let updated;
if (entityName === 'notes') { if (entityName === 'note_reordering') {
updated = updateNote(entity, hash, sourceId); updated = updateNoteReordering(entityChange, entity, sourceId);
}
else if (entityName === 'note_contents') {
updated = updateNoteContent(entity, hash, sourceId);
}
else if (entityName === 'branches') {
updated = updateBranch(entity, hash, sourceId);
}
else if (entityName === 'note_revisions') {
updated = updateNoteRevision(entity, hash, sourceId);
}
else if (entityName === 'note_revision_contents') {
updated = updateNoteRevisionContent(entity, hash, sourceId);
}
else if (entityName === 'note_reordering') {
updated = updateNoteReordering(entityChange.entityId, entity, sourceId);
}
else if (entityName === 'options') {
updated = updateOptions(entity, hash, sourceId);
}
else if (entityName === 'recent_notes') {
updated = updateRecentNotes(entity, hash, sourceId);
}
else if (entityName === 'attributes') {
updated = updateAttribute(entity, hash, sourceId);
}
else if (entityName === 'api_tokens') {
updated = updateApiToken(entity, hash, sourceId);
} }
else { else {
throw new Error(`Unrecognized entity type ${entityName}`); updated = updateNormalEntity(entityChange, entity, sourceId);
} }
// currently making exception for protected notes and note revisions because here // currently making exception for protected notes and note revisions because here
@ -59,113 +32,30 @@ function updateEntity(entityChange, entity, sourceId) {
return updated; return updated;
} }
function shouldWeUpdateEntity(localEntity, remoteEntity) { function updateNormalEntity(entityChange, entity, sourceId) {
if (!localEntity) { const {utcDateChanged, hash} = sql.getRow(`
return true; SELECT utcDateChanged, hash
} FROM entity_changes
WHERE entityName = ? AND entityId = ?`, [entityChange.entityName, entityChange.entityId]);
const localDate = localEntity.utcDateModified || localEntity.utcDateCreated; if (utcDateChanged < entityChange.utcDateChanged
const remoteDate = remoteEntity.utcDateModified || remoteEntity.utcDateCreated; || hash !== entityChange.hash // sync error, we should still update
) {
if (['note_contents', 'note_revision_contents'].includes(entityChange.entityName)) {
// we always use Buffer object which is different from normal saving - there we use simple string type for "string notes"
// the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order)
entity.content = entity.content === null ? null : Buffer.from(entity.content, 'base64');
if (localDate < remoteDate) { if (entity.content && entity.content.byteLength === 0) {
return true; // there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
} entity.content = "";
// this can happen in case of sync error when hashes are different but dates are the same - we should still update
if (localEntity.hash !== remoteEntity.hash && localDate === remoteDate) {
return true;
}
return false;
}
function updateNote(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace("notes", remoteEntity);
entityChangesService.addEntityChange('notes', remoteEntity.noteId, hash, sourceId);
});
return true;
}
return false;
}
function updateNoteContent(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM note_contents WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
// we always use Buffer object which is different from normal saving - there we use simple string type for "string notes"
// the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order)
remoteEntity.content = remoteEntity.content === null ? null : Buffer.from(remoteEntity.content, 'base64');
if (remoteEntity.content && remoteEntity.content.byteLength === 0) {
// there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
remoteEntity.content = "";
}
sql.transactional(() => {
sql.replace("note_contents", remoteEntity);
entityChangesService.addEntityChange("note_contents", remoteEntity.noteId, hash, sourceId);
});
return true;
}
return false;
}
function updateBranch(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM branches WHERE branchId = ?", [remoteEntity.branchId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
// isExpanded is not synced unless it's a new branch instance
// otherwise in case of full new sync we'll get all branches (even root) collapsed.
if (localEntity) {
delete remoteEntity.isExpanded;
} }
sql.replace('branches', remoteEntity);
entityChangesService.addEntityChange('branches', remoteEntity.branchId, hash, sourceId);
});
return true;
}
return false;
}
function updateNoteRevision(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM note_revisions WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
sql.transactional(() => {
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.replace('note_revisions', remoteEntity);
entityChangesService.addEntityChange('note_revisions', remoteEntity.noteRevisionId, hash, sourceId);
log.info("Update/sync note revision " + remoteEntity.noteRevisionId);
} }
});
}
function updateNoteRevisionContent(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM note_revision_contents WHERE noteRevisionId = ?", [remoteEntity.noteRevisionId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => { sql.transactional(() => {
remoteEntity.content = remoteEntity.content === null ? null : Buffer.from(remoteEntity.content, 'base64'); sql.replace(entityChange.entityName, entity);
sql.replace('note_revision_contents', remoteEntity); entityChangesService.addEntityChange(entityChange, sourceId);
entityChangesService.addEntityChange('note_revision_contents', remoteEntity.noteRevisionId, hash, sourceId);
}); });
return true; return true;
@ -174,86 +64,18 @@ function updateNoteRevisionContent(remoteEntity, hash, sourceId) {
return false; return false;
} }
function updateNoteReordering(entityId, remote, sourceId) { function updateNoteReordering(entityChange, entity, sourceId) {
sql.transactional(() => { sql.transactional(() => {
for (const key in remote) { for (const key in entity) {
sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remote[key], key]); sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]);
} }
entityChangesService.addEntityChange('note_reordering', entityId, 'none', sourceId); entityChangesService.addEntityChange(entityChange, sourceId);
}); });
return true; return true;
} }
function updateOptions(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM options WHERE name = ?", [remoteEntity.name]);
if (localEntity && !localEntity.isSynced) {
return;
}
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace('options', remoteEntity);
entityChangesService.addEntityChange('options', remoteEntity.name, hash, sourceId, true);
});
return true;
}
return false;
}
function updateRecentNotes(remoteEntity, hash, sourceId) {
const localEntity = sql.getRowOrNull("SELECT * FROM recent_notes WHERE noteId = ?", [remoteEntity.noteId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace('recent_notes', remoteEntity);
entityChangesService.addEntityChange('recent_notes', remoteEntity.noteId, hash, sourceId);
});
return true;
}
return false;
}
function updateAttribute(remoteEntity, hash, sourceId) {
const localEntity = sql.getRow("SELECT * FROM attributes WHERE attributeId = ?", [remoteEntity.attributeId]);
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.transactional(() => {
sql.replace("attributes", remoteEntity);
entityChangesService.addEntityChange('attributes', remoteEntity.attributeId, hash, sourceId);
});
return true;
}
return false;
}
function updateApiToken(entity, hash, sourceId) {
const apiTokenId = sql.getRow("SELECT * FROM api_tokens WHERE apiTokenId = ?", [entity.apiTokenId]);
if (!apiTokenId) {
sql.transactional(() => {
sql.replace("api_tokens", entity);
entityChangesService.addEntityChange('api_tokens',entity.apiTokenId, hash, sourceId);
});
return true;
}
return false;
}
module.exports = { module.exports = {
updateEntity updateEntity
}; };