gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

erasing rows of deleted entities

Browse Source
This commit is contained in:
zadam 2020-12-14 13:47:33 +01:00
parent 20c7c657da
commit 248fa780e8
7 changed files with 103 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/migrations/0175__rename_eraseNotesAfterTimeInSeconds.sql Normal file
View File

@ -0,0 +1 @@
UPDATE options SET name = 'eraseNotesAfterTimeInSeconds' WHERE name = 'eraseNotesAfterTimeInSeconds';

16
src/public/app/dialogs/options/other.js
View File

@ -42,14 +42,14 @@ const TPL = `
<div>
<h4>Note erasure timeout</h4>
<p>Deleted notes are at first only marked as deleted and it is possible to recover them
<p>Deleted notes (and attributes, revisions...) are at first only marked as deleted and it is possible to recover them
from Recent Notes dialog. After a period of time, deleted notes are "erased" which means
their content is not recoverable anymore. This setting allows you to configure the length
of the period between deleting and erasing the note.</p>
<div class="form-group">
<label for="erase-notes-after-time-in-seconds">Erase notes after X seconds</label>
<input class="form-control" id="erase-notes-after-time-in-seconds" type="number" min="0">
<label for="erase-entities-after-time-in-seconds">Erase notes after X seconds</label>
<input class="form-control" id="erase-entities-after-time-in-seconds" type="number" min="0">
</div>
<p>You can also trigger erasing manually:</p>
@ -111,12 +111,12 @@ export default class ProtectedSessionOptions {
this.$availableLanguageCodes.text(webContents.session.availableSpellCheckerLanguages.join(', '));
}
this.$eraseNotesAfterTimeInSeconds = $("#erase-notes-after-time-in-seconds");
this.$eraseEntitiesAfterTimeInSeconds = $("#erase-entities-after-time-in-seconds");
this.$eraseNotesAfterTimeInSeconds.on('change', () => {
const eraseNotesAfterTimeInSeconds = this.$eraseNotesAfterTimeInSeconds.val();
this.$eraseEntitiesAfterTimeInSeconds.on('change', () => {
const eraseEntitiesAfterTimeInSeconds = this.$eraseEntitiesAfterTimeInSeconds.val();
server.put('options', { 'eraseNotesAfterTimeInSeconds': eraseNotesAfterTimeInSeconds }).then(() => {
server.put('options', { 'eraseEntitiesAfterTimeInSeconds': eraseEntitiesAfterTimeInSeconds }).then(() => {
toastService.showMessage("Options change have been saved.");
});
@ -173,7 +173,7 @@ export default class ProtectedSessionOptions {
this.$spellCheckEnabled.prop("checked", options['spellCheckEnabled'] === 'true');
this.$spellCheckLanguageCode.val(options['spellCheckLanguageCode']);
this.$eraseNotesAfterTimeInSeconds.val(options['eraseNotesAfterTimeInSeconds']);
this.$eraseEntitiesAfterTimeInSeconds.val(options['eraseEntitiesAfterTimeInSeconds']);
this.$protectedSessionTimeout.val(options['protectedSessionTimeout']);
this.$noteRevisionsTimeInterval.val(options['noteRevisionSnapshotTimeInterval']);

2
src/routes/api/options.js
View File

@ -7,7 +7,7 @@ const attributes = require('../../services/attributes');
// options allowed to be updated directly in options dialog
const ALLOWED_OPTIONS = new Set([
'username', // not exposed for update (not harmful anyway), needed for reading
'eraseNotesAfterTimeInSeconds',
'eraseEntitiesAfterTimeInSeconds',
'protectedSessionTimeout',
'noteRevisionSnapshotTimeInterval',
'zoomFactor',

2
src/services/app_info.js
View File

@ -4,7 +4,7 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 173;
const APP_DB_VERSION = 175;
const SYNC_VERSION = 17;
const CLIPPER_PROTOCOL_VERSION = "1.0";

115
src/services/notes.js
View File

@ -673,61 +673,90 @@ function scanForLinks(note) {
}
}
function eraseDeletedNotes(eraseNotesAfterTimeInSeconds = null) {
if (eraseNotesAfterTimeInSeconds === null) {
eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
}
const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000);
const noteIdsToErase = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND isErased = 0 AND notes.utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
function eraseNotes(noteIdsToErase) {
if (noteIdsToErase.length === 0) {
return;
}
// it's better to not use repository for this because:
// - it would complain about saving protected notes out of protected session
// - we don't want these changes to be synced (since they are done on all instances anyway)
// - we don't want change the hash since this erasing happens on each instance separately
// and changing the hash would fire up the sync errors temporarily
sql.executeMany(`DELETE FROM notes WHERE noteId IN (???)`, noteIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'notes' AND entityId IN (???)`, noteIdsToErase);
sql.executeMany(`
UPDATE notes
SET title = '[erased]',
isProtected = 0,
isErased = 1
WHERE noteId IN (???)`, noteIdsToErase);
sql.executeMany(`DELETE FROM note_contents WHERE noteId IN (???)`, noteIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_contents' AND entityId IN (???)`, noteIdsToErase);
sql.executeMany(`
UPDATE note_contents
SET content = NULL
WHERE noteId IN (???)`, noteIdsToErase);
// we also need to erase all "dependent" entities of the erased notes
const branchIdsToErase = sql.getManyRows(`SELECT branchId FROM branches WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.branchId);
// deleting first contents since the WHERE relies on isErased = 0
sql.executeMany(`
UPDATE note_revision_contents
SET content = NULL
WHERE noteRevisionId IN
(SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN (???))`, noteIdsToErase);
eraseBranches(branchIdsToErase);
sql.executeMany(`
UPDATE note_revisions
SET isErased = 1,
title = NULL
WHERE isErased = 0 AND noteId IN (???)`, noteIdsToErase);
const attributeIdsToErase = sql.getManyRows(`SELECT attributeId FROM attributes WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.attributeId);
sql.executeMany(`
UPDATE attributes
SET name = 'deleted',
value = ''
WHERE noteId IN (???)`, noteIdsToErase);
eraseAttributes(attributeIdsToErase);
const noteRevisionIdsToErase = sql.getManyRows(`SELECT noteRevisionId FROM note_revisions WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.noteRevisionId);
eraseNoteRevisions(noteRevisionIdsToErase);
log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
}
function eraseBranches(branchIdsToErase) {
if (branchIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM branches WHERE branchId IN (???)`, branchIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'branches' AND entityId IN (???)`, branchIdsToErase);
}
function eraseAttributes(attributeIdsToErase) {
if (attributeIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM attributes WHERE attributeId IN (???)`, attributeIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'attributes' AND entityId IN (???)`, attributeIdsToErase);
}
function eraseNoteRevisions(noteRevisionIdsToErase) {
if (noteRevisionIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM note_revisions WHERE noteRevisionId IN (???)`, noteRevisionIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_revisions' AND entityId IN (???)`, noteRevisionIdsToErase);
sql.executeMany(`DELETE FROM note_revision_contents WHERE noteRevisionId IN (???)`, noteRevisionIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_revision_contents' AND entityId IN (???)`, noteRevisionIdsToErase);
}
function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
if (eraseEntitiesAfterTimeInSeconds === null) {
eraseEntitiesAfterTimeInSeconds = optionService.getOptionInt('eraseEntitiesAfterTimeInSeconds');
}
const cutoffDate = new Date(Date.now() - eraseEntitiesAfterTimeInSeconds * 1000);
const noteIdsToErase = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
eraseNotes(noteIdsToErase);
const branchIdsToErase = sql.getColumn("SELECT branchId FROM branches WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
eraseBranches(branchIdsToErase);
const attributeIdsToErase = sql.getColumn("SELECT attributeId FROM attributes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
eraseAttributes(attributeIdsToErase);
}
function eraseDeletedNotesNow() {
eraseDeletedNotes(0);
eraseDeletedEntities(0);
}
// do a replace in str - all keys should be replaced by the corresponding values
@ -836,9 +865,9 @@ function getNoteIdMapping(origNote) {
sqlInit.dbReady.then(() => {
// first cleanup kickoff 5 minutes after startup
setTimeout(cls.wrap(() => eraseDeletedNotes()), 5 * 60 * 1000);
setTimeout(cls.wrap(() => eraseDeletedEntities()), 5 * 60 * 1000);
setInterval(cls.wrap(() => eraseDeletedNotes()), 4 * 3600 * 1000);
setInterval(cls.wrap(() => eraseDeletedEntities()), 4 * 3600 * 1000);
});
module.exports = {

2
src/services/options_init.js
View File

@ -82,7 +82,7 @@ const defaultOptions = [
{ name: 'rightPaneWidth', value: '25', isSynced: false },
{ name: 'rightPaneVisible', value: 'true', isSynced: false },
{ name: 'nativeTitleBarVisible', value: 'false', isSynced: false },
{ name: 'eraseNotesAfterTimeInSeconds', value: '604800', isSynced: true }, // default is 7 days
{ name: 'eraseEntitiesAfterTimeInSeconds', value: '604800', isSynced: true }, // default is 7 days
{ name: 'hideArchivedNotes_main', value: 'false', isSynced: false },
{ name: 'hideIncludedImages_main', value: 'true', isSynced: false },
{ name: 'attributeListExpanded', value: 'false', isSynced: false },

40
src/services/sync_update.js
View File

@ -1,5 +1,4 @@
const sql = require('./sql');
const log = require('./log');
const entityChangesService = require('./entity_changes.js');
const eventService = require('./events');
@ -9,22 +8,15 @@ function updateEntity(entityChange, entity, sourceId) {
return false;
}
const {entityName, hash} = entityChange;
let updated;
if (entityName === 'note_reordering') {
updated = updateNoteReordering(entityChange, entity, sourceId);
}
else {
updated = updateNormalEntity(entityChange, entity, sourceId);
}
const updated = entityChange.entityName === 'note_reordering'
? updateNoteReordering(entityChange, entity, sourceId)
: updateNormalEntity(entityChange, entity, sourceId);
// currently making exception for protected notes and note revisions because here
// the title and content are not available decrypted as listeners would expect
if (updated &&
(!['notes', 'note_contents', 'note_revisions', 'note_revision_contents'].includes(entityName) || !entity.isProtected)) {
if (updated && !entity.isProtected) {
eventService.emit(eventService.ENTITY_SYNCED, {
entityName,
entityName: entityChange.entityName,
entity
});
}
@ -42,14 +34,7 @@ function updateNormalEntity(entityChange, entity, sourceId) {
|| hash !== entityChange.hash // sync error, we should still update
) {
if (['note_contents', 'note_revision_contents'].includes(entityChange.entityName)) {
// we always use Buffer object which is different from normal saving - there we use simple string type for "string notes"
// the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order)
entity.content = entity.content === null ? null : Buffer.from(entity.content, 'base64');
if (entity.content && entity.content.byteLength === 0) {
// there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
entity.content = "";
}
entity.content = handleContent(entity.content);
}
sql.transactional(() => {
@ -76,6 +61,19 @@ function updateNoteReordering(entityChange, entity, sourceId) {
return true;
}
function handleContent(content) {
// we always use Buffer object which is different from normal saving - there we use simple string type for "string notes"
// the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order)
content = content === null ? null : Buffer.from(content, 'base64');
if (content && content.byteLength === 0) {
// there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
content = "";
}
return content;
}
module.exports = {
updateEntity
};