gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

scheduled erasure of attachments WIP

Browse Source
This commit is contained in:
zadam 2023-04-21 00:19:17 +02:00
parent e71b0d82a1
commit 5e2efca933
11 changed files with 505 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
db/migrations/0217__attachments.sql
View File

@ -10,9 +10,12 @@ CREATE TABLE IF NOT EXISTS "attachments"
blobId TEXT DEFAULT null,
dateModified TEXT NOT NULL,
utcDateModified TEXT not null,
utcDateScheduledForDeletionSince TEXT DEFAULT NULL,
utcDateScheduledForErasureSince TEXT DEFAULT NULL,
isDeleted INT not null,
deleteId TEXT DEFAULT NULL);
CREATE INDEX IDX_attachments_parentId_role
on attachments (parentId, role);
CREATE INDEX IDX_attachments_utcDateScheduledForErasureSince
on attachments (utcDateScheduledForErasureSince);

2
db/schema.sql
View File

@ -121,7 +121,7 @@ CREATE TABLE IF NOT EXISTS "attachments"
blobId TEXT DEFAULT null,
dateModified TEXT NOT NULL,
utcDateModified TEXT not null,
utcDateScheduledForDeletionSince TEXT DEFAULT NULL,
utcDateScheduledForErasureSince TEXT DEFAULT NULL,
isDeleted INT not null,
deleteId TEXT DEFAULT NULL);
CREATE INDEX IDX_attachments_parentId_role

662
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

14
package.json
View File

@ -33,10 +33,10 @@
"dependencies": {
"@braintree/sanitize-url": "6.0.2",
"@electron/remote": "2.0.9",
"@excalidraw/excalidraw": "0.14.2",
"@excalidraw/excalidraw": "0.15.2",
"archiver": "5.3.1",
"async-mutex": "0.4.0",
"axios": "1.3.5",
"axios": "1.3.6",
"better-sqlite3": "7.4.5",
"chokidar": "3.5.3",
"cls-hooked": "4.2.2",
@ -51,13 +51,13 @@
"electron-debug": "3.2.0",
"electron-dl": "3.5.0",
"electron-window-state": "5.0.3",
"escape-html": "^1.0.3",
"escape-html": "1.0.3",
"express": "4.18.2",
"express-partial-content": "1.0.2",
"express-rate-limit": "6.7.0",
"express-session": "1.17.3",
"fs-extra": "11.1.1",
"helmet": "6.1.2",
"helmet": "6.1.5",
"html": "1.0.0",
"html2plaintext": "2.1.4",
"http-proxy-agent": "5.0.0",
@ -71,7 +71,7 @@
"jsdom": "21.1.1",
"mime-types": "2.1.35",
"multer": "1.4.5-lts.1",
"node-abi": "3.35.0",
"node-abi": "3.40.0",
"normalize-strings": "1.1.1",
"open": "8.4.1",
"rand-token": "1.0.1",
@ -83,7 +83,7 @@
"sanitize-filename": "1.6.3",
"sanitize-html": "2.10.0",
"sax": "1.2.4",
"semver": "7.3.8",
"semver": "7.5.0",
"serve-favicon": "2.5.0",
"session-file-store": "1.5.0",
"stream-throttle": "0.1.3",
@ -117,7 +117,7 @@
"prettier": "2.8.7",
"nodemon": "^2.0.22",
"rcedit": "3.0.1",
"webpack": "5.78.0",
"webpack": "5.80.0",
"webpack-cli": "5.0.1"
},
"optionalDependencies": {

8
src/becca/entities/battachment.js
View File

@ -20,7 +20,7 @@ class BAttachment extends AbstractBeccaEntity {
static get entityName() { return "attachments"; }
static get primaryKeyName() { return "attachmentId"; }
static get hashedProperties() { return ["attachmentId", "parentId", "role", "mime", "title", "blobId",
"utcDateScheduledForDeletionSince", "utcDateModified"]; }
"utcDateScheduledForErasureSince", "utcDateModified"]; }
constructor(row) {
super();
@ -56,7 +56,7 @@ class BAttachment extends AbstractBeccaEntity {
/** @type {string} */
this.utcDateModified = row.utcDateModified;
/** @type {string} */
this.utcDateScheduledForDeletionSince = row.utcDateScheduledForDeletionSince;
this.utcDateScheduledForErasureSince = row.utcDateScheduledForErasureSince;
}
/** @returns {BAttachment} */
@ -68,7 +68,7 @@ class BAttachment extends AbstractBeccaEntity {
title: this.title,
blobId: this.blobId,
isProtected: this.isProtected,
utcDateScheduledForDeletionSince: this.utcDateScheduledForDeletionSince
utcDateScheduledForErasureSince: this.utcDateScheduledForErasureSince
});
}
@ -171,7 +171,7 @@ class BAttachment extends AbstractBeccaEntity {
isDeleted: false,
dateModified: this.dateModified,
utcDateModified: this.utcDateModified,
utcDateScheduledForDeletionSince: this.utcDateScheduledForDeletionSince
utcDateScheduledForErasureSince: this.utcDateScheduledForErasureSince
};
}

2
src/public/app/entities/fattachment.js
View File

@ -21,7 +21,7 @@ class FAttachment {
/** @type {string} */
this.utcDateModified = row.utcDateModified;
/** @type {string} */
this.utcDateScheduledForDeletionSince = row.utcDateScheduledForDeletionSince;
this.utcDateScheduledForErasureSince = row.utcDateScheduledForErasureSince;
this.froca.attachments[this.attachmentId] = this;
}

34
src/public/app/services/utils.js
View File

@ -27,6 +27,39 @@ function formatTimeWithSeconds(date) {
return `${padNum(date.getHours())}:${padNum(date.getMinutes())}:${padNum(date.getSeconds())}`;
}
function formatTimeInterval(ms) {
const seconds = Math.round(ms / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
const days = Math.floor(hours / 24);
const plural = (count, name) => `${count} ${name}${count > 1 ? 's' : ''}`;
const segments = [];
if (days > 0) {
segments.push(plural(days, 'day'));
}
if (days < 2) {
if (hours % 24 > 0) {
segments.push(plural(hours % 24, 'hour'));
}
if (hours < 4) {
if (minutes % 60 > 0) {
segments.push(plural(minutes % 60, 'minute'));
}
if (minutes < 5) {
if (seconds % 60 > 0) {
segments.push(plural(seconds % 60, 'second'));
}
}
}
}
return segments.join(", ");
}
// this is producing local time!
function formatDate(date) {
// return padNum(date.getDate()) + ". " + padNum(date.getMonth() + 1) + ". " + date.getFullYear();
@ -489,6 +522,7 @@ export default {
formatDate,
formatDateISO,
formatDateTime,
formatTimeInterval,
formatSize,
localNowDateTime,
now,

35
src/public/app/widgets/attachment_detail.js
View File

@ -2,6 +2,7 @@ import utils from "../services/utils.js";
import AttachmentActionsWidget from "./buttons/attachments_actions.js";
import BasicWidget from "./basic_widget.js";
import server from "../services/server.js";
import options from "../services/options.js";
const TPL = `
<div class="attachment-detail">
@ -44,6 +45,10 @@ const TPL = `
max-width: 90%;
object-fit: contain;
}
.attachment-detail-wrapper.scheduled-for-deletion .attachment-content img {
filter: contrast(10%);
}
</style>
<div class="attachment-detail-wrapper">
@ -54,6 +59,8 @@ const TPL = `
<div class="attachment-actions-container"></div>
</div>
<div class="attachment-deletion-warning alert alert-info"></div>
<div class="attachment-content"></div>
</div>
</div>`;
@ -100,15 +107,29 @@ export default class AttachmentDetailWidget extends BasicWidget {
.text(this.attachment.title);
}
const {utcDateScheduledForDeletionSince} = this.attachment;
const $deletionWarning = this.$wrapper.find('.attachment-deletion-warning');
const {utcDateScheduledForErasureSince} = this.attachment;
if (utcDateScheduledForDeletionSince) {
const scheduledSinceTimestamp = utils.parseDate(utcDateScheduledForDeletionSince)?.getTime();
const interval = 3600 * 1000;
const deletionTimestamp = scheduledSinceTimestamp + interval;
const willBeDeletedInSeconds = Math.round((deletionTimestamp - Date.now()) / 1000);
if (utcDateScheduledForErasureSince) {
this.$wrapper.addClass("scheduled-for-deletion");
this.$wrapper.find('.attachment-title').append(`Will be deleted in ${willBeDeletedInSeconds} seconds.`);
const scheduledSinceTimestamp = utils.parseDate(utcDateScheduledForErasureSince)?.getTime();
const intervalMs = options.getInt('eraseUnusedImageAttachmentsAfterSeconds') * 1000;
const deletionTimestamp = scheduledSinceTimestamp + intervalMs;
const willBeDeletedInMs = deletionTimestamp - Date.now();
$deletionWarning.show();
if (willBeDeletedInMs >= 60000) {
$deletionWarning.text(`This attachment will be deleted in ${utils.formatTimeInterval(willBeDeletedInMs)}`);
} else {
$deletionWarning.text(`This attachment will be deleted soon`);
}
$deletionWarning.append(", because the image attachment is not used. To prevent deletion, add the image back into the note.");
} else {
this.$wrapper.removeClass("scheduled-for-deletion");
$deletionWarning.hide();
}
this.$wrapper.find('.attachment-details')

3
src/routes/api/options.js
View File

@ -61,7 +61,8 @@ const ALLOWED_OPTIONS = new Set([
'downloadImagesAutomatically',
'minTocHeadings',
'checkForUpdates',
'disableTray'
'disableTray',
'eraseUnusedImageAttachmentsAfterSeconds'
]);
function getOptions() {

54
src/services/notes.js
View File

@ -21,6 +21,7 @@ const htmlSanitizer = require("./html_sanitizer");
const ValidationError = require("../errors/validation_error");
const noteTypesService = require("./note_types");
const fs = require("fs");
const BAttachment = require("../becca/entities/battachment");
/** @param {BNote} parentNote */
function getNewNotePosition(parentNote) {
@ -342,17 +343,17 @@ function checkImageAttachments(note, content) {
let match;
while (match = re.exec(content)) {
foundAttachmentIds.push(match[1]);
foundAttachmentIds.add(match[1]);
}
for (const attachment of note.getAttachmentByRole('image')) {
const imageInContent = foundAttachmentIds.has(attachment.attachmentId);
if (attachment.utcDateScheduledForDeletionSince && imageInContent) {
attachment.utcDateScheduledForDeletionSince = null;
if (attachment.utcDateScheduledForErasureSince && imageInContent) {
attachment.utcDateScheduledForErasureSince = null;
attachment.save();
} else if (!attachment.utcDateScheduledForDeletionSince && !imageInContent) {
attachment.utcDateScheduledForDeletionSince = dateUtils.utcNowDateTime();
} else if (!attachment.utcDateScheduledForErasureSince && !imageInContent) {
attachment.utcDateScheduledForErasureSince = dateUtils.utcNowDateTime();
attachment.save();
}
}
@ -841,6 +842,33 @@ function eraseAttributes(attributeIdsToErase) {
log.info(`Erased attributes: ${JSON.stringify(attributeIdsToErase)}`);
}
function eraseAttachments(attachmentIdsToErase) {
if (attachmentIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM attachments WHERE attachmentId IN (???)`, attachmentIdsToErase);
setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'attachments' AND entityId IN (???)`, attachmentIdsToErase));
log.info(`Erased attachments: ${JSON.stringify(attachmentIdsToErase)}`);
}
function eraseUnusedBlobs() {
const unusedBlobIds = sql.getColumn(`
SELECT blobId
FROM blobs
LEFT JOIN notes ON notes.blobId = blobs.blobId
LEFT JOIN attachments ON attachments.blobId = blobs.blobId
WHERE notes.noteId IS NULL AND attachments.attachmentId IS NULL`);
sql.executeMany(`DELETE FROM blobs WHERE blobId IN (???)`, unusedBlobIds);
setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'blobs' AND entityId IN (???)`, unusedBlobIds));
log.info(`Erased unused blobs: ${JSON.stringify(unusedBlobIds)}`);
}
function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
// this is important also so that the erased entity changes are sent to the connected clients
sql.transactional(() => {
@ -861,6 +889,12 @@ function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
const attributeIdsToErase = sql.getColumn("SELECT attributeId FROM attributes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
eraseAttributes(attributeIdsToErase);
const attachmentIdsToErase = sql.getColumn("SELECT attachmentId FROM attachments WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
eraseAttachments(attachmentIdsToErase);
eraseUnusedBlobs();
});
}
@ -1013,11 +1047,21 @@ function getNoteIdMapping(origNote) {
return noteIdMapping;
}
function eraseScheduledAttachments() {
const eraseIntervalSeconds = optionService.getOptionInt('eraseUnusedImageAttachmentsAfterSeconds');
const cutOffDate = dateUtils.utcDateTimeStr(new Date(Date.now() - (eraseIntervalSeconds * 1000)));
const attachmentIdsToErase = sql.getColumn('SELECT attachmentId FROM attachments WHERE utcDateScheduledForErasureSince < ?', [cutOffDate]);
eraseAttachments(attachmentIdsToErase);
}
sqlInit.dbReady.then(() => {
// first cleanup kickoff 5 minutes after startup
setTimeout(cls.wrap(() => eraseDeletedEntities()), 5 * 60 * 1000);
setTimeout(cls.wrap(() => eraseScheduledAttachments()), 6 * 60 * 1000);
setInterval(cls.wrap(() => eraseDeletedEntities()), 4 * 3600 * 1000);
setInterval(cls.wrap(() => eraseScheduledAttachments()), 3600 * 1000);
});
module.exports = {

1
src/services/options_init.js
View File

@ -88,6 +88,7 @@ const defaultOptions = [
{ name: 'minTocHeadings', value: '5', isSynced: true },
{ name: 'checkForUpdates', value: 'true', isSynced: true },
{ name: 'disableTray', value: 'false', isSynced: false },
{ name: 'eraseUnusedImageAttachmentsAfterSeconds', value: '86400', isSynced: false },
];
function initStartupOptions() {