gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

cleanup obsolete image code

Browse Source
This commit is contained in:
azivner 2018-11-08 11:14:13 +01:00
parent d7afbe4059
commit afcbfcfa03
12 changed files with 1 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
db/main_images.sql
View File

File diff suppressed because one or more lines are too long

2
db/main_note_images.sql
View File

@ -1,2 +0,0 @@
INSERT INTO note_images VALUES('2EtgRRPfk4Fi','1Heh2acXfPNt','ed64aET6i379',0,'2018-01-08T04:41:30.663Z','2018-01-08T04:41:30.663Z','');
INSERT INTO note_images VALUES('T7qQAw9BVi0E','prjUbW6QtsL4','0mLHhGv61RDM',0,'2018-08-30T07:53:53.165Z','2018-08-30T07:53:53.164Z','lSz2qni/Rx');

10
src/entities/entity_constructor.js
View File

@ -1,7 +1,5 @@
const Note = require('../entities/note');
const NoteRevision = require('../entities/note_revision');
const Image = require('../entities/image');
const NoteImage = require('../entities/note_image');
const Link = require('../entities/link');
const Branch = require('../entities/branch');
const Attribute = require('../entities/attribute');
@ -12,8 +10,6 @@ const repository = require('../services/repository');
const ENTITY_NAME_TO_ENTITY = {
"attributes": Attribute,
"images": Image,
"note_images": NoteImage,
"branches": Branch,
"notes": Note,
"note_revisions": NoteRevision,
@ -42,12 +38,6 @@ function createEntityFromRow(row) {
else if (row.linkId) {
entity = new Link(row);
}
else if (row.noteImageId) {
entity = new NoteImage(row);
}
else if (row.imageId) {
entity = new Image(row);
}
else if (row.branchId && row.notePath) {
entity = new RecentNote(row);
}

42
src/entities/image.js
View File

@ -1,42 +0,0 @@
"use strict";
const Entity = require('./entity');
const dateUtils = require('../services/date_utils');
/**
* This class represents image data.
*
* @param {string} imageId
* @param {string} format
* @param {string} checksum
* @param {string} name
* @param {blob} data
* @param {boolean} isDeleted
* @param {string} dateModified
* @param {string} dateCreated
*
* @extends Entity
*/
class Image extends Entity {
static get entityName() { return "images"; }
static get primaryKeyName() { return "imageId"; }
static get hashedProperties() { return ["imageId", "format", "checksum", "name", "isDeleted", "dateCreated"]; }
beforeSaving() {
if (!this.isDeleted) {
this.isDeleted = false;
}
if (!this.dateCreated) {
this.dateCreated = dateUtils.nowDate();
}
super.beforeSaving();
if (this.isChanged) {
this.dateModified = dateUtils.nowDate();
}
}
}
module.exports = Image;

49
src/entities/note_image.js
View File

@ -1,49 +0,0 @@
"use strict";
const Entity = require('./entity');
const repository = require('../services/repository');
const dateUtils = require('../services/date_utils');
/**
* This class represents image's placement in the note(s). One image may be placed into several notes.
*
* @param {string} noteImageId
* @param {string} noteId
* @param {string} imageId
* @param {boolean} isDeleted
* @param {string} dateModified
* @param {string} dateCreated
*
* @extends Entity
*/
class NoteImage extends Entity {
static get entityName() { return "note_images"; }
static get primaryKeyName() { return "noteImageId"; }
static get hashedProperties() { return ["noteImageId", "noteId", "imageId", "isDeleted", "dateCreated"]; }
async getNote() {
return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]);
}
async getImage() {
return await repository.getEntity("SELECT * FROM images WHERE imageId = ?", [this.imageId]);
}
beforeSaving() {
if (!this.isDeleted) {
this.isDeleted = false;
}
if (!this.dateCreated) {
this.dateCreated = dateUtils.nowDate();
}
super.beforeSaving();
if (this.isChanged) {
this.dateModified = dateUtils.nowDate();
}
}
}
module.exports = NoteImage;

26
src/routes/api/cleanup.js
View File

@ -2,25 +2,6 @@
const sql = require('../../services/sql');
const log = require('../../services/log');
const repository = require('../../services/repository');
async function cleanupUnusedImages() {
const unusedImages = await repository.getEntities(`
SELECT images.*
FROM images
LEFT JOIN note_images ON note_images.imageId = images.imageId AND note_images.isDeleted = 0
WHERE
images.isDeleted = 0
AND note_images.noteImageId IS NULL`);
for (const image of unusedImages) {
log.info(`Deleting unused image: ${image.imageId}`);
image.isDeleted = true;
image.data = null;
await image.save();
}
}
async function vacuumDatabase() {
await sql.execute("VACUUM");
@ -28,13 +9,6 @@ async function vacuumDatabase() {
log.info("Database has been vacuumed.");
}
// Running this periodically is a bit dangerous because it is possible during the normal usage
// that user removed image from its only note, but keeps its URL in clipboard and pastes it into
// a different note. If this cleanup happens during this moment, we delete the image before new note_images
// reference is created. But currently we don't have a better way to do this.
setInterval(cleanupUnusedImages, 4 * 3600 * 1000);
module.exports = {
cleanupUnusedImages,
vacuumDatabase
};

4
src/services/content_hash.js
View File

@ -7,10 +7,8 @@ const eventLogService = require('./event_log');
const messagingService = require('./messaging');
const ApiToken = require('../entities/api_token');
const Branch = require('../entities/branch');
const Image = require('../entities/image');
const Note = require('../entities/note');
const Attribute = require('../entities/attribute');
const NoteImage = require('../entities/note_image');
const NoteRevision = require('../entities/note_revision');
const RecentNote = require('../entities/recent_note');
const Option = require('../entities/option');
@ -38,8 +36,6 @@ async function getHashes() {
note_revisions: await getHash(NoteRevision),
recent_notes: await getHash(RecentNote),
options: await getHash(Option, "isSynced = 1"),
images: await getHash(Image),
note_images: await getHash(NoteImage),
attributes: await getHash(Attribute),
api_tokens: await getHash(ApiToken)
};

2
src/services/sql_init.js
View File

@ -73,8 +73,6 @@ async function createInitialDatabase(username, password) {
const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
const notesSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_notes.sql', 'UTF-8');
const notesTreeSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_branches.sql', 'UTF-8');
const imagesSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_images.sql', 'UTF-8');
const notesImageSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_note_images.sql', 'UTF-8');
const attributesSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_attributes.sql', 'UTF-8');
await sql.transactional(async () => {

2
src/services/sync.js
View File

@ -243,8 +243,6 @@ const primaryKeys = {
"branches": "branchId",
"note_revisions": "noteRevisionId",
"recent_notes": "branchId",
"images": "imageId",
"note_images": "noteImageId",
"api_tokens": "apiTokenId",
"options": "name",
"attributes": "attributeId",

11
src/services/sync_table.js
View File

@ -32,14 +32,6 @@ async function addLinkSync(linkId, sourceId) {
await addEntitySync("links", linkId, sourceId);
}
async function addImageSync(imageId, sourceId) {
await addEntitySync("images", imageId, sourceId);
}
async function addNoteImageSync(noteImageId, sourceId) {
await addEntitySync("note_images", noteImageId, sourceId);
}
async function addAttributeSync(attributeId, sourceId) {
await addEntitySync("attributes", attributeId, sourceId);
}
@ -108,10 +100,9 @@ module.exports = {
addNoteRevisionSync,
addOptionsSync,
addRecentNoteSync,
addImageSync,
addNoteImageSync,
addAttributeSync,
addApiTokenSync,
addLinkSync,
addEntitySync,
fillAllSyncRows
};

38
src/services/sync_update.js
View File

@ -27,12 +27,6 @@ async function updateEntity(sync, entity, sourceId) {
else if (entityName === 'links') {
await updateLink(entity, sourceId);
}
else if (entityName === 'images') {
await updateImage(entity, sourceId);
}
else if (entityName === 'note_images') {
await updateNoteImage(entity, sourceId);
}
else if (entityName === 'attributes') {
await updateAttribute(entity, sourceId);
}
@ -156,38 +150,6 @@ async function updateLink(entity, sourceId) {
}
}
async function updateImage(entity, sourceId) {
if (entity.data !== null) {
entity.data = Buffer.from(entity.data, 'base64');
}
const origImage = await sql.getRow("SELECT * FROM images WHERE imageId = ?", [entity.imageId]);
if (!origImage || origImage.dateModified <= entity.dateModified) {
await sql.transactional(async () => {
await sql.replace("images", entity);
await syncTableService.addImageSync(entity.imageId, sourceId);
});
log.info("Update/sync image " + entity.imageId);
}
}
async function updateNoteImage(entity, sourceId) {
const origNoteImage = await sql.getRow("SELECT * FROM note_images WHERE noteImageId = ?", [entity.noteImageId]);
if (!origNoteImage || origNoteImage.dateModified <= entity.dateModified) {
await sql.transactional(async () => {
await sql.replace("note_images", entity);
await syncTableService.addNoteImageSync(entity.noteImageId, sourceId);
});
log.info("Update/sync note image " + entity.noteImageId);
}
}
async function updateAttribute(entity, sourceId) {
const origAttribute = await sql.getRow("SELECT * FROM attributes WHERE attributeId = ?", [entity.attributeId]);

8
src/views/dialogs/options.ejs
View File

@ -164,14 +164,6 @@
<p>This action will create a new copy of the database and anonymise it (remove all note content and leave only structure and metadata)
for sharing online for debugging purposes without fear of leaking your personal data.</p>
<h4>Image cleanup</h4>
<p>This will remove all image data of images not used in any current version of note from the database (metadata will remain).
This means that some images can disappear from note revisions.</p>
<button id="cleanup-unused-images-button" class="btn btn-sm">Permanently cleanup unused images</button>
<h4>Vacuum database</h4>
<p>This will rebuild database which will typically result in smaller database file. No data will be actually changed.</p>