From 2af86927b06ddbf1168662e5bc4f6411d5af595c Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 1 Nov 2019 22:09:51 +0100 Subject: [PATCH] added flag for the erased notes --- db/migrations/0151__add_isCleaned_to_note.sql | 31 ++++++++++++++++ db/schema.sql | 35 +++++++++++-------- package-lock.json | 8 ++--- package.json | 2 +- src/entities/note.js | 1 + src/services/app_info.js | 2 +- src/services/consistency_checks.js | 17 +++++++++ src/services/notes.js | 30 +++++++++++++--- src/services/sql.js | 6 ++++ 9 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 db/migrations/0151__add_isCleaned_to_note.sql diff --git a/db/migrations/0151__add_isCleaned_to_note.sql b/db/migrations/0151__add_isCleaned_to_note.sql new file mode 100644 index 000000000..57ab1985a --- /dev/null +++ b/db/migrations/0151__add_isCleaned_to_note.sql @@ -0,0 +1,31 @@ +CREATE TABLE IF NOT EXISTS "notes_mig" ( + `noteId` TEXT NOT NULL, + `title` TEXT NOT NULL DEFAULT "note", + `isProtected` INT NOT NULL DEFAULT 0, + `type` TEXT NOT NULL DEFAULT 'text', + `mime` TEXT NOT NULL DEFAULT 'text/html', + `hash` TEXT DEFAULT "" NOT NULL, + `isDeleted` INT NOT NULL DEFAULT 0, + `isErased` INT NOT NULL DEFAULT 0, + `dateCreated` TEXT NOT NULL, + `dateModified` TEXT NOT NULL, + `utcDateCreated` TEXT NOT NULL, + `utcDateModified` TEXT NOT NULL, + PRIMARY KEY(`noteId`)); + +INSERT INTO notes_mig (noteId, title, isProtected, type, mime, hash, isDeleted, isErased, dateCreated, dateModified, utcDateCreated, utcDateModified) +SELECT noteId, title, isProtected, type, mime, hash, isDeleted, 0, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes; + +DROP TABLE notes; +ALTER TABLE notes_mig RENAME TO notes; + +UPDATE notes SET isErased = 1 WHERE isDeleted = 1 +AND (SELECT CASE content WHEN NULL THEN 1 ELSE 0 END FROM note_contents WHERE note_contents.noteId = notes.noteId); + +CREATE INDEX `IDX_notes_isDeleted` ON `notes` (`isDeleted`); +CREATE INDEX `IDX_notes_title` ON `notes` (`title`); +CREATE INDEX `IDX_notes_type` ON `notes` (`type`); +CREATE INDEX `IDX_notes_dateCreated` ON `notes` (`dateCreated`); +CREATE INDEX `IDX_notes_dateModified` ON `notes` (`dateModified`); +CREATE INDEX `IDX_notes_utcDateModified` ON `notes` (`utcDateModified`); +CREATE INDEX `IDX_notes_utcDateCreated` ON `notes` (`utcDateCreated`); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index c13e2bae5..c1d27d76f 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -38,20 +38,6 @@ CREATE TABLE IF NOT EXISTS "attributes" isDeleted INT not null, hash TEXT default "" not null, isInheritable int DEFAULT 0 NULL); -CREATE TABLE IF NOT EXISTS "notes" ( - `noteId` TEXT NOT NULL, - `title` TEXT NOT NULL DEFAULT "note", - `isProtected` INT NOT NULL DEFAULT 0, - `type` TEXT NOT NULL DEFAULT 'text', - `mime` TEXT NOT NULL DEFAULT 'text/html', - `hash` TEXT DEFAULT "" NOT NULL, - `isDeleted` INT NOT NULL DEFAULT 0, - `dateCreated` TEXT NOT NULL, - `dateModified` TEXT NOT NULL, - `utcDateCreated` TEXT NOT NULL, - `utcDateModified` TEXT NOT NULL, - PRIMARY KEY(`noteId`) -); CREATE UNIQUE INDEX `IDX_sync_entityName_entityId` ON `sync` ( `entityName`, `entityId` @@ -119,3 +105,24 @@ CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCr CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`); CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`); CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`); +CREATE TABLE IF NOT EXISTS "notes" ( + `noteId` TEXT NOT NULL, + `title` TEXT NOT NULL DEFAULT "note", + `isProtected` INT NOT NULL DEFAULT 0, + `type` TEXT NOT NULL DEFAULT 'text', + `mime` TEXT NOT NULL DEFAULT 'text/html', + `hash` TEXT DEFAULT "" NOT NULL, + `isDeleted` INT NOT NULL DEFAULT 0, + `isErased` INT NOT NULL DEFAULT 0, + `dateCreated` TEXT NOT NULL, + `dateModified` TEXT NOT NULL, + `utcDateCreated` TEXT NOT NULL, + `utcDateModified` TEXT NOT NULL, + PRIMARY KEY(`noteId`)); +CREATE INDEX `IDX_notes_isDeleted` ON `notes` (`isDeleted`); +CREATE INDEX `IDX_notes_title` ON `notes` (`title`); +CREATE INDEX `IDX_notes_type` ON `notes` (`type`); +CREATE INDEX `IDX_notes_dateCreated` ON `notes` (`dateCreated`); +CREATE INDEX `IDX_notes_dateModified` ON `notes` (`dateModified`); +CREATE INDEX `IDX_notes_utcDateModified` ON `notes` (`utcDateModified`); +CREATE INDEX `IDX_notes_utcDateCreated` ON `notes` (`utcDateCreated`); diff --git a/package-lock.json b/package-lock.json index d732e93a1..0432bf7d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "trilium", - "version": "0.36.1-beta", + "version": "0.36.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -5219,9 +5219,9 @@ } }, "file-type": { - "version": "12.3.1", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-12.3.1.tgz", - "integrity": "sha512-FXxY5h6vSYMjrRal4YqbtfuoKD/oE0AMjJ7E5Hm+BdaQECcFVD03B41RAWYJ7wyuLr/wRnCtFo7y37l+nh+TAA==" + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-12.4.0.tgz", + "integrity": "sha512-WTvyKq8yjtNmUtVAD8LGcTkvtCdJglM6ks2HTqEClm6+65XTqM6MoZYA1Vtra50DLRWLiM38fEs1y56f5VhnUA==" }, "filename-regex": { "version": "2.0.1", diff --git a/package.json b/package.json index 7fb5df40f..2c9d330ea 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "electron-window-state": "5.0.3", "express": "4.17.1", "express-session": "1.17.0", - "file-type": "12.3.1", + "file-type": "12.4.0", "fs-extra": "8.1.0", "helmet": "3.21.2", "html": "1.0.0", diff --git a/src/entities/note.js b/src/entities/note.js index 2c59f4ce9..f8b2748a4 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -23,6 +23,7 @@ const RELATION_DEFINITION = 'relation-definition'; * @property {string} title - note title * @property {boolean} isProtected - true if note is protected * @property {boolean} isDeleted - true if note is deleted + * @property {boolean} isErased - true if note's content is erased after it has been deleted * @property {string} dateCreated - local date time (with offset) * @property {string} dateModified - local date time (with offset) * @property {string} utcDateCreated diff --git a/src/services/app_info.js b/src/services/app_info.js index 402425b92..b1c19ab5d 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -4,7 +4,7 @@ const build = require('./build'); const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); -const APP_DB_VERSION = 150; +const APP_DB_VERSION = 151; const SYNC_VERSION = 11; const CLIPPER_PROTOCOL_VERSION = "1.0"; diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index 9393b6224..e669b63b6 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -228,6 +228,23 @@ async function findLogicIssues() { AND content IS NULL`, ({noteId}) => `Note ${noteId} content is null even though it is not deleted`); + await findIssues(` + SELECT noteId + FROM notes + JOIN note_contents USING(noteId) + WHERE + isErased = 1 + AND content IS NOT NULL`, + ({noteId}) => `Note ${noteId} content is not null even though the note is erased`); + + await findIssues(` + SELECT noteId + FROM notes + WHERE + isErased = 1 + AND isDeleted = 0`, + ({noteId}) => `Note ${noteId} is not deleted even though it is erased`); + await findIssues(` SELECT parentNoteId FROM diff --git a/src/services/notes.js b/src/services/notes.js index 7a37e3c2d..888e9d3d6 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -457,15 +457,35 @@ async function scanForLinks(noteId) { } } -async function cleanupDeletedNotes() { +async function eraseDeletedNotes() { const cutoffDate = new Date(Date.now() - 48 * 3600 * 1000); + const noteIdsToErase = await sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND isErased = 0 AND notes.utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]); + + const utcNowDateTime = dateUtils.utcNowDateTime(); + const localNowDateTime = dateUtils.localNowDateTime(); + // it's better to not use repository for this because it will complain about saving protected notes // out of protected session - await sql.execute("UPDATE note_contents SET content = NULL WHERE content IS NOT NULL AND noteId IN (SELECT noteId FROM notes WHERE isDeleted = 1 AND notes.utcDateModified <= ?)", [dateUtils.utcDateStr(cutoffDate)]); + await sql.executeMany(` + UPDATE notes + SET isErased = 1, + utcDateModified = '${utcNowDateTime}', + dateModified = '${localNowDateTime}' + WHERE noteId IN (???)`, noteIdsToErase); - await sql.execute("UPDATE note_revisions SET content = NULL WHERE note_revisions.content IS NOT NULL AND noteId IN (SELECT noteId FROM notes WHERE isDeleted = 1 AND notes.utcDateModified <= ?)", [dateUtils.utcDateStr(cutoffDate)]); + await sql.executeMany(` + UPDATE note_contents + SET content = NULL, + utcDateModified = '${utcNowDateTime}' + WHERE noteId IN (???)`, noteIdsToErase); + + await sql.executeMany(` + UPDATE note_revisions + SET content = NULL, + utcDateModified = '${utcNowDateTime}' + WHERE noteId IN (???)`, noteIdsToErase); } async function duplicateNote(noteId, parentNoteId) { @@ -508,9 +528,9 @@ async function duplicateNote(noteId, parentNoteId) { sqlInit.dbReady.then(() => { // first cleanup kickoff 5 minutes after startup - setTimeout(cls.wrap(cleanupDeletedNotes), 5 * 60 * 1000); + setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000); - setInterval(cls.wrap(cleanupDeletedNotes), 4 * 3600 * 1000); + setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000); }); module.exports = { diff --git a/src/services/sql.js b/src/services/sql.js index 659b315d2..26b1bb2b8 100644 --- a/src/services/sql.js +++ b/src/services/sql.js @@ -152,6 +152,11 @@ async function execute(query, params = []) { return result; } +async function executeMany(query, params) { + // essentially just alias + await getManyRows(query, params); +} + async function executeScript(query) { return await wrap(async db => db.exec(query)); } @@ -235,6 +240,7 @@ module.exports = { getMap, getColumn, execute, + executeMany, executeScript, transactional, upsert