From 01370a5968f5e773e378753bfc5d9d452bc9bc7d Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 29 Nov 2019 21:42:24 +0100 Subject: [PATCH 1/7] fix anonymization according to latest schema --- package-lock.json | 2 +- src/services/anonymization.js | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5a19e325c..b51033fc2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "trilium", - "version": "0.37.5", + "version": "0.37.6", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/services/anonymization.js b/src/services/anonymization.js index e5c356eb5..b41179485 100644 --- a/src/services/anonymization.js +++ b/src/services/anonymization.js @@ -18,7 +18,10 @@ async function anonymize() { await db.run("UPDATE notes SET title = 'title'"); await db.run("UPDATE note_contents SET content = 'text'"); - await db.run("UPDATE note_revisions SET title = 'title', content = 'text'"); + await db.run("UPDATE note_revisions SET title = 'title'"); + await db.run("UPDATE note_revision_contents SET content = 'title'"); + await db.run("UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label'"); + await db.run("UPDATE attributes SET name = 'name' WHERE type = 'relation'"); await db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL"); await db.run(`UPDATE options SET value = 'anonymized' WHERE name IN ('documentSecret', 'encryptedDataKey', 'passwordVerificationHash', From d0e6be3e0c2d6df725e0f032d1aef1d26376ac8a Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 30 Nov 2019 09:15:08 +0100 Subject: [PATCH 2/7] entity stat as part of consistency checks --- src/services/consistency_checks.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index a8f81bd31..ae9d5010c 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -626,12 +626,31 @@ async function runAllChecks() { return !unrecoveredConsistencyErrors; } +async function showEntityStat(name, query) { + const map = await sql.getMap(query); + + map[0] = map[0] || 0; + map[1] = map[1] || 0; + + log.info(`${name} deleted: ${map[1]}, not deleted ${map[0]}`); +} + +async function runDbDiagnostics() { + await showEntityStat("Notes", `SELECT isDeleted, count(noteId) FROM notes GROUP BY isDeleted`); + await showEntityStat("Note revisions", `SELECT isErased, count(noteRevisionId) FROM note_revisions GROUP BY isErased`); + await showEntityStat("Branches", `SELECT isDeleted, count(branchId) FROM branches GROUP BY isDeleted`); + await showEntityStat("Attributes", `SELECT isDeleted, count(attributeId) FROM attributes GROUP BY isDeleted`); + await showEntityStat("API tokens", `SELECT isDeleted, count(apiTokenId) FROM api_tokens GROUP BY isDeleted`); +} + async function runChecks() { let elapsedTimeMs; await syncMutexService.doExclusively(async () => { const startTime = new Date(); + await runDbDiagnostics(); + await runAllChecks(); elapsedTimeMs = Date.now() - startTime.getTime(); @@ -663,7 +682,7 @@ sqlInit.dbReady.then(() => { setInterval(cls.wrap(runChecks), 60 * 60 * 1000); // kickoff checks soon after startup (to not block the initial load) - setTimeout(cls.wrap(runChecks), 10 * 1000); + setTimeout(cls.wrap(runChecks), 20 * 1000); }); module.exports = {}; \ No newline at end of file From 995ebbf57776135c3f94ee01da3a9fd60fe86f4a Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 30 Nov 2019 10:41:53 +0100 Subject: [PATCH 3/7] removed foreign keys PRAGMAs since foreign key constraints are not used anymore --- src/services/migration.js | 7 ------- src/services/sql_init.js | 2 -- 2 files changed, 9 deletions(-) diff --git a/src/services/migration.js b/src/services/migration.js index 2b93f0064..4c82a36d6 100644 --- a/src/services/migration.js +++ b/src/services/migration.js @@ -43,9 +43,6 @@ async function migrate() { try { log.info("Attempting migration to version " + mig.dbVersion); - // needs to happen outside of the transaction (otherwise it's a NO-OP) - await sql.execute("PRAGMA foreign_keys = OFF"); - await sql.transactional(async () => { if (mig.type === 'sql') { const migrationSql = fs.readFileSync(resourceDir.MIGRATIONS_DIR + "/" + mig.file).toString('utf8'); @@ -76,10 +73,6 @@ async function migrate() { utils.crash(); } - finally { - // make sure foreign keys are enabled even if migration script disables them - await sql.execute("PRAGMA foreign_keys = ON"); - } } if (await sqlInit.isDbUpToDate()) { diff --git a/src/services/sql_init.js b/src/services/sql_init.js index 45f8272a1..7e81d2d02 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -57,8 +57,6 @@ async function initDbConnection() { return; } - await sql.execute("PRAGMA foreign_keys = ON"); - const currentDbVersion = await getDbVersion(); if (currentDbVersion > appInfo.dbVersion) { From 67663fba506918f9ca60af43711b4fe979b5072e Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 30 Nov 2019 11:36:36 +0100 Subject: [PATCH 4/7] fixes --- src/services/notes.js | 4 ++-- src/services/sql_init.js | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/services/notes.js b/src/services/notes.js index cd516eaab..171e0c8f0 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -480,7 +480,7 @@ async function eraseDeletedNotes() { SET content = NULL, utcDateModified = '${utcNowDateTime}' WHERE noteRevisionId IN - (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN ((???)))`, noteIdsToErase); + (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN (???))`, noteIdsToErase); await sql.executeMany(` UPDATE note_revisions @@ -530,7 +530,7 @@ async function duplicateNote(noteId, parentNoteId) { sqlInit.dbReady.then(() => { // first cleanup kickoff 5 minutes after startup - setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000); + setTimeout(cls.wrap(eraseDeletedNotes), 0 * 60 * 1000); setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000); }); diff --git a/src/services/sql_init.js b/src/services/sql_init.js index 7e81d2d02..e93ff5ad0 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -173,9 +173,11 @@ async function isDbUpToDate() { } async function dbInitialized() { - await optionService.setOption('initialized', 'true'); + if (!await isDbInitialized()) { + await optionService.setOption('initialized', 'true'); - await initDbConnection(); + await initDbConnection(); + } } dbReady.then(async () => { From 07b3d11fe5d52a2a54b9565f8e14cde7a02e9239 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 1 Dec 2019 09:19:16 +0100 Subject: [PATCH 5/7] fix generate new document script --- src/tools/generate_document.js | 39 ++++++++++------------------------ 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/src/tools/generate_document.js b/src/tools/generate_document.js index 7c2fb6b1e..159243b4c 100644 --- a/src/tools/generate_document.js +++ b/src/tools/generate_document.js @@ -1,37 +1,22 @@ -const fs = require('fs'); -const dataDir = require('../services/data_dir'); - -fs.unlinkSync(dataDir.DOCUMENT_PATH); +/** + * Usage: node src/tools/generate_document.js 1000 + * will create 1000 new notes and some clones into a current document.db + */ require('../entities/entity_constructor'); -const optionService = require('../services/options'); const sqlInit = require('../services/sql_init'); -const myScryptService = require('../services/my_scrypt'); -const passwordEncryptionService = require('../services/password_encryption'); -const utils = require('../services/utils'); const noteService = require('../services/notes'); const cls = require('../services/cls'); const cloningService = require('../services/cloning'); -const loremIpsum = require('lorem-ipsum'); - -async function setUserNamePassword() { - const username = "test"; - const password = "test"; - - await optionService.setOption('username', username); - - await optionService.setOption('passwordVerificationSalt', utils.randomSecureToken(32)); - await optionService.setOption('passwordDerivedKeySalt', utils.randomSecureToken(32)); - - const passwordVerificationKey = utils.toBase64(await myScryptService.getVerificationHash(password)); - await optionService.setOption('passwordVerificationHash', passwordVerificationKey); - - await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16)); - - await sqlInit.initDbConnection(); -} +const loremIpsum = require('lorem-ipsum').loremIpsum; const noteCount = parseInt(process.argv[2]); + +if (!noteCount) { + console.error(`Please enter number of notes as program parameter.`); + process.exit(1); +} + const notes = ['root']; function getRandomParentNoteId() { @@ -41,8 +26,6 @@ function getRandomParentNoteId() { } async function start() { - await setUserNamePassword(); - for (let i = 0; i < noteCount; i++) { const title = loremIpsum({ count: 1, units: 'sentences', sentenceLowerBound: 1, sentenceUpperBound: 10 }); From 29c5e394abcd31d51c3d37c0165ae08729b2dc47 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 1 Dec 2019 10:20:18 +0100 Subject: [PATCH 6/7] generate document now creates also labels and relations --- src/services/notes.js | 2 +- src/tools/generate_document.js | 37 ++++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/services/notes.js b/src/services/notes.js index 171e0c8f0..8675666c1 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -117,7 +117,7 @@ async function createNewNote(parentNoteId, noteData) { isExpanded: !!noteData.isExpanded }).save(); - for (const attr of await parentNote.getAttributes()) { + for (const attr of await parentNote.getOwnedAttributes()) { if (attr.name.startsWith("child:")) { await new Attribute({ noteId: note.noteId, diff --git a/src/tools/generate_document.js b/src/tools/generate_document.js index 159243b4c..80c2b6318 100644 --- a/src/tools/generate_document.js +++ b/src/tools/generate_document.js @@ -6,6 +6,7 @@ require('../entities/entity_constructor'); const sqlInit = require('../services/sql_init'); const noteService = require('../services/notes'); +const attributeService = require('../services/attributes'); const cls = require('../services/cls'); const cloningService = require('../services/cloning'); const loremIpsum = require('lorem-ipsum').loremIpsum; @@ -19,7 +20,7 @@ if (!noteCount) { const notes = ['root']; -function getRandomParentNoteId() { +function getRandomNoteId() { const index = Math.floor(Math.random() * notes.length); return notes[index]; @@ -33,17 +34,17 @@ async function start() { const content = loremIpsum({ count: paragraphCount, units: 'paragraphs', sentenceLowerBound: 1, sentenceUpperBound: 15, paragraphLowerBound: 3, paragraphUpperBound: 10, format: 'html' }); - const {note} = await noteService.createNote(getRandomParentNoteId(), title, content); + const {note} = await noteService.createNote(getRandomNoteId(), title, content); console.log(`Created note ${i}: ${title}`); notes.push(note.noteId); } - // we'll create clones for 20% of notes - for (let i = 0; i < (noteCount / 50); i++) { - const noteIdToClone = getRandomParentNoteId(); - const parentNoteId = getRandomParentNoteId(); + // we'll create clones for 4% of notes + for (let i = 0; i < (noteCount / 25); i++) { + const noteIdToClone = getRandomNoteId(); + const parentNoteId = getRandomNoteId(); const prefix = Math.random() > 0.8 ? "prefix" : null; const result = await cloningService.cloneNoteToParent(noteIdToClone, parentNoteId, prefix); @@ -51,6 +52,30 @@ async function start() { console.log(`Cloning ${i}:`, result.success ? "succeeded" : "FAILED"); } + for (let i = 0; i < noteCount; i++) { + await attributeService.createAttribute({ + noteId: getRandomNoteId(), + type: 'label', + name: 'label', + value: 'value', + isInheritable: Math.random() > 0.1 // 10% are inheritable + }); + + console.log(`Creating label ${i}`); + } + + for (let i = 0; i < noteCount; i++) { + await attributeService.createAttribute({ + noteId: getRandomNoteId(), + type: 'relation', + name: 'relation', + value: getRandomNoteId(), + isInheritable: Math.random() > 0.1 // 10% are inheritable + }); + + console.log(`Creating relation ${i}`); + } + process.exit(0); } From 5de92171a7bbf9c96a74e6d84691f8105bec7748 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 1 Dec 2019 10:28:05 +0100 Subject: [PATCH 7/7] use owned attributes where it's a better fit --- src/services/handlers.js | 2 +- src/services/notes.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/handlers.js b/src/services/handlers.js index 67bd9ed34..634844e08 100644 --- a/src/services/handlers.js +++ b/src/services/handlers.js @@ -81,7 +81,7 @@ eventService.subscribe(eventService.CHILD_NOTE_CREATED, async ({ parentNote, chi async function processInverseRelations(entityName, entity, handler) { if (entityName === 'attributes' && entity.type === 'relation') { const note = await entity.getNote(); - const attributes = (await note.getAttributes(entity.name)).filter(relation => relation.type === 'relation-definition'); + const attributes = (await note.getOwnedAttributes(entity.name)).filter(relation => relation.type === 'relation-definition'); for (const attribute of attributes) { const definition = attribute.value; diff --git a/src/services/notes.js b/src/services/notes.js index 8675666c1..344a60f62 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -514,7 +514,7 @@ async function duplicateNote(noteId, parentNoteId) { notePosition: origBranch ? origBranch.notePosition + 1 : null }).save(); - for (const attribute of await origNote.getAttributes()) { + for (const attribute of await origNote.getOwnedAttributes()) { const attr = new Attribute(attribute); attr.attributeId = undefined; // force creation of new attribute attr.noteId = newNote.noteId; @@ -530,7 +530,7 @@ async function duplicateNote(noteId, parentNoteId) { sqlInit.dbReady.then(() => { // first cleanup kickoff 5 minutes after startup - setTimeout(cls.wrap(eraseDeletedNotes), 0 * 60 * 1000); + setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000); setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000); });