diff --git a/src/services/anonymization.js b/src/services/anonymization.js index e5c356eb5..b41179485 100644 --- a/src/services/anonymization.js +++ b/src/services/anonymization.js @@ -18,7 +18,10 @@ async function anonymize() { await db.run("UPDATE notes SET title = 'title'"); await db.run("UPDATE note_contents SET content = 'text'"); - await db.run("UPDATE note_revisions SET title = 'title', content = 'text'"); + await db.run("UPDATE note_revisions SET title = 'title'"); + await db.run("UPDATE note_revision_contents SET content = 'title'"); + await db.run("UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label'"); + await db.run("UPDATE attributes SET name = 'name' WHERE type = 'relation'"); await db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL"); await db.run(`UPDATE options SET value = 'anonymized' WHERE name IN ('documentSecret', 'encryptedDataKey', 'passwordVerificationHash', diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index a8f81bd31..ae9d5010c 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -626,12 +626,31 @@ async function runAllChecks() { return !unrecoveredConsistencyErrors; } +async function showEntityStat(name, query) { + const map = await sql.getMap(query); + + map[0] = map[0] || 0; + map[1] = map[1] || 0; + + log.info(`${name} deleted: ${map[1]}, not deleted ${map[0]}`); +} + +async function runDbDiagnostics() { + await showEntityStat("Notes", `SELECT isDeleted, count(noteId) FROM notes GROUP BY isDeleted`); + await showEntityStat("Note revisions", `SELECT isErased, count(noteRevisionId) FROM note_revisions GROUP BY isErased`); + await showEntityStat("Branches", `SELECT isDeleted, count(branchId) FROM branches GROUP BY isDeleted`); + await showEntityStat("Attributes", `SELECT isDeleted, count(attributeId) FROM attributes GROUP BY isDeleted`); + await showEntityStat("API tokens", `SELECT isDeleted, count(apiTokenId) FROM api_tokens GROUP BY isDeleted`); +} + async function runChecks() { let elapsedTimeMs; await syncMutexService.doExclusively(async () => { const startTime = new Date(); + await runDbDiagnostics(); + await runAllChecks(); elapsedTimeMs = Date.now() - startTime.getTime(); @@ -663,7 +682,7 @@ sqlInit.dbReady.then(() => { setInterval(cls.wrap(runChecks), 60 * 60 * 1000); // kickoff checks soon after startup (to not block the initial load) - setTimeout(cls.wrap(runChecks), 10 * 1000); + setTimeout(cls.wrap(runChecks), 20 * 1000); }); module.exports = {}; \ No newline at end of file diff --git a/src/services/handlers.js b/src/services/handlers.js index 67bd9ed34..634844e08 100644 --- a/src/services/handlers.js +++ b/src/services/handlers.js @@ -81,7 +81,7 @@ eventService.subscribe(eventService.CHILD_NOTE_CREATED, async ({ parentNote, chi async function processInverseRelations(entityName, entity, handler) { if (entityName === 'attributes' && entity.type === 'relation') { const note = await entity.getNote(); - const attributes = (await note.getAttributes(entity.name)).filter(relation => relation.type === 'relation-definition'); + const attributes = (await note.getOwnedAttributes(entity.name)).filter(relation => relation.type === 'relation-definition'); for (const attribute of attributes) { const definition = attribute.value; diff --git a/src/services/migration.js b/src/services/migration.js index 2b93f0064..4c82a36d6 100644 --- a/src/services/migration.js +++ b/src/services/migration.js @@ -43,9 +43,6 @@ async function migrate() { try { log.info("Attempting migration to version " + mig.dbVersion); - // needs to happen outside of the transaction (otherwise it's a NO-OP) - await sql.execute("PRAGMA foreign_keys = OFF"); - await sql.transactional(async () => { if (mig.type === 'sql') { const migrationSql = fs.readFileSync(resourceDir.MIGRATIONS_DIR + "/" + mig.file).toString('utf8'); @@ -76,10 +73,6 @@ async function migrate() { utils.crash(); } - finally { - // make sure foreign keys are enabled even if migration script disables them - await sql.execute("PRAGMA foreign_keys = ON"); - } } if (await sqlInit.isDbUpToDate()) { diff --git a/src/services/notes.js b/src/services/notes.js index eba8f5da1..effd51097 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -57,7 +57,7 @@ function deriveMime(type, mime) { } async function copyChildAttributes(parentNote, childNote) { - for (const attr of await parentNote.getAttributes()) { + for (const attr of await parentNote.getOwnedAttributes()) { if (attr.name.startsWith("child:")) { await new Attribute({ noteId: childNote.noteId, @@ -489,7 +489,7 @@ async function eraseDeletedNotes() { SET content = NULL, utcDateModified = '${utcNowDateTime}' WHERE noteRevisionId IN - (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN ((???)))`, noteIdsToErase); + (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN (???))`, noteIdsToErase); await sql.executeMany(` UPDATE note_revisions @@ -523,7 +523,7 @@ async function duplicateNote(noteId, parentNoteId) { notePosition: origBranch ? origBranch.notePosition + 1 : null }).save(); - for (const attribute of await origNote.getAttributes()) { + for (const attribute of await origNote.getOwnedAttributes()) { const attr = new Attribute(attribute); attr.attributeId = undefined; // force creation of new attribute attr.noteId = newNote.noteId; diff --git a/src/services/sql_init.js b/src/services/sql_init.js index 45f8272a1..e93ff5ad0 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -57,8 +57,6 @@ async function initDbConnection() { return; } - await sql.execute("PRAGMA foreign_keys = ON"); - const currentDbVersion = await getDbVersion(); if (currentDbVersion > appInfo.dbVersion) { @@ -175,9 +173,11 @@ async function isDbUpToDate() { } async function dbInitialized() { - await optionService.setOption('initialized', 'true'); + if (!await isDbInitialized()) { + await optionService.setOption('initialized', 'true'); - await initDbConnection(); + await initDbConnection(); + } } dbReady.then(async () => { diff --git a/src/tools/generate_document.js b/src/tools/generate_document.js index bf250ca77..2d0613755 100644 --- a/src/tools/generate_document.js +++ b/src/tools/generate_document.js @@ -1,48 +1,32 @@ -const fs = require('fs'); -const dataDir = require('../services/data_dir'); - -fs.unlinkSync(dataDir.DOCUMENT_PATH); +/** + * Usage: node src/tools/generate_document.js 1000 + * will create 1000 new notes and some clones into a current document.db + */ require('../entities/entity_constructor'); -const optionService = require('../services/options'); const sqlInit = require('../services/sql_init'); -const myScryptService = require('../services/my_scrypt'); -const passwordEncryptionService = require('../services/password_encryption'); -const utils = require('../services/utils'); const noteService = require('../services/notes'); +const attributeService = require('../services/attributes'); const cls = require('../services/cls'); const cloningService = require('../services/cloning'); -const loremIpsum = require('lorem-ipsum'); - -async function setUserNamePassword() { - const username = "test"; - const password = "test"; - - await optionService.setOption('username', username); - - await optionService.setOption('passwordVerificationSalt', utils.randomSecureToken(32)); - await optionService.setOption('passwordDerivedKeySalt', utils.randomSecureToken(32)); - - const passwordVerificationKey = utils.toBase64(await myScryptService.getVerificationHash(password)); - await optionService.setOption('passwordVerificationHash', passwordVerificationKey); - - await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16)); - - await sqlInit.initDbConnection(); -} +const loremIpsum = require('lorem-ipsum').loremIpsum; const noteCount = parseInt(process.argv[2]); + +if (!noteCount) { + console.error(`Please enter number of notes as program parameter.`); + process.exit(1); +} + const notes = ['root']; -function getRandomParentNoteId() { +function getRandomNoteId() { const index = Math.floor(Math.random() * notes.length); return notes[index]; } async function start() { - await setUserNamePassword(); - for (let i = 0; i < noteCount; i++) { const title = loremIpsum({ count: 1, units: 'sentences', sentenceLowerBound: 1, sentenceUpperBound: 10 }); @@ -51,7 +35,7 @@ async function start() { paragraphLowerBound: 3, paragraphUpperBound: 10, format: 'html' }); const {note} = await noteService.createNewNote({ - parentNoteId: getRandomParentNoteId(), + parentNoteId: getRandomNoteId(), title, content, type: 'text' @@ -62,10 +46,10 @@ async function start() { notes.push(note.noteId); } - // we'll create clones for 20% of notes - for (let i = 0; i < (noteCount / 50); i++) { - const noteIdToClone = getRandomParentNoteId(); - const parentNoteId = getRandomParentNoteId(); + // we'll create clones for 4% of notes + for (let i = 0; i < (noteCount / 25); i++) { + const noteIdToClone = getRandomNoteId(); + const parentNoteId = getRandomNoteId(); const prefix = Math.random() > 0.8 ? "prefix" : null; const result = await cloningService.cloneNoteToParent(noteIdToClone, parentNoteId, prefix); @@ -73,6 +57,30 @@ async function start() { console.log(`Cloning ${i}:`, result.success ? "succeeded" : "FAILED"); } + for (let i = 0; i < noteCount; i++) { + await attributeService.createAttribute({ + noteId: getRandomNoteId(), + type: 'label', + name: 'label', + value: 'value', + isInheritable: Math.random() > 0.1 // 10% are inheritable + }); + + console.log(`Creating label ${i}`); + } + + for (let i = 0; i < noteCount; i++) { + await attributeService.createAttribute({ + noteId: getRandomNoteId(), + type: 'relation', + name: 'relation', + value: getRandomNoteId(), + isInheritable: Math.random() > 0.1 // 10% are inheritable + }); + + console.log(`Creating relation ${i}`); + } + process.exit(0); }