From abfc64af9588cb3b510e38938c1cc8faa38005b8 Mon Sep 17 00:00:00 2001 From: azivner Date: Tue, 3 Apr 2018 22:15:28 -0400 Subject: [PATCH] script to generate large documents, closes #55 --- package-lock.json | 28 ++++++- package.json | 1 + src/entities/branch.js | 8 +- .../javascripts/dialogs/jump_to_note.js | 2 +- .../javascripts/services/autocomplete.js | 4 + src/routes/api/branches.js | 8 +- src/routes/api/cloning.js | 50 +------------ src/routes/api/import.js | 6 +- src/routes/api/search.js | 4 +- src/services/cloning.js | 56 ++++++++++++++ src/services/consistency_checks.js | 8 +- src/services/notes.js | 4 +- src/services/sql_init.js | 23 +++--- src/services/sync_update.js | 2 +- src/tools/generate_document.js | 74 +++++++++++++++++++ 15 files changed, 203 insertions(+), 75 deletions(-) create mode 100644 src/services/cloning.js create mode 100644 src/tools/generate_document.js diff --git a/package-lock.json b/package-lock.json index fa9c22cbd..e2845d44c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "trilium", - "version": "0.9.1-beta", + "version": "0.9.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -7121,6 +7121,32 @@ "js-tokens": "3.0.2" } }, + "lorem-ipsum": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lorem-ipsum/-/lorem-ipsum-1.0.4.tgz", + "integrity": "sha1-MLcqOx4ZH1UGKvjH36spGuT72RI=", + "dev": true, + "requires": { + "optimist": "0.3.7" + }, + "dependencies": { + "optimist": { + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.7.tgz", + "integrity": "sha1-yQlBrVnkJzMokjB00s8ufLxuwNk=", + "dev": true, + "requires": { + "wordwrap": "0.0.3" + } + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "dev": true + } + } + }, "loud-rejection": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", diff --git a/package.json b/package.json index 494765c8e..cce6f0d0c 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "electron-compile": "^6.4.2", "electron-packager": "^11.1.0", "electron-prebuilt-compile": "2.0.0-beta.5", + "lorem-ipsum": "^1.0.4", "tape": "^4.9.0", "xo": "^0.18.0" }, diff --git a/src/entities/branch.js b/src/entities/branch.js index b17b68c79..463786c29 100644 --- a/src/entities/branch.js +++ b/src/entities/branch.js @@ -3,6 +3,7 @@ const Entity = require('./entity'); const dateUtils = require('../services/date_utils'); const repository = require('../services/repository'); +const sql = require('../services/sql'); class Branch extends Entity { static get tableName() { return "branches"; } @@ -12,9 +13,14 @@ class Branch extends Entity { return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]); } - beforeSaving() { + async beforeSaving() { super.beforeSaving(); + if (this.notePosition === undefined) { + const maxNotePos = await sql.getValue('SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0', [this.parentNoteId]); + this.notePosition = maxNotePos === null ? 0 : maxNotePos + 1; + } + if (!this.isDeleted) { this.isDeleted = false; } diff --git a/src/public/javascripts/dialogs/jump_to_note.js b/src/public/javascripts/dialogs/jump_to_note.js index 50f2f5e22..0b8fcd046 100644 --- a/src/public/javascripts/dialogs/jump_to_note.js +++ b/src/public/javascripts/dialogs/jump_to_note.js @@ -19,7 +19,7 @@ async function showDialog() { await $autoComplete.autocomplete({ source: await utils.stopWatch("building autocomplete", autocompleteService.getAutocompleteItems), - minLength: 0 + minLength: 1 }); } diff --git a/src/public/javascripts/services/autocomplete.js b/src/public/javascripts/services/autocomplete.js index cea699667..7054c14d7 100644 --- a/src/public/javascripts/services/autocomplete.js +++ b/src/public/javascripts/services/autocomplete.js @@ -46,6 +46,10 @@ async function getAutocompleteItems(parentNoteId, notePath, titlePath) { } } + if (parentNoteId === 'root') { + console.log(`Generated ${autocompleteItems.length} autocomplete items`); + } + return autocompleteItems; } diff --git a/src/routes/api/branches.js b/src/routes/api/branches.js index 7c09ce613..67c88659a 100644 --- a/src/routes/api/branches.js +++ b/src/routes/api/branches.js @@ -8,8 +8,8 @@ const notes = require('../../services/notes'); const repository = require('../../services/repository'); /** - * Code in this file deals with moving and cloning note tree rows. Relationship between note and parent note is unique - * for not deleted note trees. There may be multiple deleted note-parent note relationships. + * Code in this file deals with moving and cloning branches. Relationship between note and parent note is unique + * for not deleted branches. There may be multiple deleted note-parent note relationships. */ async function moveBranchToParent(req) { @@ -49,7 +49,7 @@ async function moveBranchBeforeNote(req) { } // we don't change dateModified so other changes are prioritized in case of conflict - // also we would have to sync all those modified note trees otherwise hash checks would fail + // also we would have to sync all those modified branches otherwise hash checks would fail await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition >= ? AND isDeleted = 0", [beforeNote.parentNoteId, beforeNote.notePosition]); @@ -77,7 +77,7 @@ async function moveBranchAfterNote(req) { } // we don't change dateModified so other changes are prioritized in case of conflict - // also we would have to sync all those modified note trees otherwise hash checks would fail + // also we would have to sync all those modified branches otherwise hash checks would fail await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0", [afterNote.parentNoteId, afterNote.notePosition]); diff --git a/src/routes/api/cloning.js b/src/routes/api/cloning.js index 9c9626bb2..46ea6452a 100644 --- a/src/routes/api/cloning.js +++ b/src/routes/api/cloning.js @@ -1,64 +1,20 @@ "use strict"; -const sql = require('../../services/sql'); -const syncTable = require('../../services/sync_table'); -const tree = require('../../services/tree'); -const Branch = require('../../entities/branch'); +const cloningService = require('../../services/cloning'); async function cloneNoteToParent(req) { const noteId = req.params.noteId; const parentNoteId = req.params.parentNoteId; const prefix = req.body.prefix; - const validationResult = await tree.validateParentChild(parentNoteId, noteId); - - if (!validationResult.success) { - return validationResult; - } - - const maxNotePos = await sql.getValue('SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0', [parentNoteId]); - const newNotePos = maxNotePos === null ? 0 : maxNotePos + 1; - - const branch = await new Branch({ - noteId: noteId, - parentNoteId: parentNoteId, - prefix: prefix, - notePosition: newNotePos, - isExpanded: 0 - }).save(); - - await sql.execute("UPDATE branches SET isExpanded = 1 WHERE noteId = ?", [parentNoteId]); - - return { success: true }; + return await cloningService.cloneNoteToParent(noteId, parentNoteId, prefix); } async function cloneNoteAfter(req) { const noteId = req.params.noteId; const afterBranchId = req.params.afterBranchId; - const afterNote = await tree.getBranch(afterBranchId); - - const validationResult = await tree.validateParentChild(afterNote.parentNoteId, noteId); - - if (!validationResult.result) { - return validationResult; - } - - // we don't change dateModified so other changes are prioritized in case of conflict - // also we would have to sync all those modified note trees otherwise hash checks would fail - await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0", - [afterNote.parentNoteId, afterNote.notePosition]); - - await syncTable.addNoteReorderingSync(afterNote.parentNoteId); - - const branch = await new Branch({ - noteId: noteId, - parentNoteId: afterNote.parentNoteId, - notePosition: afterNote.notePosition + 1, - isExpanded: 0 - }).save(); - - return { success: true }; + return await cloningService.cloneNoteAfter(noteId, afterBranchId); } module.exports = { diff --git a/src/routes/api/import.js b/src/routes/api/import.js index 71e4dedbf..a2db68927 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -110,17 +110,17 @@ async function importNotes(files, parentNoteId) { file.data = file.data.toString("UTF-8"); } - const noteId = await noteService.createNote(parentNoteId, file.meta.title, file.data, { + const {note} = await noteService.createNote(parentNoteId, file.meta.title, file.data, { type: file.meta.type, mime: file.meta.mime }); for (const label of file.meta.labels) { - await labelService.createLabel(noteId, label.name, label.value); + await labelService.createLabel(note.noteId, label.name, label.value); } if (file.children.length > 0) { - await importNotes(file.children, noteId); + await importNotes(file.children, note.noteId); } } } diff --git a/src/routes/api/search.js b/src/routes/api/search.js index da60d7edb..e03aa96ce 100644 --- a/src/routes/api/search.js +++ b/src/routes/api/search.js @@ -20,13 +20,13 @@ async function saveSearchToNote(req) { searchString: req.params.searchString }; - const noteId = await noteService.createNote('root', 'Search note', noteContent, { + const {note} = await noteService.createNote('root', 'Search note', noteContent, { json: true, type: 'search', mime: "application/json" }); - return { noteId }; + return { noteId: note.noteId }; } module.exports = { diff --git a/src/services/cloning.js b/src/services/cloning.js new file mode 100644 index 000000000..36cd6561a --- /dev/null +++ b/src/services/cloning.js @@ -0,0 +1,56 @@ +"use strict"; + +const sql = require('./sql'); +const syncTable = require('./sync_table'); +const tree = require('./tree'); +const Branch = require('../entities/branch'); + +async function cloneNoteToParent(noteId, parentNoteId, prefix) { + const validationResult = await tree.validateParentChild(parentNoteId, noteId); + + if (!validationResult.success) { + return validationResult; + } + + await new Branch({ + noteId: noteId, + parentNoteId: parentNoteId, + prefix: prefix, + isExpanded: 0 + }).save(); + + await sql.execute("UPDATE branches SET isExpanded = 1 WHERE noteId = ?", [parentNoteId]); + + return { success: true }; +} + +async function cloneNoteAfter(noteId, afterBranchId) { + const afterNote = await tree.getBranch(afterBranchId); + + const validationResult = await tree.validateParentChild(afterNote.parentNoteId, noteId); + + if (!validationResult.result) { + return validationResult; + } + + // we don't change dateModified so other changes are prioritized in case of conflict + // also we would have to sync all those modified branches otherwise hash checks would fail + await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0", + [afterNote.parentNoteId, afterNote.notePosition]); + + await syncTable.addNoteReorderingSync(afterNote.parentNoteId); + + await new Branch({ + noteId: noteId, + parentNoteId: afterNote.parentNoteId, + notePosition: afterNote.notePosition + 1, + isExpanded: 0 + }).save(); + + return { success: true }; +} + +module.exports = { + cloneNoteToParent, + cloneNoteAfter +}; \ No newline at end of file diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index b7c3aae4f..07722805f 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -105,7 +105,7 @@ async function runAllChecks() { LEFT JOIN notes USING(noteId) WHERE notes.noteId IS NULL`, - "Missing notes records for following note tree ID > note ID", errorList); + "Missing notes records for following branch ID > note ID", errorList); await runCheck(` SELECT @@ -116,7 +116,7 @@ async function runAllChecks() { WHERE notes.isDeleted = 1 AND branches.isDeleted = 0`, - "Note tree is not deleted even though main note is deleted for following note tree IDs", errorList); + "Note tree is not deleted even though main note is deleted for following branch IDs", errorList); await runCheck(` SELECT @@ -128,7 +128,7 @@ async function runAllChecks() { AND child.parentNoteId != 'root' AND (SELECT COUNT(*) FROM branches AS parent WHERE parent.noteId = child.parentNoteId AND parent.isDeleted = 0) = 0`, - "All parent note trees are deleted but child note tree is not for these child note tree IDs", errorList); + "All parent branches are deleted but child note tree is not for these child note tree IDs", errorList); // we do extra JOIN to eliminate orphan notes without note tree (which are reported separately) await runCheck(` @@ -140,7 +140,7 @@ async function runAllChecks() { WHERE (SELECT COUNT(*) FROM branches WHERE notes.noteId = branches.noteId AND branches.isDeleted = 0) = 0 AND notes.isDeleted = 0 - `, 'No undeleted note trees for note IDs', errorList); + `, 'No undeleted branches for note IDs', errorList); await runCheck(` SELECT diff --git a/src/services/notes.js b/src/services/notes.js index c4b2a3d80..eaa338df2 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -83,7 +83,7 @@ async function createNote(parentNoteId, title, content = "", extraOptions = {}) noteData.mime = "application/json"; } - const {note} = await createNewNote(parentNoteId, noteData); + const {note, branch} = await createNewNote(parentNoteId, noteData); if (extraOptions.labels) { for (const labelName in extraOptions.labels) { @@ -91,7 +91,7 @@ async function createNote(parentNoteId, title, content = "", extraOptions = {}) } } - return note.noteId; + return {note, branch}; } async function protectNoteRecursively(note, protect) { diff --git a/src/services/sql_init.js b/src/services/sql_init.js index 451214378..9fb5a94a3 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -11,6 +11,9 @@ async function createConnection() { return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise}); } +let schemaReadyResolve = null; +const schemaReady = new Promise((resolve, reject) => schemaReadyResolve = resolve); + let dbReadyResolve = null; const dbReady = new Promise((resolve, reject) => { cls.init(async () => { @@ -29,19 +32,20 @@ const dbReady = new Promise((resolve, reject) => { if (tableResults.length !== 1) { await createInitialDatabase(); } - else { - if (!await isUserInitialized()) { - log.info("Login/password not initialized. DB not ready."); - return; - } + schemaReadyResolve(); - if (!await isDbUpToDate()) { - return; - } + if (!await isUserInitialized()) { + log.info("Login/password not initialized. DB not ready."); - resolve(db); + return; } + + if (!await isDbUpToDate()) { + return; + } + + resolve(db); }); }); @@ -97,6 +101,7 @@ async function isUserInitialized() { module.exports = { dbReady, + schemaReady, isUserInitialized, setDbReadyAsResolved, isDbUpToDate diff --git a/src/services/sync_update.js b/src/services/sync_update.js index 6635f37f2..deab3c43e 100644 --- a/src/services/sync_update.js +++ b/src/services/sync_update.js @@ -37,7 +37,7 @@ async function updateBranch(entity, sourceId) { await syncTableService.addBranchSync(entity.branchId, sourceId); - log.info("Update/sync note tree " + entity.branchId); + log.info("Update/sync branch " + entity.branchId); } }); } diff --git a/src/tools/generate_document.js b/src/tools/generate_document.js new file mode 100644 index 000000000..7933615c8 --- /dev/null +++ b/src/tools/generate_document.js @@ -0,0 +1,74 @@ +const fs = require('fs'); +const dataDir = require('../services/data_dir'); + +fs.unlinkSync(dataDir.DOCUMENT_PATH); + +require('../entities/entity_constructor'); +const optionService = require('../services/options'); +const sqlInit = require('../services/sql_init'); +const myScryptService = require('../services/my_scrypt'); +const passwordEncryptionService = require('../services/password_encryption'); +const utils = require('../services/utils'); +const noteService = require('../services/notes'); +const cls = require('../services/cls'); +const cloningService = require('../services/cloning'); +const loremIpsum = require('lorem-ipsum'); + +async function setUserNamePassword() { + const username = "test"; + const password = "test"; + + await optionService.setOption('username', username); + + await optionService.setOption('passwordVerificationSalt', utils.randomSecureToken(32)); + await optionService.setOption('passwordDerivedKeySalt', utils.randomSecureToken(32)); + + const passwordVerificationKey = utils.toBase64(await myScryptService.getVerificationHash(password)); + await optionService.setOption('passwordVerificationHash', passwordVerificationKey); + + await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16)); + + sqlInit.setDbReadyAsResolved(); +} + +const noteCount = parseInt(process.argv[2]); +const notes = ['root']; + +function getRandomParentNoteId() { + const index = Math.floor(Math.random() * notes.length); + + return notes[index]; +} + +async function start() { + await setUserNamePassword(); + + for (let i = 0; i < noteCount; i++) { + const title = loremIpsum({ count: 1, units: 'sentences', sentenceLowerBound: 1, sentenceUpperBound: 10 }); + + const paragraphCount = Math.floor(Math.random() * Math.random() * 100); + const content = loremIpsum({ count: paragraphCount, units: 'paragraphs', sentenceLowerBound: 1, sentenceUpperBound: 15, + paragraphLowerBound: 3, paragraphUpperBound: 10, format: 'html' }); + + const {note} = await noteService.createNote(getRandomParentNoteId(), title, content); + + console.log(`Created note ${i}: ${title}`); + + notes.push(note.noteId); + } + + // we'll create clones for 20% of notes + for (let i = 0; i < (noteCount / 5); i++) { + const noteIdToClone = getRandomParentNoteId(); + const parentNoteId = getRandomParentNoteId(); + const prefix = Math.random() > 0.8 ? "prefix" : null; + + const result = await cloningService.cloneNoteToParent(noteIdToClone, parentNoteId, prefix); + + console.log(`Cloning ${i}:`, result.success ? "succeeded" : "FAILED"); + } + + process.exit(0); +} + +sqlInit.schemaReady.then(cls.wrap(start)); \ No newline at end of file