Merge remote-tracking branch 'origin/optimize-tree'

This commit is contained in:
zadam 2020-12-10 22:54:16 +01:00
commit 0b6fa4ab06
17 changed files with 208 additions and 105 deletions

View File

@ -777,7 +777,7 @@ class Note extends Entity {
* @returns {NoteRevision[]} * @returns {NoteRevision[]}
*/ */
getRevisions() { getRevisions() {
return this.repository.getEntities("SELECT * FROM note_revisions WHERE noteId = ?", [this.noteId]); return this.repository.getEntities("SELECT * FROM note_revisions WHERE isErased = 0 AND noteId = ?", [this.noteId]);
} }
/** /**

View File

@ -51,6 +51,12 @@ const TPL = `
<label for="erase-notes-after-time-in-seconds">Erase notes after X seconds</label> <label for="erase-notes-after-time-in-seconds">Erase notes after X seconds</label>
<input class="form-control" id="erase-notes-after-time-in-seconds" type="number" min="0"> <input class="form-control" id="erase-notes-after-time-in-seconds" type="number" min="0">
</div> </div>
<p>You can also trigger erasing manually:</p>
<button id="erase-deleted-notes-now-button" class="btn">Erase deleted notes now</button>
<br/><br/>
</div> </div>
<div> <div>
@ -117,6 +123,13 @@ export default class ProtectedSessionOptions {
return false; return false;
}); });
this.$eraseDeletedNotesButton = $("#erase-deleted-notes-now-button");
this.$eraseDeletedNotesButton.on('click', () => {
server.post('notes/erase-deleted-notes-now').then(() => {
toastService.showMessage("Deleted notes have been erased.");
});
});
this.$protectedSessionTimeout = $("#protected-session-timeout-in-seconds"); this.$protectedSessionTimeout = $("#protected-session-timeout-in-seconds");
this.$protectedSessionTimeout.on('change', () => { this.$protectedSessionTimeout.on('change', () => {

View File

@ -75,14 +75,16 @@ class NoteShort {
this.parentToBranch[parentNoteId] = branchId; this.parentToBranch[parentNoteId] = branchId;
} }
addChild(childNoteId, branchId) { addChild(childNoteId, branchId, sort = true) {
if (!this.children.includes(childNoteId)) { if (!this.children.includes(childNoteId)) {
this.children.push(childNoteId); this.children.push(childNoteId);
} }
this.childToBranch[childNoteId] = branchId; this.childToBranch[childNoteId] = branchId;
this.sortChildren(); if (sort) {
this.sortChildren();
}
} }
sortChildren() { sortChildren() {

View File

@ -21,9 +21,6 @@ class TreeCache {
async loadInitialTree() { async loadInitialTree() {
const resp = await server.get('tree'); const resp = await server.get('tree');
// FIXME: we need to do this to cover for ascendants of template notes which are not loaded
await this.loadParents(resp, false);
// clear the cache only directly before adding new content which is important for e.g. switching to protected session // clear the cache only directly before adding new content which is important for e.g. switching to protected session
/** @type {Object.<string, NoteShort>} */ /** @type {Object.<string, NoteShort>} */
@ -44,50 +41,18 @@ class TreeCache {
async loadSubTree(subTreeNoteId) { async loadSubTree(subTreeNoteId) {
const resp = await server.get('tree?subTreeNoteId=' + subTreeNoteId); const resp = await server.get('tree?subTreeNoteId=' + subTreeNoteId);
await this.loadParents(resp, true);
this.addResp(resp); this.addResp(resp);
return this.notes[subTreeNoteId]; return this.notes[subTreeNoteId];
} }
async loadParents(resp, additiveLoad) {
const noteIds = new Set(resp.notes.map(note => note.noteId));
const missingNoteIds = [];
const existingNotes = additiveLoad ? this.notes : {};
for (const branch of resp.branches) {
if (!(branch.parentNoteId in existingNotes) && !noteIds.has(branch.parentNoteId) && branch.parentNoteId !== 'none') {
missingNoteIds.push(branch.parentNoteId);
}
}
for (const attr of resp.attributes) {
if (attr.type === 'relation' && attr.name === 'template' && !(attr.value in existingNotes) && !noteIds.has(attr.value)) {
missingNoteIds.push(attr.value);
}
if (!(attr.noteId in existingNotes) && !noteIds.has(attr.noteId)) {
missingNoteIds.push(attr.noteId);
}
}
if (missingNoteIds.length > 0) {
const newResp = await server.post('tree/load', { noteIds: missingNoteIds });
resp.notes = resp.notes.concat(newResp.notes);
resp.branches = resp.branches.concat(newResp.branches);
resp.attributes = resp.attributes.concat(newResp.attributes);
await this.loadParents(resp, additiveLoad);
}
}
addResp(resp) { addResp(resp) {
const noteRows = resp.notes; const noteRows = resp.notes;
const branchRows = resp.branches; const branchRows = resp.branches;
const attributeRows = resp.attributes; const attributeRows = resp.attributes;
const noteIdsToSort = new Set();
for (const noteRow of noteRows) { for (const noteRow of noteRows) {
const {noteId} = noteRow; const {noteId} = noteRow;
@ -154,7 +119,9 @@ class TreeCache {
const parentNote = this.notes[branch.parentNoteId]; const parentNote = this.notes[branch.parentNoteId];
if (parentNote) { if (parentNote) {
parentNote.addChild(branch.noteId, branch.branchId); parentNote.addChild(branch.noteId, branch.branchId, false);
noteIdsToSort.add(parentNote.noteId);
} }
} }
@ -179,6 +146,11 @@ class TreeCache {
} }
} }
} }
// sort all of them at once, this avoids repeated sorts (#1480)
for (const noteId of noteIdsToSort) {
this.notes[noteId].sortChildren();
}
} }
async reloadNotes(noteIds) { async reloadNotes(noteIds) {
@ -190,7 +162,6 @@ class TreeCache {
const resp = await server.post('tree/load', { noteIds }); const resp = await server.post('tree/load', { noteIds });
await this.loadParents(resp, true);
this.addResp(resp); this.addResp(resp);
const searchNoteIds = []; const searchNoteIds = [];

View File

@ -249,13 +249,22 @@ export default class NoteDetailWidget extends TabAwareWidget {
this.$widget.find('.note-detail-printable:visible').printThis({ this.$widget.find('.note-detail-printable:visible').printThis({
header: $("<h2>").text(this.note && this.note.title).prop('outerHTML'), header: $("<h2>").text(this.note && this.note.title).prop('outerHTML'),
footer: "<script>document.body.className += ' ck-content printed-content';</script>", footer: `
<script src="libraries/katex/katex.min.js"></script>
<script src="libraries/katex/auto-render.min.js"></script>
<script>
document.body.className += ' ck-content printed-content';
renderMathInElement(document.body, {});
</script>
`,
importCSS: false, importCSS: false,
loadCSS: [ loadCSS: [
"libraries/codemirror/codemirror.css", "libraries/codemirror/codemirror.css",
"libraries/ckeditor/ckeditor-content.css", "libraries/ckeditor/ckeditor-content.css",
"libraries/ckeditor/ckeditor-content.css", "libraries/ckeditor/ckeditor-content.css",
"libraries/bootstrap/css/bootstrap.min.css", "libraries/bootstrap/css/bootstrap.min.css",
"libraries/katex/katex.min.css",
"stylesheets/print.css", "stylesheets/print.css",
"stylesheets/relation_map.css", "stylesheets/relation_map.css",
"stylesheets/themes.css" "stylesheets/themes.css"

View File

@ -193,6 +193,10 @@ function duplicateSubtree(req) {
return noteService.duplicateSubtree(noteId, parentNoteId); return noteService.duplicateSubtree(noteId, parentNoteId);
} }
function eraseDeletedNotesNow() {
noteService.eraseDeletedNotesNow();
}
module.exports = { module.exports = {
getNote, getNote,
updateNote, updateNote,
@ -204,5 +208,6 @@ module.exports = {
setNoteTypeMime, setNoteTypeMime,
getRelationMap, getRelationMap,
changeTitle, changeTitle,
duplicateSubtree duplicateSubtree,
eraseDeletedNotesNow
}; };

View File

@ -23,7 +23,8 @@ function getRecentChanges(req) {
note_revisions.dateCreated AS date note_revisions.dateCreated AS date
FROM FROM
note_revisions note_revisions
JOIN notes USING(noteId)`); JOIN notes USING(noteId)
WHERE note_revisions.isErased = 0`);
for (const noteRevision of noteRevisions) { for (const noteRevision of noteRevisions) {
if (noteCacheService.isInAncestor(noteRevision.noteId, ancestorNoteId)) { if (noteCacheService.isInAncestor(noteRevision.noteId, ancestorNoteId)) {

View File

@ -38,6 +38,8 @@ function saveSyncSeed(req) {
}] }]
} }
log.info("Saved sync seed.");
sqlInit.createDatabaseForSync(options); sqlInit.createDatabaseForSync(options);
} }

View File

@ -5,14 +5,15 @@ const optionService = require('../../services/options');
const treeService = require('../../services/tree'); const treeService = require('../../services/tree');
function getNotesAndBranchesAndAttributes(noteIds) { function getNotesAndBranchesAndAttributes(noteIds) {
noteIds = Array.from(new Set(noteIds)); const notes = treeService.getNotesIncludingAscendants(noteIds);
const notes = treeService.getNotes(noteIds);
noteIds = notes.map(note => note.noteId); noteIds = new Set(notes.map(note => note.noteId));
sql.fillNoteIdList(noteIds);
// joining child note to filter out not completely synchronised notes which would then cause errors later // joining child note to filter out not completely synchronised notes which would then cause errors later
// cannot do that with parent because of root note's 'none' parent // cannot do that with parent because of root note's 'none' parent
const branches = sql.getManyRows(` const branches = sql.getRows(`
SELECT SELECT
branches.branchId, branches.branchId,
branches.noteId, branches.noteId,
@ -20,28 +21,45 @@ function getNotesAndBranchesAndAttributes(noteIds) {
branches.notePosition, branches.notePosition,
branches.prefix, branches.prefix,
branches.isExpanded branches.isExpanded
FROM branches FROM param_list
JOIN branches ON param_list.paramId = branches.noteId OR param_list.paramId = branches.parentNoteId
JOIN notes AS child ON child.noteId = branches.noteId JOIN notes AS child ON child.noteId = branches.noteId
WHERE branches.isDeleted = 0 WHERE branches.isDeleted = 0`);
AND (branches.noteId IN (???) OR parentNoteId IN (???))`, noteIds);
const attributes = sql.getRows(`
SELECT
attributes.attributeId,
attributes.noteId,
attributes.type,
attributes.name,
attributes.value,
attributes.position,
attributes.isInheritable
FROM param_list
JOIN attributes ON attributes.noteId = param_list.paramId
OR (attributes.type = 'relation' AND attributes.value = param_list.paramId)
WHERE attributes.isDeleted = 0`);
// we don't really care about the direction of the relation
const missingTemplateNoteIds = attributes
.filter(attr => attr.type === 'relation'
&& attr.name === 'template'
&& !noteIds.has(attr.value))
.map(attr => attr.value);
if (missingTemplateNoteIds.length > 0) {
const templateData = getNotesAndBranchesAndAttributes(missingTemplateNoteIds);
// there are going to be duplicates with simple concatenation, however:
// 1) shouldn't matter for the frontend which will update the entity twice
// 2) there shouldn't be many duplicates. There isn't that many templates
addArrays(notes, templateData.notes);
addArrays(branches, templateData.branches);
addArrays(attributes, templateData.attributes);
}
// sorting in memory is faster // sorting in memory is faster
branches.sort((a, b) => a.notePosition - b.notePosition < 0 ? -1 : 1); branches.sort((a, b) => a.notePosition - b.notePosition < 0 ? -1 : 1);
const attributes = sql.getManyRows(`
SELECT
attributeId,
noteId,
type,
name,
value,
position,
isInheritable
FROM attributes
WHERE isDeleted = 0
AND (noteId IN (???) OR (type = 'relation' AND value IN (???)))`, noteIds);
// sorting in memory is faster
attributes.sort((a, b) => a.position - b.position < 0 ? -1 : 1); attributes.sort((a, b) => a.position - b.position < 0 ? -1 : 1);
return { return {
@ -51,6 +69,16 @@ function getNotesAndBranchesAndAttributes(noteIds) {
}; };
} }
// should be fast based on https://stackoverflow.com/a/64826145/944162
// in this case it is assumed that target is potentially much larger than elementsToAdd
function addArrays(target, elementsToAdd) {
while (elementsToAdd.length) {
target.push(elementsToAdd.shift());
}
return target;
}
function getTree(req) { function getTree(req) {
const subTreeNoteId = req.query.subTreeNoteId || 'root'; const subTreeNoteId = req.query.subTreeNoteId || 'root';
@ -64,25 +92,8 @@ function getTree(req) {
JOIN treeWithDescendants ON branches.parentNoteId = treeWithDescendants.noteId JOIN treeWithDescendants ON branches.parentNoteId = treeWithDescendants.noteId
WHERE treeWithDescendants.isExpanded = 1 WHERE treeWithDescendants.isExpanded = 1
AND branches.isDeleted = 0 AND branches.isDeleted = 0
),
treeWithDescendantsAndAscendants AS (
SELECT noteId FROM treeWithDescendants
UNION
SELECT branches.parentNoteId FROM branches
JOIN treeWithDescendantsAndAscendants ON branches.noteId = treeWithDescendantsAndAscendants.noteId
WHERE branches.isDeleted = 0
AND branches.parentNoteId != ?
),
treeWithDescendantsAscendantsAndTemplates AS (
SELECT noteId FROM treeWithDescendantsAndAscendants
UNION
SELECT attributes.value FROM attributes
JOIN treeWithDescendantsAscendantsAndTemplates ON attributes.noteId = treeWithDescendantsAscendantsAndTemplates.noteId
WHERE attributes.isDeleted = 0
AND attributes.type = 'relation'
AND attributes.name = 'template'
) )
SELECT noteId FROM treeWithDescendantsAscendantsAndTemplates`, [subTreeNoteId, subTreeNoteId]); SELECT noteId FROM treeWithDescendants`, [subTreeNoteId]);
noteIds.push(subTreeNoteId); noteIds.push(subTreeNoteId);

View File

@ -154,6 +154,7 @@ function register(app) {
route(GET, '/api/notes/:noteId/revisions/:noteRevisionId/download', [auth.checkApiAuthOrElectron], noteRevisionsApiRoute.downloadNoteRevision); route(GET, '/api/notes/:noteId/revisions/:noteRevisionId/download', [auth.checkApiAuthOrElectron], noteRevisionsApiRoute.downloadNoteRevision);
apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision); apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision);
apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap); apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap);
apiRoute(POST, '/api/notes/erase-deleted-notes-now', notesApiRoute.eraseDeletedNotesNow);
apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle); apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle);
apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree); apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree);

View File

@ -651,7 +651,7 @@ class ConsistencyChecks {
// root branch should always be expanded // root branch should always be expanded
sql.execute("UPDATE branches SET isExpanded = 1 WHERE branchId = 'root'"); sql.execute("UPDATE branches SET isExpanded = 1 WHERE branchId = 'root'");
if (this.unrecoveredConsistencyErrors) { if (!this.unrecoveredConsistencyErrors) {
// we run this only if basic checks passed since this assumes basic data consistency // we run this only if basic checks passed since this assumes basic data consistency
this.checkTreeCycles(); this.checkTreeCycles();

View File

@ -70,6 +70,10 @@ eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) =>
if (templateNoteContent) { if (templateNoteContent) {
note.setContent(templateNoteContent); note.setContent(templateNoteContent);
} }
note.type = templateNote.type;
note.mime = templateNote.mime;
note.save();
} }
noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId); noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId);

View File

@ -2,6 +2,7 @@
const NoteRevision = require('../entities/note_revision'); const NoteRevision = require('../entities/note_revision');
const dateUtils = require('../services/date_utils'); const dateUtils = require('../services/date_utils');
const log = require('../services/log');
/** /**
* @param {Note} note * @param {Note} note
@ -9,14 +10,21 @@ const dateUtils = require('../services/date_utils');
function protectNoteRevisions(note) { function protectNoteRevisions(note) {
for (const revision of note.getRevisions()) { for (const revision of note.getRevisions()) {
if (note.isProtected !== revision.isProtected) { if (note.isProtected !== revision.isProtected) {
const content = revision.getContent(); try {
const content = revision.getContent();
revision.isProtected = note.isProtected; revision.isProtected = note.isProtected;
// this will force de/encryption // this will force de/encryption
revision.setContent(content); revision.setContent(content);
revision.save(); revision.save();
}
catch (e) {
log.error("Could not un/protect note revision ID = " + revision.noteRevisionId);
throw e;
}
} }
} }
} }

View File

@ -183,18 +183,25 @@ function protectNoteRecursively(note, protect, includingSubTree, taskContext) {
} }
function protectNote(note, protect) { function protectNote(note, protect) {
if (protect !== note.isProtected) { try {
const content = note.getContent(); if (protect !== note.isProtected) {
const content = note.getContent();
note.isProtected = protect; note.isProtected = protect;
// this will force de/encryption // this will force de/encryption
note.setContent(content); note.setContent(content);
note.save(); note.save();
}
noteRevisionService.protectNoteRevisions(note);
} }
catch (e) {
log.error("Could not un/protect note ID = " + note.noteId);
noteRevisionService.protectNoteRevisions(note); throw e;
}
} }
function findImageLinks(content, foundLinks) { function findImageLinks(content, foundLinks) {
@ -459,7 +466,7 @@ function saveNoteRevision(note) {
const revisionCutoff = dateUtils.utcDateStr(new Date(now.getTime() - noteRevisionSnapshotTimeInterval * 1000)); const revisionCutoff = dateUtils.utcDateStr(new Date(now.getTime() - noteRevisionSnapshotTimeInterval * 1000));
const existingNoteRevisionId = sql.getValue( const existingNoteRevisionId = sql.getValue(
"SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND utcDateCreated >= ?", [note.noteId, revisionCutoff]); "SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND isErased = 0 AND utcDateCreated >= ?", [note.noteId, revisionCutoff]);
const msSinceDateCreated = now.getTime() - dateUtils.parseDateTime(note.utcDateCreated).getTime(); const msSinceDateCreated = now.getTime() - dateUtils.parseDateTime(note.utcDateCreated).getTime();
@ -666,8 +673,10 @@ function scanForLinks(note) {
} }
} }
function eraseDeletedNotes() { function eraseDeletedNotes(eraseNotesAfterTimeInSeconds = null) {
const eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds'); if (eraseNotesAfterTimeInSeconds === null) {
eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
}
const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000); const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000);
@ -717,6 +726,10 @@ function eraseDeletedNotes() {
log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`); log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
} }
function eraseDeletedNotesNow() {
eraseDeletedNotes(0);
}
// do a replace in str - all keys should be replaced by the corresponding values // do a replace in str - all keys should be replaced by the corresponding values
function replaceByMap(str, mapObj) { function replaceByMap(str, mapObj) {
const re = new RegExp(Object.keys(mapObj).join("|"),"g"); const re = new RegExp(Object.keys(mapObj).join("|"),"g");
@ -823,9 +836,9 @@ function getNoteIdMapping(origNote) {
sqlInit.dbReady.then(() => { sqlInit.dbReady.then(() => {
// first cleanup kickoff 5 minutes after startup // first cleanup kickoff 5 minutes after startup
setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000); setTimeout(cls.wrap(() => eraseDeletedNotes()), 5 * 60 * 1000);
setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000); setInterval(cls.wrap(() => eraseDeletedNotes()), 4 * 3600 * 1000);
}); });
module.exports = { module.exports = {
@ -839,5 +852,6 @@ module.exports = {
duplicateSubtree, duplicateSubtree,
duplicateSubtreeWithoutRoot, duplicateSubtreeWithoutRoot,
getUndeletedParentBranches, getUndeletedParentBranches,
triggerNoteTitleChanged triggerNoteTitleChanged,
eraseDeletedNotesNow
}; };

View File

@ -236,6 +236,29 @@ function transactional(func) {
return ret; return ret;
} }
function fillNoteIdList(noteIds, truncate = true) {
if (noteIds.length === 0) {
return;
}
if (truncate) {
execute("DELETE FROM param_list");
}
noteIds = Array.from(new Set(noteIds));
if (noteIds.length > 30000) {
fillNoteIdList(noteIds.slice(30000), false);
noteIds = noteIds.slice(0, 30000);
}
// doing it manually to avoid this showing up on the sloq query list
const s = stmt(`INSERT INTO param_list VALUES ` + noteIds.map(noteId => `(?)`).join(','), noteIds);
s.run(noteIds);
}
module.exports = { module.exports = {
dbConnection, dbConnection,
insert, insert,
@ -253,5 +276,6 @@ module.exports = {
executeMany, executeMany,
executeScript, executeScript,
transactional, transactional,
upsert upsert,
fillNoteIdList
}; };

View File

@ -40,6 +40,8 @@ async function initDbConnection() {
require('./options_init').initStartupOptions(); require('./options_init').initStartupOptions();
sql.execute('CREATE TEMP TABLE "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)');
dbReady.resolve(); dbReady.resolve();
} }

View File

@ -6,6 +6,41 @@ const Branch = require('../entities/branch');
const entityChangesService = require('./entity_changes.js'); const entityChangesService = require('./entity_changes.js');
const protectedSessionService = require('./protected_session'); const protectedSessionService = require('./protected_session');
function getNotesIncludingAscendants(noteIds) {
noteIds = Array.from(new Set(noteIds));
sql.fillNoteIdList(noteIds);
// we return also deleted notes which have been specifically asked for
const notes = sql.getRows(`
WITH RECURSIVE
treeWithAscendants AS (
SELECT paramId AS noteId FROM param_list
UNION
SELECT branches.parentNoteId FROM branches
JOIN treeWithAscendants ON branches.noteId = treeWithAscendants.noteId
WHERE branches.isDeleted = 0
)
SELECT
noteId,
title,
isProtected,
type,
mime,
isDeleted
FROM notes
JOIN treeWithAscendants USING(noteId)`);
protectedSessionService.decryptNotes(notes);
notes.forEach(note => {
note.isProtected = !!note.isProtected
});
return notes;
}
function getNotes(noteIds) { function getNotes(noteIds) {
// we return also deleted notes which have been specifically asked for // we return also deleted notes which have been specifically asked for
const notes = sql.getManyRows(` const notes = sql.getManyRows(`
@ -190,6 +225,7 @@ function setNoteToParent(noteId, prefix, parentNoteId) {
module.exports = { module.exports = {
getNotes, getNotes,
getNotesIncludingAscendants,
validateParentChild, validateParentChild,
sortNotesAlphabetically, sortNotesAlphabetically,
setNoteToParent setNoteToParent