diff --git a/package-lock.json b/package-lock.json index 7d06c685e..392bdae31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "trilium", - "version": "0.45.2", + "version": "0.45.4", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -4838,6 +4838,11 @@ "type-check": "~0.3.2" } }, + "limiter": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", + "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" + }, "line-column": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/line-column/-/line-column-1.0.2.tgz", @@ -6913,6 +6918,22 @@ "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" }, + "stream-throttle": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/stream-throttle/-/stream-throttle-0.1.3.tgz", + "integrity": "sha1-rdV8jXzHOoFjDTHNVdOWHPr7qcM=", + "requires": { + "commander": "^2.2.0", + "limiter": "^1.0.5" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + } + } + }, "streamsearch": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", diff --git a/package.json b/package.json index 3f9d1b2c5..651dffb0b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "trilium", "productName": "Trilium Notes", "description": "Trilium Notes", - "version": "0.45.3", + "version": "0.45.4", "license": "AGPL-3.0-only", "main": "electron.js", "bin": { @@ -65,6 +65,7 @@ "semver": "7.3.2", "serve-favicon": "2.5.0", "session-file-store": "1.5.0", + "stream-throttle": "^0.1.3", "striptags": "3.1.1", "tmp": "^0.2.1", "turndown": "7.0.0", diff --git a/src/entities/attribute.js b/src/entities/attribute.js index 857531ffe..b5aecf6e1 100644 --- a/src/entities/attribute.js +++ b/src/entities/attribute.js @@ -34,6 +34,10 @@ class Attribute extends Entity { this.isInheritable = !!this.isInheritable; } + isAutoLink() { + return this.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(this.name); + } + /** * @returns {Note|null} */ diff --git a/src/public/app/services/frontend_script_api.js b/src/public/app/services/frontend_script_api.js index 3aa28b7e1..f589bef1a 100644 --- a/src/public/app/services/frontend_script_api.js +++ b/src/public/app/services/frontend_script_api.js @@ -274,7 +274,10 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain * * @method * @param {string} notePath (or noteId) - * @param {string} [noteTitle] - if not present we'll use note title + * @param {object} [params] + * @param {boolean} [params.showTooltip=true] - enable/disable tooltip on the link + * @param {boolean} [params.showNotePath=false] - show also whole note's path as part of the link + * @param {string} [title=] - custom link tile with note's title as default */ this.createNoteLink = linkService.createNoteLink; diff --git a/src/public/app/services/open.js b/src/public/app/services/open.js index 6054e223c..238e784fb 100644 --- a/src/public/app/services/open.js +++ b/src/public/app/services/open.js @@ -64,6 +64,7 @@ function getHost() { } export default { + download, downloadFileNote, openFileNote, downloadNoteRevision, diff --git a/src/public/app/services/ws.js b/src/public/app/services/ws.js index afcff6216..243570b1a 100644 --- a/src/public/app/services/ws.js +++ b/src/public/app/services/ws.js @@ -8,8 +8,6 @@ import options from "./options.js"; import treeCache from "./tree_cache.js"; import noteAttributeCache from "./note_attribute_cache.js"; -const $outstandingSyncsCount = $("#outstanding-syncs-count"); - const messageHandlers = []; let ws; @@ -64,8 +62,6 @@ async function handleMessage(event) { let syncRows = message.data; lastPingTs = Date.now(); - $outstandingSyncsCount.html(message.outstandingSyncs); - if (syncRows.length > 0) { logRows(syncRows); diff --git a/src/public/app/setup.js b/src/public/app/setup.js index 602a6fc7e..1c792deba 100644 --- a/src/public/app/setup.js +++ b/src/public/app/setup.js @@ -130,7 +130,7 @@ function SetupModel() { } async function checkOutstandingSyncs() { - const { stats, initialized } = await $.get('api/sync/stats'); + const { outstandingPullCount, initialized } = await $.get('api/sync/stats'); if (initialized) { if (utils.isElectron()) { @@ -143,9 +143,7 @@ async function checkOutstandingSyncs() { } } else { - const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls; - - $("#outstanding-syncs").html(totalOutstandingSyncs); + $("#outstanding-syncs").html(outstandingPullCount); } } diff --git a/src/public/app/widgets/global_menu.js b/src/public/app/widgets/global_menu.js index ea0c0bde8..3fc0dbfbe 100644 --- a/src/public/app/widgets/global_menu.js +++ b/src/public/app/widgets/global_menu.js @@ -41,7 +41,7 @@ const TPL = ` - Sync now (0) + Sync now diff --git a/src/routes/api/attributes.js b/src/routes/api/attributes.js index ca1c98230..ffbd6696b 100644 --- a/src/routes/api/attributes.js +++ b/src/routes/api/attributes.js @@ -28,8 +28,10 @@ function updateNoteAttribute(req) { || body.name !== attribute.name || (body.type === 'relation' && body.value !== attribute.value)) { + let newAttribute; + if (body.type !== 'relation' || !!body.value.trim()) { - const newAttribute = attribute.createClone(body.type, body.name, body.value); + newAttribute = attribute.createClone(body.type, body.name, body.value); newAttribute.save(); } @@ -37,7 +39,7 @@ function updateNoteAttribute(req) { attribute.save(); return { - attributeId: attribute.attributeId + attributeId: newAttribute ? newAttribute.attributeId : null }; } } @@ -54,6 +56,7 @@ function updateNoteAttribute(req) { if (attribute.type === 'label' || body.value.trim()) { attribute.value = body.value; + attribute.isDeleted = false; } else { // relations should never have empty target @@ -160,8 +163,10 @@ function updateNoteAttributes(req) { // all the remaining existing attributes are not defined anymore and should be deleted for (const toDeleteAttr of existingAttrs) { - toDeleteAttr.isDeleted = true; - toDeleteAttr.save(); + if (!toDeleteAttr.isAutoLink()) { + toDeleteAttr.isDeleted = true; + toDeleteAttr.save(); + } } } diff --git a/src/routes/api/recent_changes.js b/src/routes/api/recent_changes.js index 1a8e1a331..3f6b9c539 100644 --- a/src/routes/api/recent_changes.js +++ b/src/routes/api/recent_changes.js @@ -31,19 +31,36 @@ function getRecentChanges(req) { } } + // now we need to also collect date points not represented in note revisions: + // 1. creation for all notes (dateCreated) + // 2. deletion for deleted notes (dateModified) const notes = sql.getRows(` - SELECT - notes.noteId, - notes.isDeleted AS current_isDeleted, - notes.deleteId AS current_deleteId, - notes.isErased AS current_isErased, - notes.title AS current_title, - notes.isProtected AS current_isProtected, - notes.title, - notes.utcDateCreated AS utcDate, - notes.dateCreated AS date - FROM - notes`); + SELECT + notes.noteId, + notes.isDeleted AS current_isDeleted, + notes.deleteId AS current_deleteId, + notes.isErased AS current_isErased, + notes.title AS current_title, + notes.isProtected AS current_isProtected, + notes.title, + notes.utcDateCreated AS utcDate, + notes.dateCreated AS date + FROM + notes + UNION ALL + SELECT + notes.noteId, + notes.isDeleted AS current_isDeleted, + notes.deleteId AS current_deleteId, + notes.isErased AS current_isErased, + notes.title AS current_title, + notes.isProtected AS current_isProtected, + notes.title, + notes.utcDateModified AS utcDate, + notes.dateModified AS date + FROM + notes + WHERE notes.isDeleted = 1 AND notes.isErased = 0`); for (const note of notes) { if (noteCacheService.isInAncestor(note.noteId, ancestorNoteId)) { diff --git a/src/routes/api/sender.js b/src/routes/api/sender.js index 031c3fcf1..404eb4a9b 100644 --- a/src/routes/api/sender.js +++ b/src/routes/api/sender.js @@ -4,6 +4,7 @@ const imageType = require('image-type'); const imageService = require('../../services/image'); const dateNoteService = require('../../services/date_notes'); const noteService = require('../../services/notes'); +const attributeService = require('../../services/attributes'); function uploadImage(req) { const file = req.file; @@ -35,12 +36,10 @@ function saveNote(req) { mime: 'text/html' }); - if (req.body.label && req.body.label.trim()){ - let value; - if (req.body.labelValue && req.body.labelValue.trim()){ - value = req.body.labelValue; + if (req.body.labels) { + for (const {name, value} of req.body.labels) { + note.setLabel(attributeService.sanitizeAttributeName(name), value); } - note.setLabel(req.body.label, value); } return { diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js index b5d6590ea..5cbbda945 100644 --- a/src/routes/api/sync.js +++ b/src/routes/api/sync.js @@ -13,13 +13,13 @@ const dateUtils = require('../../services/date_utils'); const entityConstructor = require('../../entities/entity_constructor'); const utils = require('../../services/utils'); -function testSync() { +async function testSync() { try { if (!syncOptions.isSyncSetup()) { return { success: false, message: "Sync server host is not configured. Please configure sync first." }; } - syncService.login(); + await syncService.login(); // login was successful so we'll kick off sync now // this is important in case when sync server has been just initialized @@ -43,7 +43,7 @@ function getStats() { const stats = { initialized: optionService.getOption('initialized') === 'true', - stats: syncService.stats + outstandingPullCount: syncService.getOutstandingPullCount() }; log.info(`Returning sync stats: ${JSON.stringify(stats)}`); diff --git a/src/services/attributes.js b/src/services/attributes.js index d3e583fe2..d0383cbaa 100644 --- a/src/services/attributes.js +++ b/src/services/attributes.js @@ -2,7 +2,6 @@ const repository = require('./repository'); const sql = require('./sql'); -const utils = require('./utils'); const Attribute = require('../entities/attribute'); const ATTRIBUTE_TYPES = [ 'label', 'relation' ]; @@ -146,6 +145,20 @@ function getBuiltinAttributeNames() { ]); } +function sanitizeAttributeName(origName) { + let fixedName; + + if (origName === '') { + fixedName = "unnamed"; + } + else { + // any not allowed character should be replaced with underscore + fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_"); + } + + return fixedName; +} + module.exports = { getNotesWithLabel, getNotesWithLabels, @@ -156,5 +169,6 @@ module.exports = { getAttributeNames, isAttributeType, isAttributeDangerous, - getBuiltinAttributeNames + getBuiltinAttributeNames, + sanitizeAttributeName }; diff --git a/src/services/build.js b/src/services/build.js index ebfa1ae67..197f60332 100644 --- a/src/services/build.js +++ b/src/services/build.js @@ -1 +1 @@ -module.exports = { buildDate:"2020-11-10T22:54:39+01:00", buildRevision: "5157fc15e9f7fa960ee35685426868d5599933dc" }; +module.exports = { buildDate:"2020-11-12T22:15:23+01:00", buildRevision: "6c57b2220ff05059d7460369b195d281fcd9cbb6" }; diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index c36a02591..07c6d9679 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -11,6 +11,7 @@ const entityChangesService = require('./entity_changes.js'); const optionsService = require('./options'); const Branch = require('../entities/branch'); const dateUtils = require('./date_utils'); +const attributeService = require('./attributes'); class ConsistencyChecks { constructor(autoFix) { @@ -607,20 +608,10 @@ class ConsistencyChecks { findWronglyNamedAttributes() { const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`); - const attrNameMatcher = new RegExp("^[\\p{L}\\p{N}_:]+$", "u"); - for (const origName of attrNames) { - if (!attrNameMatcher.test(origName)) { - let fixedName; - - if (origName === '') { - fixedName = "unnamed"; - } - else { - // any not allowed character should be replaced with underscore - fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_"); - } + const fixedName = attributeService.sanitizeAttributeName(origName); + if (fixedName !== origName) { if (this.autoFix) { // there isn't a good way to update this: // - just SQL query will fix it in DB but not notify frontend (or other caches) that it has been fixed diff --git a/src/services/image.js b/src/services/image.js index 9420245e3..1fe432f5b 100644 --- a/src/services/image.js +++ b/src/services/image.js @@ -37,7 +37,7 @@ function getImageType(buffer) { } } else { - return imageType(buffer); + return imageType(buffer) || "jpg"; // optimistic JPG default } } diff --git a/src/services/import/enex.js b/src/services/import/enex.js index e80224af0..498fa3043 100644 --- a/src/services/import/enex.js +++ b/src/services/import/enex.js @@ -1,5 +1,6 @@ const sax = require("sax"); const stream = require('stream'); +const {Throttle} = require('stream-throttle'); const log = require("../log"); const utils = require("../utils"); const sql = require("../sql"); @@ -7,6 +8,7 @@ const noteService = require("../notes"); const imageService = require("../image"); const protectedSessionService = require('../protected_session'); const htmlSanitizer = require("../html_sanitizer"); +const attributeService = require("../attributes"); // date format is e.g. 20181121T193703Z function parseDate(text) { @@ -37,10 +39,6 @@ function importEnex(taskContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), })).note; - // we're persisting notes as we parse the document, but these are run asynchronously and may not be finished - // when we finish parsing. We use this to be sure that all saving has been finished before returning successfully. - const saveNotePromises = []; - function extractContent(content) { const openingNoteIndex = content.indexOf(''); @@ -105,9 +103,17 @@ function importEnex(taskContext, file, parentNote) { const previousTag = getPreviousTag(); if (previousTag === 'note-attributes') { + let labelName = currentTag; + + if (labelName === 'source-url') { + labelName = 'sourceUrl'; + } + + labelName = attributeService.sanitizeAttributeName(labelName); + note.attributes.push({ type: 'label', - name: currentTag, + name: labelName, value: text }); } @@ -149,7 +155,7 @@ function importEnex(taskContext, file, parentNote) { } else if (currentTag === 'tag') { note.attributes.push({ type: 'label', - name: text, + name: attributeService.sanitizeAttributeName(text), value: '' }) } @@ -227,6 +233,10 @@ function importEnex(taskContext, file, parentNote) { taskContext.increaseProgressCount(); for (const resource of resources) { + if (!resource.content) { + continue; + } + const hash = utils.md5(resource.content); const mediaRegex = new RegExp(`]*>`, 'g'); @@ -300,13 +310,7 @@ function importEnex(taskContext, file, parentNote) { updateDates(noteEntity.noteId, utcDateCreated, utcDateModified); } - saxStream.on("closetag", tag => { - path.pop(); - - if (tag === 'note') { - saveNotePromises.push(saveNote()); - } - }); + saxStream.on("closetag", tag => path.pop()); saxStream.on("opencdata", () => { //console.log("opencdata"); @@ -323,12 +327,15 @@ function importEnex(taskContext, file, parentNote) { return new Promise((resolve, reject) => { // resolve only when we parse the whole document AND saving of all notes have been finished - saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) }); + saxStream.on("end", () => resolve(rootNote)); const bufferStream = new stream.PassThrough(); bufferStream.end(file.buffer); - bufferStream.pipe(saxStream); + bufferStream + // rate limiting to improve responsiveness during / after import + .pipe(new Throttle({rate: 500000})) + .pipe(saxStream); }); } diff --git a/src/services/search/services/search.js b/src/services/search/services/search.js index ca2c9f178..eab2b32af 100644 --- a/src/services/search/services/search.js +++ b/src/services/search/services/search.js @@ -17,10 +17,14 @@ const utils = require('../../utils.js'); */ function findNotesWithExpression(expression) { const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()]; - const allNotes = (hoistedNote && hoistedNote.noteId !== 'root') + let allNotes = (hoistedNote && hoistedNote.noteId !== 'root') ? hoistedNote.subtreeNotes : Object.values(noteCache.notes); + // in the process of loading data sometimes we create "skeleton" note instances which are expected to be filled later + // in case of inconsistent data this might not work and search will then crash on these + allNotes = allNotes.filter(note => note.type !== undefined); + const allNoteSet = new NoteSet(allNotes); const searchContext = { diff --git a/src/services/sync.js b/src/services/sync.js index 30976a5e4..2e6bb06f7 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -20,10 +20,7 @@ const entityConstructor = require('../entities/entity_constructor'); let proxyToggle = true; -const stats = { - outstandingPushes: 0, - outstandingPulls: 0 -}; +let outstandingPullCount = 0; async function sync() { try { @@ -135,11 +132,7 @@ async function pullChanges(syncContext) { const pulledDate = Date.now(); - stats.outstandingPulls = resp.maxEntityChangeId - lastSyncedPull; - - if (stats.outstandingPulls < 0) { - stats.outstandingPulls = 0; - } + outstandingPullCount = Math.max(0, resp.maxEntityChangeId - lastSyncedPull); const {entityChanges} = resp; @@ -159,13 +152,13 @@ async function pullChanges(syncContext) { syncUpdateService.updateEntity(entityChange, entity, syncContext.sourceId); } - stats.outstandingPulls = resp.maxEntityChangeId - entityChange.id; + outstandingPullCount = Math.max(0, resp.maxEntityChangeId - entityChange.id); } setLastSyncedPull(entityChanges[entityChanges.length - 1].entityChange.id); }); - log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${stats.outstandingPulls} outstanding pulls`); + log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${outstandingPullCount} outstanding pulls`); } if (atLeastOnePullApplied) { @@ -359,31 +352,25 @@ function setLastSyncedPush(entityChangeId) { optionService.setOption('lastSyncedPush', entityChangeId); } -function updatePushStats() { - if (syncOptions.isSyncSetup()) { - const lastSyncedPush = optionService.getOption('lastSyncedPush'); - - stats.outstandingPushes = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE isSynced = 1 AND id > ?", [lastSyncedPush]); - } -} - function getMaxEntityChangeId() { return sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes'); } +function getOutstandingPullCount() { + return outstandingPullCount; +} + sqlInit.dbReady.then(() => { setInterval(cls.wrap(sync), 60000); // kickoff initial sync immediately setTimeout(cls.wrap(sync), 3000); - - setInterval(cls.wrap(updatePushStats), 1000); }); module.exports = { sync, login, getEntityChangesRecords, - stats, + getOutstandingPullCount, getMaxEntityChangeId }; diff --git a/src/services/ws.js b/src/services/ws.js index 400932d31..320a5e6f9 100644 --- a/src/services/ws.js +++ b/src/services/ws.js @@ -110,8 +110,7 @@ function sendPing(client, syncRows = []) { sendMessage(client, { type: 'sync', - data: syncRows, - outstandingSyncs: stats.outstandingPushes + stats.outstandingPulls + data: syncRows }); }