Merge remote-tracking branch 'origin/stable'

# Conflicts:
#	src/routes/api/attributes.js
#	src/routes/api/sender.js

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.45.2",
+  "version": "0.45.4",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -4838,6 +4838,11 @@
         "type-check": "~0.3.2"
       }
     },
+    "limiter": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz",
+      "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA=="
+    },
     "line-column": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/line-column/-/line-column-1.0.2.tgz",
@@ -6913,6 +6918,22 @@
       "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
       "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
     },
+    "stream-throttle": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/stream-throttle/-/stream-throttle-0.1.3.tgz",
+      "integrity": "sha1-rdV8jXzHOoFjDTHNVdOWHPr7qcM=",
+      "requires": {
+        "commander": "^2.2.0",
+        "limiter": "^1.0.5"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "2.20.3",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+          "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
+        }
+      }
+    },
     "streamsearch": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",

package.json

@@ -2,7 +2,7 @@
   "name": "trilium",
   "productName": "Trilium Notes",
   "description": "Trilium Notes",
-  "version": "0.45.3",
+  "version": "0.45.4",
   "license": "AGPL-3.0-only",
   "main": "electron.js",
   "bin": {
@@ -65,6 +65,7 @@
     "semver": "7.3.2",
     "serve-favicon": "2.5.0",
     "session-file-store": "1.5.0",
+    "stream-throttle": "^0.1.3",
     "striptags": "3.1.1",
     "tmp": "^0.2.1",
     "turndown": "7.0.0",

src/entities/attribute.js

@@ -34,6 +34,10 @@ class Attribute extends Entity {
         this.isInheritable = !!this.isInheritable;
     }
 
+    isAutoLink() {
+        return this.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(this.name);
+    }
+
     /**
      * @returns {Note|null}
      */

src/public/app/services/frontend_script_api.js

@@ -274,7 +274,10 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
      *
      * @method
      * @param {string} notePath (or noteId)
-     * @param {string} [noteTitle] - if not present we'll use note title
+     * @param {object} [params]
+     * @param {boolean} [params.showTooltip=true] - enable/disable tooltip on the link
+     * @param {boolean} [params.showNotePath=false] - show also whole note's path as part of the link
+     * @param {string} [title=] - custom link tile with note's title as default
      */
     this.createNoteLink = linkService.createNoteLink;
 

src/public/app/services/open.js

@@ -64,6 +64,7 @@ function getHost() {
 }
 
 export default {
+    download,
     downloadFileNote,
     openFileNote,
     downloadNoteRevision,

src/public/app/services/ws.js

@@ -8,8 +8,6 @@ import options from "./options.js";
 import treeCache from "./tree_cache.js";
 import noteAttributeCache from "./note_attribute_cache.js";
 
-const $outstandingSyncsCount = $("#outstanding-syncs-count");
-
 const messageHandlers = [];
 
 let ws;
@@ -64,8 +62,6 @@ async function handleMessage(event) {
         let syncRows = message.data;
         lastPingTs = Date.now();
 
-        $outstandingSyncsCount.html(message.outstandingSyncs);
-
         if (syncRows.length > 0) {
             logRows(syncRows);
 

src/public/app/setup.js

@@ -130,7 +130,7 @@ function SetupModel() {
 }
 
 async function checkOutstandingSyncs() {
-    const { stats, initialized } = await $.get('api/sync/stats');
+    const { outstandingPullCount, initialized } = await $.get('api/sync/stats');
 
     if (initialized) {
         if (utils.isElectron()) {
@@ -143,9 +143,7 @@ async function checkOutstandingSyncs() {
         }
     }
     else {
-        const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls;
+        $("#outstanding-syncs").html(outstandingPullCount);
-
-        $("#outstanding-syncs").html(totalOutstandingSyncs);
     }
 }
 

src/public/app/widgets/global_menu.js

@@ -41,7 +41,7 @@ const TPL = `
 
             <a class="dropdown-item sync-now-button" title="Trigger sync">
                 <span class="bx bx-refresh"></span>
-                Sync now (<span id="outstanding-syncs-count">0</span>)
+                Sync now
             </a>
 
             <a class="dropdown-item" data-trigger-command="openNewWindow">

src/routes/api/attributes.js

@@ -28,8 +28,10 @@ function updateNoteAttribute(req) {
             || body.name !== attribute.name
             || (body.type === 'relation' && body.value !== attribute.value)) {
 
+            let newAttribute;
+
             if (body.type !== 'relation' || !!body.value.trim()) {
-                const newAttribute = attribute.createClone(body.type, body.name, body.value);
+                newAttribute = attribute.createClone(body.type, body.name, body.value);
                 newAttribute.save();
             }
 
@@ -37,7 +39,7 @@ function updateNoteAttribute(req) {
             attribute.save();
 
             return {
-                attributeId: attribute.attributeId
+                attributeId: newAttribute ? newAttribute.attributeId : null
             };
         }
     }
@@ -54,6 +56,7 @@ function updateNoteAttribute(req) {
 
     if (attribute.type === 'label' || body.value.trim()) {
         attribute.value = body.value;
+        attribute.isDeleted = false;
     }
     else {
         // relations should never have empty target
@@ -160,8 +163,10 @@ function updateNoteAttributes(req) {
 
     // all the remaining existing attributes are not defined anymore and should be deleted
     for (const toDeleteAttr of existingAttrs) {
-        toDeleteAttr.isDeleted = true;
+        if (!toDeleteAttr.isAutoLink()) {
-        toDeleteAttr.save();
+            toDeleteAttr.isDeleted = true;
+            toDeleteAttr.save();
+        }
     }
 }
 

src/routes/api/recent_changes.js

@@ -31,19 +31,36 @@ function getRecentChanges(req) {
         }
     }
 
+    // now we need to also collect date points not represented in note revisions:
+    // 1. creation for all notes (dateCreated)
+    // 2. deletion for deleted notes (dateModified)
     const notes = sql.getRows(`
-        SELECT
+            SELECT
-            notes.noteId,
+                notes.noteId,
-            notes.isDeleted AS current_isDeleted,
+                notes.isDeleted AS current_isDeleted,
-            notes.deleteId AS current_deleteId,
+                notes.deleteId AS current_deleteId,
-            notes.isErased AS current_isErased,
+                notes.isErased AS current_isErased,
-            notes.title AS current_title,
+                notes.title AS current_title,
-            notes.isProtected AS current_isProtected,
+                notes.isProtected AS current_isProtected,
-            notes.title,
+                notes.title,
-            notes.utcDateCreated AS utcDate,
+                notes.utcDateCreated AS utcDate,
-            notes.dateCreated AS date
+                notes.dateCreated AS date
-        FROM
+            FROM
-            notes`);
+                notes
+        UNION ALL
+            SELECT
+                notes.noteId,
+                notes.isDeleted AS current_isDeleted,
+                notes.deleteId AS current_deleteId,
+                notes.isErased AS current_isErased,
+                notes.title AS current_title,
+                notes.isProtected AS current_isProtected,
+                notes.title,
+                notes.utcDateModified AS utcDate,
+                notes.dateModified AS date
+            FROM
+                notes
+            WHERE notes.isDeleted = 1 AND notes.isErased = 0`);
 
     for (const note of notes) {
         if (noteCacheService.isInAncestor(note.noteId, ancestorNoteId)) {

src/routes/api/sender.js

@@ -4,6 +4,7 @@ const imageType = require('image-type');
 const imageService = require('../../services/image');
 const dateNoteService = require('../../services/date_notes');
 const noteService = require('../../services/notes');
+const attributeService = require('../../services/attributes');
 
 function uploadImage(req) {
     const file = req.file;
@@ -35,12 +36,10 @@ function saveNote(req) {
         mime: 'text/html'
     });
 
-    if (req.body.label && req.body.label.trim()){
+    if (req.body.labels) {
-        let value;
+        for (const {name, value} of req.body.labels) {
-        if (req.body.labelValue && req.body.labelValue.trim()){
+            note.setLabel(attributeService.sanitizeAttributeName(name), value);
-            value = req.body.labelValue;
         }
-        note.setLabel(req.body.label, value);
     }
 
     return {

src/routes/api/sync.js

@@ -13,13 +13,13 @@ const dateUtils = require('../../services/date_utils');
 const entityConstructor = require('../../entities/entity_constructor');
 const utils = require('../../services/utils');
 
-function testSync() {
+async function testSync() {
     try {
         if (!syncOptions.isSyncSetup()) {
             return { success: false, message: "Sync server host is not configured. Please configure sync first." };
         }
 
-        syncService.login();
+        await syncService.login();
 
         // login was successful so we'll kick off sync now
         // this is important in case when sync server has been just initialized
@@ -43,7 +43,7 @@ function getStats() {
 
     const stats = {
         initialized: optionService.getOption('initialized') === 'true',
-        stats: syncService.stats
+        outstandingPullCount: syncService.getOutstandingPullCount()
     };
 
     log.info(`Returning sync stats: ${JSON.stringify(stats)}`);

src/services/attributes.js

@@ -2,7 +2,6 @@
 
 const repository = require('./repository');
 const sql = require('./sql');
-const utils = require('./utils');
 const Attribute = require('../entities/attribute');
 
 const ATTRIBUTE_TYPES = [ 'label', 'relation' ];
@@ -146,6 +145,20 @@ function getBuiltinAttributeNames() {
         ]);
 }
 
+function sanitizeAttributeName(origName) {
+    let fixedName;
+
+    if (origName === '') {
+        fixedName = "unnamed";
+    }
+    else {
+        // any not allowed character should be replaced with underscore
+        fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
+    }
+
+    return fixedName;
+}
+
 module.exports = {
     getNotesWithLabel,
     getNotesWithLabels,
@@ -156,5 +169,6 @@ module.exports = {
     getAttributeNames,
     isAttributeType,
     isAttributeDangerous,
-    getBuiltinAttributeNames
+    getBuiltinAttributeNames,
+    sanitizeAttributeName
 };

src/services/build.js

@@ -1 +1 @@
-module.exports = { buildDate:"2020-11-10T22:54:39+01:00", buildRevision: "5157fc15e9f7fa960ee35685426868d5599933dc" };
+module.exports = { buildDate:"2020-11-12T22:15:23+01:00", buildRevision: "6c57b2220ff05059d7460369b195d281fcd9cbb6" };

src/services/consistency_checks.js

@@ -11,6 +11,7 @@ const entityChangesService = require('./entity_changes.js');
 const optionsService = require('./options');
 const Branch = require('../entities/branch');
 const dateUtils = require('./date_utils');
+const attributeService = require('./attributes');
 
 class ConsistencyChecks {
     constructor(autoFix) {
@@ -607,20 +608,10 @@ class ConsistencyChecks {
     findWronglyNamedAttributes() {
         const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`);
 
-        const attrNameMatcher = new RegExp("^[\\p{L}\\p{N}_:]+$", "u");
-
         for (const origName of attrNames) {
-            if (!attrNameMatcher.test(origName)) {
+            const fixedName = attributeService.sanitizeAttributeName(origName);
-                let fixedName;
-
-                if (origName === '') {
-                    fixedName = "unnamed";
-                }
-                else {
-                    // any not allowed character should be replaced with underscore
-                    fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
-                }
 
+            if (fixedName !== origName) {
                 if (this.autoFix) {
                     // there isn't a good way to update this:
                     // - just SQL query will fix it in DB but not notify frontend (or other caches) that it has been fixed

src/services/image.js

@@ -37,7 +37,7 @@ function getImageType(buffer) {
         }
     }
     else {
-        return imageType(buffer);
+        return imageType(buffer) || "jpg"; // optimistic JPG default
     }
 }
 

src/services/import/enex.js

@@ -1,5 +1,6 @@
 const sax = require("sax");
 const stream = require('stream');
+const {Throttle} = require('stream-throttle');
 const log = require("../log");
 const utils = require("../utils");
 const sql = require("../sql");
@@ -7,6 +8,7 @@ const noteService = require("../notes");
 const imageService = require("../image");
 const protectedSessionService = require('../protected_session');
 const htmlSanitizer = require("../html_sanitizer");
+const attributeService = require("../attributes");
 
 // date format is e.g. 20181121T193703Z
 function parseDate(text) {
@@ -37,10 +39,6 @@ function importEnex(taskContext, file, parentNote) {
         isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
     })).note;
 
-    // we're persisting notes as we parse the document, but these are run asynchronously and may not be finished
-    // when we finish parsing. We use this to be sure that all saving has been finished before returning successfully.
-    const saveNotePromises = [];
-
     function extractContent(content) {
         const openingNoteIndex = content.indexOf('<en-note>');
 
@@ -105,9 +103,17 @@ function importEnex(taskContext, file, parentNote) {
         const previousTag = getPreviousTag();
 
         if (previousTag === 'note-attributes') {
+            let labelName = currentTag;
+
+            if (labelName === 'source-url') {
+                labelName = 'sourceUrl';
+            }
+
+            labelName = attributeService.sanitizeAttributeName(labelName);
+
             note.attributes.push({
                 type: 'label',
-                name: currentTag,
+                name: labelName,
                 value: text
             });
         }
@@ -149,7 +155,7 @@ function importEnex(taskContext, file, parentNote) {
             } else if (currentTag === 'tag') {
                 note.attributes.push({
                     type: 'label',
-                    name: text,
+                    name: attributeService.sanitizeAttributeName(text),
                     value: ''
                 })
             }
@@ -227,6 +233,10 @@ function importEnex(taskContext, file, parentNote) {
         taskContext.increaseProgressCount();
 
         for (const resource of resources) {
+            if (!resource.content) {
+                continue;
+            }
+
             const hash = utils.md5(resource.content);
 
             const mediaRegex = new RegExp(`<en-media hash="${hash}"[^>]*>`, 'g');
@@ -300,13 +310,7 @@ function importEnex(taskContext, file, parentNote) {
         updateDates(noteEntity.noteId, utcDateCreated, utcDateModified);
     }
 
-    saxStream.on("closetag", tag => {
+    saxStream.on("closetag", tag => path.pop());
-        path.pop();
-
-        if (tag === 'note') {
-            saveNotePromises.push(saveNote());
-        }
-    });
 
     saxStream.on("opencdata", () => {
         //console.log("opencdata");
@@ -323,12 +327,15 @@ function importEnex(taskContext, file, parentNote) {
     return new Promise((resolve, reject) =>
     {
         // resolve only when we parse the whole document AND saving of all notes have been finished
-        saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) });
+        saxStream.on("end", () => resolve(rootNote));
 
         const bufferStream = new stream.PassThrough();
         bufferStream.end(file.buffer);
 
-        bufferStream.pipe(saxStream);
+        bufferStream
+            // rate limiting to improve responsiveness during / after import
+            .pipe(new Throttle({rate: 500000}))
+            .pipe(saxStream);
     });
 }
 

src/services/search/services/search.js

@@ -17,10 +17,14 @@ const utils = require('../../utils.js');
  */
 function findNotesWithExpression(expression) {
     const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()];
-    const allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
+    let allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
         ? hoistedNote.subtreeNotes
         : Object.values(noteCache.notes);
 
+    // in the process of loading data sometimes we create "skeleton" note instances which are expected to be filled later
+    // in case of inconsistent data this might not work and search will then crash on these
+    allNotes = allNotes.filter(note => note.type !== undefined);
+
     const allNoteSet = new NoteSet(allNotes);
 
     const searchContext = {

src/services/sync.js

@@ -20,10 +20,7 @@ const entityConstructor = require('../entities/entity_constructor');
 
 let proxyToggle = true;
 
-const stats = {
+let outstandingPullCount = 0;
-    outstandingPushes: 0,
-    outstandingPulls: 0
-};
 
 async function sync() {
     try {
@@ -135,11 +132,7 @@ async function pullChanges(syncContext) {
 
         const pulledDate = Date.now();
 
-        stats.outstandingPulls = resp.maxEntityChangeId - lastSyncedPull;
+        outstandingPullCount = Math.max(0, resp.maxEntityChangeId - lastSyncedPull);
-
-        if (stats.outstandingPulls < 0) {
-            stats.outstandingPulls = 0;
-        }
 
         const {entityChanges} = resp;
 
@@ -159,13 +152,13 @@ async function pullChanges(syncContext) {
                     syncUpdateService.updateEntity(entityChange, entity, syncContext.sourceId);
                 }
 
-                stats.outstandingPulls = resp.maxEntityChangeId - entityChange.id;
+                outstandingPullCount = Math.max(0, resp.maxEntityChangeId - entityChange.id);
             }
 
             setLastSyncedPull(entityChanges[entityChanges.length - 1].entityChange.id);
         });
 
-        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${stats.outstandingPulls} outstanding pulls`);
+        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${outstandingPullCount} outstanding pulls`);
     }
 
     if (atLeastOnePullApplied) {
@@ -359,31 +352,25 @@ function setLastSyncedPush(entityChangeId) {
     optionService.setOption('lastSyncedPush', entityChangeId);
 }
 
-function updatePushStats() {
-    if (syncOptions.isSyncSetup()) {
-        const lastSyncedPush = optionService.getOption('lastSyncedPush');
-
-        stats.outstandingPushes = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE isSynced = 1 AND id > ?", [lastSyncedPush]);
-    }
-}
-
 function getMaxEntityChangeId() {
     return sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes');
 }
 
+function getOutstandingPullCount() {
+    return outstandingPullCount;
+}
+
 sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(sync), 60000);
 
     // kickoff initial sync immediately
     setTimeout(cls.wrap(sync), 3000);
-
-    setInterval(cls.wrap(updatePushStats), 1000);
 });
 
 module.exports = {
     sync,
     login,
     getEntityChangesRecords,
-    stats,
+    getOutstandingPullCount,
     getMaxEntityChangeId
 };

src/services/ws.js

@@ -110,8 +110,7 @@ function sendPing(client, syncRows = []) {
 
     sendMessage(client, {
         type: 'sync',
-        data: syncRows,
+        data: syncRows
-        outstandingSyncs: stats.outstandingPushes + stats.outstandingPulls
     });
 }