Merge branch 'beta'

# Conflicts:
#	docs/backend_api/BAttachment.html
#	docs/backend_api/BNote.html
#	docs/backend_api/becca_entities_bnote.js.html
#	docs/frontend_api/FAttribute.html
#	docs/frontend_api/FBranch.html
#	docs/frontend_api/FNote.html
#	docs/frontend_api/FrontendScriptApi.html
#	docs/frontend_api/entities_fattachment.js.html
#	docs/frontend_api/entities_fattribute.js.html
#	docs/frontend_api/entities_fblob.js.html
#	docs/frontend_api/entities_fbranch.js.html
#	docs/frontend_api/entities_fnote.js.html
#	docs/frontend_api/global.html
#	docs/frontend_api/index.html
#	docs/frontend_api/services_frontend_script_api.js.html
#	package-lock.json

.gitpod.yml

@@ -2,7 +2,7 @@ image:
   file: .gitpod.dockerfile
 
 tasks:
-    - before: nvm install 18.16.1 && nvm use 18.16.1
+    - before: nvm install 18.18.0 && nvm use 18.18.0
       init: npm install
       command: npm run start-server
 

Dockerfile

@@ -1,5 +1,5 @@
 # !!! Don't try to build this Dockerfile directly, run it through bin/build-docker.sh script !!!
-FROM node:18.16.1-alpine
+FROM node:18.18.0-alpine
 
 # Create app directory
 WORKDIR /usr/src/app

bin/build-debian.sh

@@ -5,18 +5,3 @@ echo "Packaging debian x64 distribution..."
 VERSION=`jq -r ".version" package.json`
 
 ./node_modules/.bin/electron-installer-debian --config bin/deb-options.json --options.version=${VERSION} --arch amd64
-
-
-# hacky stop-gag measure to produce debian compatible XZ compressed debs until this is fixed: https://github.com/electron-userland/electron-installer-debian/issues/272
-cd dist
-ar x trilium_${VERSION}_amd64.deb
-rm trilium_${VERSION}_amd64.deb
-# recompress
-< control.tar.zst zstd -d | xz > control.tar.xz
-< data.tar.zst zstd -d | xz > data.tar.xz
-# create deb archive (I really do not know, what argument "sdsd" is for but something is required for ar to create the archive as desired)
-ar -m -c -a sdsd trilium_${VERSION}_amd64.deb debian-binary control.tar.xz data.tar.xz
-
-rm control* data* debian-binary
-
-echo "Converted to XZ deb"

bin/build-server.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 PKG_DIR=dist/trilium-linux-x64-server
-NODE_VERSION=18.16.1
+NODE_VERSION=18.18.0
 
 if [ "$1" != "DONTCOPY" ]
 then

bin/copy-trilium.sh

@@ -5,7 +5,7 @@ if [[ $# -eq 0 ]] ; then
     exit 1
 fi
 
-n exec 18.16.1 npm run webpack
+n exec 18.18.0 npm run webpack
 
 DIR=$1
 
@@ -27,7 +27,7 @@ cp -r electron.js $DIR/
 cp webpack-* $DIR/
 
 # run in subshell (so we return to original dir)
-(cd $DIR && n exec 18.16.1 npm install --only=prod)
+(cd $DIR && n exec 18.18.0 npm install --only=prod)
 
 # cleanup of useless files in dependencies
 rm -r $DIR/node_modules/image-q/demo

bin/deb-options.json

@@ -1,6 +1,7 @@
 {
   "src": "dist/trilium-linux-x64",
   "dest": "dist/",
+  "compression": "xz",
   "name": "trilium",
   "productName": "Trilium Notes",
   "genericName": "Note taker",
@@ -11,4 +12,4 @@
   "bin": "trilium",
   "icon": "dist/trilium-linux-x64/icon.png",
   "categories": [ "Office" ]
-}
+}

docs/backend_api/services_sql.js.html

@@ -54,7 +54,7 @@ const LOG_ALL_QUERIES = false;
 });
 
 function insert(tableName, rec, replace = false) {
-    const keys = Object.keys(rec);
+    const keys = Object.keys(rec || {});
     if (keys.length === 0) {
         log.error(`Can't insert empty object into table ${tableName}`);
         return;
@@ -81,7 +81,7 @@ function replace(tableName, rec) {
 }
 
 function upsert(tableName, primaryKey, rec) {
-    const keys = Object.keys(rec);
+    const keys = Object.keys(rec || {});
     if (keys.length === 0) {
         log.error(`Can't upsert empty object into table ${tableName}`);
         return;

electron.js

@@ -13,6 +13,9 @@ appIconService.installLocalAppIcon();
 
 require('electron-dl')({ saveAs: true });
 
+// needed for excalidraw export https://github.com/zadam/trilium/issues/4271
+app.commandLine.appendSwitch("enable-experimental-web-platform-features");
+
 // Quit when all windows are closed, except on macOS. There, it's common
 // for applications and their menu bar to stay active until the user quits
 // explicitly with Cmd + Q.

package.json

@@ -2,7 +2,7 @@
   "name": "trilium",
   "productName": "Trilium Notes",
   "description": "Trilium Notes",
-  "version": "0.61.6-beta",
+  "version": "0.61.7-beta",
   "license": "AGPL-3.0-only",
   "main": "electron.js",
   "bin": {
@@ -33,7 +33,7 @@
   "dependencies": {
     "@braintree/sanitize-url": "6.0.4",
     "@electron/remote": "2.0.11",
-    "@excalidraw/excalidraw": "0.15.3",
+    "@excalidraw/excalidraw": "0.16.1",
     "archiver": "5.3.1",
     "async-mutex": "0.4.0",
     "axios": "1.5.0",
@@ -43,7 +43,7 @@
     "compression": "1.7.4",
     "cookie-parser": "1.4.6",
     "csurf": "1.11.0",
-    "dayjs": "1.11.9",
+    "dayjs": "1.11.10",
     "dayjs-plugin-utc": "0.1.2",
     "debounce": "1.2.1",
     "ejs": "3.1.9",
@@ -68,7 +68,7 @@
     "jimp": "0.22.10",
     "joplin-turndown-plugin-gfm": "1.0.12",
     "jsdom": "22.1.0",
-    "marked": "9.0.0",
+    "marked": "9.0.3",
     "mime-types": "2.1.35",
     "multer": "1.4.5-lts.1",
     "node-abi": "3.47.0",
@@ -91,17 +91,17 @@
     "tmp": "0.2.1",
     "turndown": "7.1.2",
     "unescape": "1.0.1",
-    "ws": "8.14.1",
+    "ws": "8.14.2",
     "xml2js": "0.6.2",
     "yauzl": "2.10.0"
   },
   "devDependencies": {
     "cross-env": "7.0.3",
-    "electron": "25.8.1",
+    "electron": "25.8.2",
     "electron-builder": "24.6.4",
     "electron-packager": "17.1.2",
     "electron-rebuild": "3.2.9",
-    "eslint": "8.48.0",
+    "eslint": "8.49.0",
     "eslint-config-airbnb-base": "15.0.0",
     "eslint-config-prettier": "9.0.0",
     "eslint-plugin-import": "2.28.1",

src/public/app/services/frontend_script_api.js

@@ -14,6 +14,7 @@ import NoteContextAwareWidget from "../widgets/note_context_aware_widget.js";
 import BasicWidget from "../widgets/basic_widget.js";
 import SpacedUpdate from "./spaced_update.js";
 import shortcutService from "./shortcuts.js";
+import dialogService from "./dialog.js";
 
 
 /**
@@ -288,7 +289,7 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
     this.parseDate = utils.parseDate;
 
     /**
-     * Show an info message to the user.
+     * Show an info toast message to the user.
      *
      * @method
      * @param {string} message
@@ -296,13 +297,43 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
     this.showMessage = toastService.showMessage;
 
     /**
-     * Show an error message to the user.
+     * Show an error toast message to the user.
      *
      * @method
      * @param {string} message
      */
     this.showError = toastService.showError;
 
+    /**
+     * Show an info dialog to the user.
+     *
+     * @method
+     * @param {string} message
+     * @returns {Promise}
+     */
+    this.showInfoDialog = dialogService.info;
+
+    /**
+     * Show confirm dialog to the user.
+     *
+     * @method
+     * @param {string} message
+     * @returns {Promise<boolean>} promise resolving to true if the user confirmed
+     */
+    this.showConfirmDialog = dialogService.confirm;
+
+    /**
+     * Show prompt dialog to the user.
+     *
+     * @method
+     * @param {object} props
+     * @param {string} props.title
+     * @param {string} props.message
+     * @param {string} props.defaultValue
+     * @returns {Promise<string>} promise resolving to the answer provided by the user
+     */
+    this.showPromptDialog = dialogService.prompt;
+
     /**
      * Trigger command. This is a very low-level API which should be avoided if possible.
      *

src/public/app/widgets/type_widgets/canvas.js

@@ -20,10 +20,13 @@ const TPL = `
             display: block;
         }
 
-
         .excalidraw-wrapper {
             height: 100%;
         }
+        
+        .excalidraw button[data-testid="json-export-button"] {
+            display: none !important;
+        }
 
         :root[dir="ltr"]
         .excalidraw
@@ -60,8 +63,8 @@ const TPL = `
  * for sketching. Excalidraw has a vibrant and active community.
  *
  * Functionality:
- * We store the excalidraw assets (elements, appState, files) in the note. In addition to that, we
+ * We store the excalidraw assets (elements and files) in the note. In addition to that, we
- * export the SVG from the canvas on every update. The SVG is also saved in the note. It is used when
+ * export the SVG from the canvas on every update and store it in the note's attachment. It is used when
  * calling api/images and makes referencing very easy.
  *
  * Paths not taken.
@@ -209,19 +212,15 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
 
                 content = {
                     elements: [],
-                    appState: {},
                     files: [],
+                    appState: {}
                 };
             }
 
-            const {elements, appState, files} = content;
+            const {elements, files, appState} = content;
 
             appState.theme = this.themeStyle;
 
-            /**
-             * use widths and offsets of current view, since stored appState has the state from
-             * previous edit. using the stored state would lead to pointer mismatch.
-             */
             const boundingClientRect = this.excalidrawWrapperRef.current.getBoundingClientRect();
             appState.width = boundingClientRect.width;
             appState.height = boundingClientRect.height;
@@ -284,10 +283,7 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
          */
         const files = this.excalidrawRef.current.getFiles();
 
-        /**
+        // parallel svg export to combat bitrot and enable rendering image for note inclusion, preview, and share
-         * parallel svg export to combat bitrot and enable rendering image for note inclusion,
-         *  preview, and share.
-         */
         const svg = await window.ExcalidrawLib.exportToSvg({
             elements,
             appState,
@@ -302,14 +298,18 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
             if (element.fileId) {
                 activeFiles[element.fileId] = files[element.fileId];
             }
-        })
+        });
 
         const content = {
             type: "excalidraw",
             version: 2,
             elements,
-            appState,
+            files: activeFiles,
-            files: activeFiles
+            appState: {
+                scrollX: appState.scrollX,
+                scrollY: appState.scrollY,
+                zoom: appState.zoom
+            }
         };
 
         const attachments = [
@@ -339,7 +339,7 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
 
         return {
             content: JSON.stringify(content),
-            attachments: attachments
+            attachments
         };
     }
 
@@ -458,6 +458,10 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
                     handleKeyboardGlobally: false,
                     autoFocus: false,
                     onLinkOpen,
+                    UIOptions: {
+                        saveToActiveFile: false,
+                        saveAsImage: false
+                    }
                 })
             )
         );

src/routes/api/sync.js

@@ -166,11 +166,7 @@ function update(req) {
 
     const {entities, instanceId} = body;
 
-    sql.transactional(() => {
+    sql.transactional(() => syncUpdateService.updateEntities(entities, instanceId));
-        for (const {entityChange, entity} of entities) {
-            syncUpdateService.updateEntity(entityChange, entity, instanceId);
-        }
-    });
 }
 
 setInterval(() => {

src/services/build.js

@@ -1 +1 @@
-module.exports = { buildDate:"2023-09-06T23:57:29+02:00", buildRevision: "6fa9d996e84f87fcb73c3388a5170affd2c2f7cc" };
+module.exports = { buildDate:"2023-09-21T23:38:18+02:00", buildRevision: "79e5e3b65ff613cdb81e2afaa832037ccf06d7b8" };

src/services/consistency_checks.js

@@ -12,6 +12,7 @@ const BBranch = require('../becca/entities/bbranch');
 const revisionService = require('./revisions');
 const becca = require("../becca/becca");
 const utils = require("../services/utils");
+const eraseService = require("../services/erase");
 const {sanitizeAttributeName} = require("./sanitize_attribute_name");
 const noteTypes = require("../services/note_types").getNoteTypeNames();
 
@@ -440,7 +441,7 @@ class ConsistencyChecks {
             this.findAndFixIssues(`
                         SELECT notes.noteId, notes.type, notes.mime
                         FROM notes
-                                 JOIN blobs USING (blobId)
+                          JOIN blobs USING (blobId)
                         WHERE isDeleted = 0
                           AND isProtected = 0
                           AND content IS NULL`,
@@ -460,19 +461,36 @@ class ConsistencyChecks {
         }
 
         this.findAndFixIssues(`
-                    SELECT revisions.revisionId
+                    SELECT revisions.revisionId, blobs.blobId
                     FROM revisions
                       LEFT JOIN blobs USING (blobId)
                     WHERE blobs.blobId IS NULL`,
-            ({revisionId}) => {
+            ({revisionId, blobId}) => {
                 if (this.autoFix) {
                     revisionService.eraseRevisions([revisionId]);
 
                     this.reloadNeeded = true;
 
-                    logFix(`Note revision content '${revisionId}' was set to erased since its content did not exist.`);
+                    logFix(`Note revision '${revisionId}' was erased since the referenced blob '${blobId}' did not exist.`);
                 } else {
-                    logError(`Note revision content '${revisionId}' does not exist`);
+                    logError(`Note revision '${revisionId}' blob '${blobId}' does not exist`);
+                }
+            });
+
+        this.findAndFixIssues(`
+                    SELECT attachments.attachmentId, blobs.blobId
+                    FROM attachments
+                      LEFT JOIN blobs USING (blobId)
+                    WHERE blobs.blobId IS NULL`,
+            ({attachmentId, blobId}) => {
+                if (this.autoFix) {
+                    eraseService.eraseAttachments([attachmentId]);
+
+                    this.reloadNeeded = true;
+
+                    logFix(`Attachment '${attachmentId}' was erased since the referenced blob '${blobId}' did not exist.`);
+                } else {
+                    logError(`Attachment '${attachmentId}' blob '${blobId}' does not exist`);
                 }
             });
 

src/services/erase.js

@@ -183,5 +183,6 @@ module.exports = {
     eraseDeletedNotesNow,
     eraseUnusedAttachmentsNow,
     eraseNotesWithDeleteId,
-    eraseUnusedBlobs
+    eraseUnusedBlobs,
+    eraseAttachments
 };

src/services/notes.js

@@ -895,6 +895,10 @@ async function asyncPostProcessContent(note, content) {
 
 // all keys should be replaced by the corresponding values
 function replaceByMap(str, mapObj) {
+    if (!mapObj) {
+        return str;
+    }
+
     const re = new RegExp(Object.keys(mapObj).join("|"),"g");
 
     return str.replace(re, matched => mapObj[matched]);

src/services/revisions.js

@@ -51,7 +51,7 @@ function eraseRevisions(revisionIdsToErase) {
         return;
     }
 
-    log.info(`Removing note revisions: ${JSON.stringify(revisionIdsToErase)}`);
+    log.info(`Removing revisions: ${JSON.stringify(revisionIdsToErase)}`);
 
     sql.executeMany(`DELETE FROM revisions WHERE revisionId IN (???)`, revisionIdsToErase);
     sql.executeMany(`UPDATE entity_changes SET isErased = 1, utcDateChanged = '${dateUtils.utcNowDateTime()}' WHERE entityName = 'revisions' AND entityId IN (???)`, revisionIdsToErase);

src/services/setup.js

@@ -93,7 +93,7 @@ async function setupSyncFromSyncServer(syncServerHost, syncProxy, password) {
         return { result: 'success' };
     }
     catch (e) {
-        log.error(`Sync failed: ${e.message}`);
+        log.error(`Sync failed: '${e.message}', stack: ${e.stack}`);
 
         return {
             result: 'failure',

src/services/sql.js

@@ -26,7 +26,7 @@ const LOG_ALL_QUERIES = false;
 });
 
 function insert(tableName, rec, replace = false) {
-    const keys = Object.keys(rec);
+    const keys = Object.keys(rec || {});
     if (keys.length === 0) {
         log.error(`Can't insert empty object into table ${tableName}`);
         return;
@@ -53,7 +53,7 @@ function replace(tableName, rec) {
 }
 
 function upsert(tableName, primaryKey, rec) {
-    const keys = Object.keys(rec);
+    const keys = Object.keys(rec || {});
     if (keys.length === 0) {
         log.error(`Can't upsert empty object into table ${tableName}`);
         return;

src/services/sync.js

@@ -71,8 +71,7 @@ async function sync() {
             };
         }
         else {
-            log.info(`sync failed: ${e.message}
+            log.info(`Sync failed: '${e.message}', stack: ${e.stack}`);
-stack: ${e.stack}`);
 
             ws.syncFailed();
 
@@ -127,8 +126,6 @@ async function doLogin() {
 }
 
 async function pullChanges(syncContext) {
-    let atLeastOnePullApplied = false;
-
     while (true) {
         const lastSyncedPull = getLastSyncedPull();
         const logMarkerId = utils.randomString(10); // to easily pair sync events between client and server logs
@@ -144,22 +141,7 @@ async function pullChanges(syncContext) {
         const pulledDate = Date.now();
 
         sql.transactional(() => {
-            for (const {entityChange, entity} of entityChanges) {
+            syncUpdateService.updateEntities(entityChanges, syncContext.instanceId);
-                const changeAppliedAlready = entityChange.changeId
-                    && !!sql.getValue("SELECT 1 FROM entity_changes WHERE changeId = ?", [entityChange.changeId]);
-
-                if (changeAppliedAlready) {
-                    continue;
-                }
-
-                if (!atLeastOnePullApplied) { // send only for first
-                    ws.syncPullInProgress();
-
-                    atLeastOnePullApplied = true;
-                }
-
-                syncUpdateService.updateEntity(entityChange, entity, syncContext.instanceId);
-            }
 
             if (lastSyncedPull !== lastEntityChangeId) {
                 setLastSyncedPull(lastEntityChangeId);

src/services/sync_update.js

@@ -3,15 +3,51 @@ const log = require('./log');
 const entityChangesService = require('./entity_changes');
 const eventService = require('./events');
 const entityConstructor = require("../becca/entity_constructor");
+const ws = require("./ws");
 
-function updateEntity(remoteEC, remoteEntityRow, instanceId) {
+function updateEntities(entityChanges, instanceId) {
+    if (entityChanges.length === 0) {
+        return;
+    }
+
+    let atLeastOnePullApplied = false;
+    const updateContext = {
+        updated: {},
+        alreadyUpdated: 0,
+        erased: 0,
+        alreadyErased: 0
+    };
+
+    for (const {entityChange, entity} of entityChanges) {
+        const changeAppliedAlready = entityChange.changeId
+            && !!sql.getValue("SELECT 1 FROM entity_changes WHERE changeId = ?", [entityChange.changeId]);
+
+        if (changeAppliedAlready) {
+            updateContext.alreadyUpdated++;
+
+            continue;
+        }
+
+        if (!atLeastOnePullApplied) { // avoid spamming and send only for first
+            ws.syncPullInProgress();
+
+            atLeastOnePullApplied = true;
+        }
+
+        updateEntity(entityChange, entity, instanceId, updateContext);
+    }
+
+    logUpdateContext(updateContext);
+}
+
+function updateEntity(remoteEC, remoteEntityRow, instanceId, updateContext) {
     if (!remoteEntityRow && remoteEC.entityName === 'options') {
         return; // can be undefined for options with isSynced=false
     }
 
     const updated = remoteEC.entityName === 'note_reordering'
         ? updateNoteReordering(remoteEC, remoteEntityRow, instanceId)
-        : updateNormalEntity(remoteEC, remoteEntityRow, instanceId);
+        : updateNormalEntity(remoteEC, remoteEntityRow, instanceId, updateContext);
 
     if (updated) {
         if (remoteEntityRow?.isDeleted) {
@@ -29,11 +65,12 @@ function updateEntity(remoteEC, remoteEntityRow, instanceId) {
     }
 }
 
-function updateNormalEntity(remoteEC, remoteEntityRow, instanceId) {
+function updateNormalEntity(remoteEC, remoteEntityRow, instanceId, updateContext) {
     const localEC = sql.getRow(`SELECT * FROM entity_changes WHERE entityName = ? AND entityId = ?`, [remoteEC.entityName, remoteEC.entityId]);
 
     if (!localEC?.isErased && remoteEC.isErased) {
         eraseEntity(remoteEC, instanceId);
+        updateContext.erased++;
 
         return true;
     } else if (localEC?.isErased && !remoteEC.isErased) {
@@ -42,10 +79,15 @@ function updateNormalEntity(remoteEC, remoteEntityRow, instanceId) {
 
         return false;
     } else if (localEC?.isErased && remoteEC.isErased) {
+        updateContext.alreadyErased++;
         return false;
     }
 
     if (!localEC || localEC.utcDateChanged <= remoteEC.utcDateChanged) {
+        if (!remoteEntityRow) {
+            throw new Error(`Empty entity row for: ${JSON.stringify(remoteEC)}`);
+        }
+
         if (remoteEC.entityName === 'blobs' && remoteEntityRow.content !== null) {
             // we always use a Buffer object which is different from normal saving - there we use a simple string type for
             // "string notes". The problem is that in general, it's not possible to detect whether a blob content
@@ -61,6 +103,9 @@ function updateNormalEntity(remoteEC, remoteEntityRow, instanceId) {
 
         sql.replace(remoteEC.entityName, remoteEntityRow);
 
+        updateContext.updated[remoteEC.entityName] = updateContext.updated[remoteEC.entityName] || [];
+        updateContext.updated[remoteEC.entityName].push(remoteEC.entityId);
+
         if (!localEC || localEC.utcDateChanged < remoteEC.utcDateChanged) {
             entityChangesService.putEntityChangeWithInstanceId(remoteEC, instanceId);
         }
@@ -77,6 +122,10 @@ function updateNormalEntity(remoteEC, remoteEntityRow, instanceId) {
 }
 
 function updateNoteReordering(remoteEC, remoteEntityRow, instanceId) {
+    if (!remoteEntityRow) {
+        throw new Error(`Empty note_reordering body for: ${JSON.stringify(remoteEC)}`);
+    }
+
     for (const key in remoteEntityRow) {
         sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remoteEntityRow[key], key]);
     }
@@ -110,6 +159,15 @@ function eraseEntity(entityChange, instanceId) {
     entityChangesService.putEntityChangeWithInstanceId(entityChange, instanceId);
 }
 
+function logUpdateContext(updateContext) {
+    const message = JSON.stringify(updateContext)
+        .replaceAll('"', '')
+        .replaceAll(":", ": ")
+        .replaceAll(",", ", ");
+
+    log.info(message.substr(1, message.length - 2));
+}
+
 module.exports = {
-    updateEntity
+    updateEntities
 };

src/services/utils.js

@@ -25,6 +25,10 @@ function md5(content) {
 }
 
 function hashedBlobId(content) {
+    if (content === null || content === undefined) {
+        content = "";
+    }
+
     // sha512 is faster than sha256
     const base64Hash = crypto.createHash('sha512').update(content).digest('base64');