use marked instead of commonmark to convert MD to HTML, which supports e.g. tables, closes #2026

.eslintrc.js

@@ -102,8 +102,6 @@ module.exports = {
         mermaid: true,
         // src\public\app\services\frontend_script_api.js
         dayjs: true,
-        // \src\public\app\widgets\dialogs\markdown_import.js
-        commonmark: true,
         // \src\public\app\widgets\note_map.js
         ForceGraph: true,
         // \src\public\app\setup.js

package-lock.json

@@ -19,7 +19,6 @@
         "better-sqlite3": "8.4.0",
         "chokidar": "3.5.3",
         "cls-hooked": "4.2.2",
-        "commonmark": "0.30.0",
         "compression": "1.7.4",
         "cookie-parser": "1.4.6",
         "csurf": "1.11.0",
@@ -48,6 +47,7 @@
         "jimp": "0.22.8",
         "joplin-turndown-plugin-gfm": "1.0.12",
         "jsdom": "22.1.0",
+        "marked": "5.1.1",
         "mime-types": "2.1.35",
         "multer": "1.4.5-lts.1",
         "node-abi": "3.45.0",
@@ -3480,28 +3480,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/commonmark": {
-      "version": "0.30.0",
-      "resolved": "https://registry.npmjs.org/commonmark/-/commonmark-0.30.0.tgz",
-      "integrity": "sha512-j1yoUo4gxPND1JWV9xj5ELih0yMv1iCWDG6eEQIPLSWLxzCXiFoyS7kvB+WwU+tZMf4snwJMMtaubV0laFpiBA==",
-      "dependencies": {
-        "entities": "~2.0",
-        "mdurl": "~1.0.1",
-        "minimist": ">=1.2.2",
-        "string.prototype.repeat": "^0.2.0"
-      },
-      "bin": {
-        "commonmark": "bin/commonmark"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
-    "node_modules/commonmark/node_modules/entities": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.3.tgz",
-      "integrity": "sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ=="
-    },
     "node_modules/compare-version": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/compare-version/-/compare-version-0.1.2.tgz",
@@ -8040,6 +8018,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/jsdoc/node_modules/marked": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz",
+      "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==",
+      "dev": true,
+      "bin": {
+        "marked": "bin/marked.js"
+      },
+      "engines": {
+        "node": ">= 12"
+      }
+    },
     "node_modules/jsdoc/node_modules/mkdirp": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
@@ -8923,15 +8913,14 @@
       }
     },
     "node_modules/marked": {
-      "version": "4.0.12",
-      "resolved": "https://registry.npmjs.org/marked/-/marked-4.0.12.tgz",
-      "integrity": "sha512-hgibXWrEDNBWgGiK18j/4lkS6ihTe9sxtV4Q1OQppb/0zzyPSzoFANBa5MfsG/zgsWklmNnhm0XACZOH/0HBiQ==",
-      "dev": true,
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-5.1.1.tgz",
+      "integrity": "sha512-bTmmGdEINWmOMDjnPWDxGPQ4qkDLeYorpYbEtFOXzOruTwUE671q4Guiuchn4N8h/v6NGd7916kXsm3Iz4iUSg==",
       "bin": {
         "marked": "bin/marked.js"
       },
       "engines": {
-        "node": ">= 12"
+        "node": ">= 18"
       }
     },
     "node_modules/matcher": {
@@ -8961,7 +8950,8 @@
     "node_modules/mdurl": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz",
-      "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4="
+      "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=",
+      "dev": true
     },
     "node_modules/media-typer": {
       "version": "0.3.0",
@@ -11876,11 +11866,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/string.prototype.repeat": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/string.prototype.repeat/-/string.prototype.repeat-0.2.0.tgz",
-      "integrity": "sha1-q6Nt4I3O5qWjN9SbLqHaGyj8Ds8="
-    },
     "node_modules/string.prototype.trim": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz",
@@ -15962,24 +15947,6 @@
       "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==",
       "devOptional": true
     },
-    "commonmark": {
-      "version": "0.30.0",
-      "resolved": "https://registry.npmjs.org/commonmark/-/commonmark-0.30.0.tgz",
-      "integrity": "sha512-j1yoUo4gxPND1JWV9xj5ELih0yMv1iCWDG6eEQIPLSWLxzCXiFoyS7kvB+WwU+tZMf4snwJMMtaubV0laFpiBA==",
-      "requires": {
-        "entities": "~2.0",
-        "mdurl": "~1.0.1",
-        "minimist": ">=1.2.2",
-        "string.prototype.repeat": "^0.2.0"
-      },
-      "dependencies": {
-        "entities": {
-          "version": "2.0.3",
-          "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.3.tgz",
-          "integrity": "sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ=="
-        }
-      }
-    },
     "compare-version": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/compare-version/-/compare-version-0.1.2.tgz",
@@ -19350,6 +19317,12 @@
           "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==",
           "dev": true
         },
+        "marked": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz",
+          "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==",
+          "dev": true
+        },
         "mkdirp": {
           "version": "1.0.4",
           "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
@@ -20016,10 +19989,9 @@
       "requires": {}
     },
     "marked": {
-      "version": "4.0.12",
-      "resolved": "https://registry.npmjs.org/marked/-/marked-4.0.12.tgz",
-      "integrity": "sha512-hgibXWrEDNBWgGiK18j/4lkS6ihTe9sxtV4Q1OQppb/0zzyPSzoFANBa5MfsG/zgsWklmNnhm0XACZOH/0HBiQ==",
-      "dev": true
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-5.1.1.tgz",
+      "integrity": "sha512-bTmmGdEINWmOMDjnPWDxGPQ4qkDLeYorpYbEtFOXzOruTwUE671q4Guiuchn4N8h/v6NGd7916kXsm3Iz4iUSg=="
     },
     "matcher": {
       "version": "3.0.0",
@@ -20041,7 +20013,8 @@
     "mdurl": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz",
-      "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4="
+      "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=",
+      "dev": true
     },
     "media-typer": {
       "version": "0.3.0",
@@ -22235,11 +22208,6 @@
         "strip-ansi": "^6.0.1"
       }
     },
-    "string.prototype.repeat": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/string.prototype.repeat/-/string.prototype.repeat-0.2.0.tgz",
-      "integrity": "sha1-q6Nt4I3O5qWjN9SbLqHaGyj8Ds8="
-    },
     "string.prototype.trim": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz",

package.json

@@ -40,7 +40,6 @@
     "better-sqlite3": "8.4.0",
     "chokidar": "3.5.3",
     "cls-hooked": "4.2.2",
-    "commonmark": "0.30.0",
     "compression": "1.7.4",
     "cookie-parser": "1.4.6",
     "csurf": "1.11.0",
@@ -69,6 +68,7 @@
     "jimp": "0.22.8",
     "joplin-turndown-plugin-gfm": "1.0.12",
     "jsdom": "22.1.0",
+    "marked": "5.1.1",
     "mime-types": "2.1.35",
     "multer": "1.4.5-lts.1",
     "node-abi": "3.45.0",
@@ -109,13 +109,13 @@
     "eslint-plugin-prettier": "5.0.0",
     "esm": "3.2.25",
     "husky": "8.0.3",
-    "jsonc-eslint-parser": "2.3.0",
-    "lint-staged": "13.2.3",
     "jasmine": "5.0.2",
     "jsdoc": "4.0.2",
+    "jsonc-eslint-parser": "2.3.0",
+    "lint-staged": "13.2.3",
     "lorem-ipsum": "2.0.8",
-    "prettier": "3.0.0",
     "nodemon": "3.0.1",
+    "prettier": "3.0.0",
     "rcedit": "3.0.1",
     "webpack": "5.88.1",
     "webpack-cli": "5.1.4"

src/public/app/services/library_loader.js

@@ -23,8 +23,6 @@ const CODE_MIRROR = {
 
 const ESLINT = {js: ["libraries/eslint.js"]};
 
-const COMMONMARK = {js: ["libraries/commonmark.min.js"]};
-
 const RELATION_MAP = {
     js: [
         "libraries/jsplumb.js",
@@ -119,7 +117,6 @@ export default {
     CKEDITOR,
     CODE_MIRROR,
     ESLINT,
-    COMMONMARK,
     RELATION_MAP,
     PRINT_THIS,
     CALENDAR_WIDGET,

src/public/app/widgets/dialogs/markdown_import.js

@@ -4,6 +4,7 @@ import utils from "../../services/utils.js";
 import appContext from "../../components/app_context.js";
 import BasicWidget from "../basic_widget.js";
 import shortcutService from "../../services/shortcuts.js";
+import server from "../../services/server.js";
 
 const TPL = `
 <div class="markdown-import-dialog modal fade mx-auto" tabindex="-1" role="dialog">
@@ -46,18 +47,14 @@ export default class MarkdownImportDialog extends BasicWidget {
         shortcutService.bindElShortcut(this.$widget, 'ctrl+return', () => this.sendForm());
     }
 
-    async convertMarkdownToHtml(text) {
-        await libraryLoader.requireLibrary(libraryLoader.COMMONMARK);
+    async convertMarkdownToHtml(markdownContent) {
+        const {htmlContent} = await server.post('other/render-markdown', { markdownContent });
 
-        const reader = new commonmark.Parser();
-        const writer = new commonmark.HtmlRenderer();
-        const parsed = reader.parse(text);
-
-        const result = writer.render(parsed);
+        console.log(htmlContent);
 
         const textEditor = await appContext.tabManager.getActiveContext().getTextEditor();
 
-        const viewFragment = textEditor.data.processor.toView(result);
+        const viewFragment = textEditor.data.processor.toView(htmlContent);
         const modelFragment = textEditor.data.toModel(viewFragment);
 
         textEditor.model.insertContent(modelFragment, textEditor.model.document.selection);

src/routes/api/other.js

@@ -1,4 +1,5 @@
 const becca = require("../../becca/becca");
+const markdownService = require("../../services/import/markdown");
 
 function getIconUsage() {
     const iconClassToCountMap = {};
@@ -24,6 +25,15 @@ function getIconUsage() {
     return { iconClassToCountMap };
 }
 
+function renderMarkdown(req) {
+    const { markdownContent } = req.body;
+
+    return {
+        htmlContent: markdownService.renderToHtml(markdownContent, '')
+    };
+}
+
 module.exports = {
-    getIconUsage
+    getIconUsage,
+    renderMarkdown
 };

src/routes/routes.js

@@ -324,6 +324,7 @@ function register(app) {
     apiRoute(PST, '/api/delete-notes-preview', notesApiRoute.getDeleteNotesPreview);
     route(GET, '/api/fonts', [auth.checkApiAuthOrElectron], fontsRoute.getFontCss);
     apiRoute(GET, '/api/other/icon-usage', otherRoute.getIconUsage);
+    apiRoute(PST, '/api/other/render-markdown', otherRoute.renderMarkdown);
     apiRoute(GET, '/api/recent-changes/:ancestorNoteId', recentChangesApiRoute.getRecentChanges);
     apiRoute(GET, '/api/edited-notes/:date', revisionsApiRoute.getEditedNotesOnDate);
 

src/services/import/markdown.js

@@ -0,0 +1,18 @@
+"use strict";
+
+const marked = require("marked");
+const htmlSanitizer = require("../html_sanitizer");
+const importUtils = require("./utils");
+
+function renderToHtml(content, title) {
+    const html = marked.parse(content, {
+        mangle: false,
+        headerIds: false
+    });
+    const h1Handled = importUtils.handleH1(html, title); // h1 handling needs to come before sanitization
+    return htmlSanitizer.sanitize(h1Handled);
+}
+
+module.exports = {
+    renderToHtml
+};

src/services/import/single.js

@@ -3,9 +3,10 @@
 const noteService = require('../../services/notes');
 const imageService = require('../../services/image');
 const protectedSessionService = require('../protected_session');
-const commonmark = require('commonmark');
+const markdownService = require('./markdown');
 const mimeService = require('./mime');
 const utils = require('../../services/utils');
+const importUtils = require('./utils');
 const htmlSanitizer = require('../html_sanitizer');
 
 function importSingleFile(taskContext, file, parentNote) {
@@ -120,16 +121,7 @@ function importMarkdown(taskContext, file, parentNote) {
     const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
 
     const markdownContent = file.buffer.toString("utf-8");
-
-    const reader = new commonmark.Parser();
-    const writer = new commonmark.HtmlRenderer();
-
-    const parsed = reader.parse(markdownContent);
-    let htmlContent = writer.render(parsed);
-
-    htmlContent = htmlSanitizer.sanitize(htmlContent);
-
-    htmlContent = handleH1(htmlContent, title);
+    const htmlContent = markdownService.renderToHtml(markdownContent, title);
 
     const {note} = noteService.createNewNote({
         parentNoteId: parentNote.noteId,
@@ -145,24 +137,12 @@ function importMarkdown(taskContext, file, parentNote) {
     return note;
 }
 
-function handleH1(content, title) {
-    content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
-        if (title.trim() === text.trim()) {
-            return ""; // remove whole H1 tag
-        } else {
-            return `<h2>${text}</h2>`;
-        }
-    });
-    return content;
-}
-
 function importHtml(taskContext, file, parentNote) {
     const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
     let content = file.buffer.toString("utf-8");
 
     content = htmlSanitizer.sanitize(content);
-
-    content = handleH1(content, title);
+    content = importUtils.handleH1(content, title);
 
     const {note} = noteService.createNewNote({
         parentNoteId: parentNote.noteId,

src/services/import/utils.js

@@ -0,0 +1,16 @@
+"use strict";
+
+function handleH1(content, title) {
+    content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
+        if (title.trim() === text.trim()) {
+            return ""; // remove whole H1 tag
+        } else {
+            return `<h2>${text}</h2>`;
+        }
+    });
+    return content;
+}
+
+module.exports = {
+    handleH1
+};

src/services/import/zip.js

@@ -7,7 +7,6 @@ const noteService = require('../../services/notes');
 const attributeService = require('../../services/attributes');
 const BBranch = require('../../becca/entities/bbranch');
 const path = require('path');
-const commonmark = require('commonmark');
 const protectedSessionService = require('../protected_session');
 const mimeService = require("./mime");
 const treeService = require("../tree");
@@ -15,6 +14,7 @@ const yauzl = require("yauzl");
 const htmlSanitizer = require('../html_sanitizer');
 const becca = require("../../becca/becca");
 const BAttachment = require("../../becca/entities/battachment");
+const markdownService = require("./markdown");
 
 /**
  * @param {TaskContext} taskContext
@@ -31,8 +31,6 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
     // path => noteId, used only when meta file is not available
     /** @type {Object.<string, string>} path => noteId | attachmentId */
     const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId };
-    const mdReader = new commonmark.Parser();
-    const mdWriter = new commonmark.HtmlRenderer();
     let metaFile = null;
     /** @type {BNote} */
     let firstNote = null;
@@ -414,8 +412,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
     function processNoteContent(noteMeta, type, mime, content, noteTitle, filePath) {
         if (noteMeta?.format === 'markdown'
             || (!noteMeta && taskContext.data.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime))) {
-            const parsed = mdReader.parse(content);
-            content = mdWriter.render(parsed);
+            content = markdownService.renderToHtml(content, noteTitle);
         }
 
         if (type === 'text') {