save paste images locally WIP

package-lock.json

@@ -4714,6 +4714,11 @@
         "concat-stream": "^1.4.7"
       }
     },
+    "html-comment-regex": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.2.tgz",
+      "integrity": "sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ=="
+    },
     "html-encoding-sniffer": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz",
@@ -5246,6 +5251,14 @@
       "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
       "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
     },
+    "is-svg": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/is-svg/-/is-svg-4.2.1.tgz",
+      "integrity": "sha512-PHx3ANecKsKNl5y5+Jvt53Y4J7MfMpbNZkv384QNiswMKAWIbvcqbPz+sYbFKJI8Xv3be01GSFniPmoaP+Ai5A==",
+      "requires": {
+        "html-comment-regex": "^1.1.2"
+      }
+    },
     "is-tar": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-tar/-/is-tar-1.0.0.tgz",
@@ -9092,9 +9105,9 @@
       "optional": true
     },
     "sqlite": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-3.0.3.tgz",
-      "integrity": "sha512-DpofdtBibbiOObtdADGZYE6bvnLpjRG4ut/MDTDau2nK40htOLj1E0c55aOkvbnRVqQ0ZPtjj7PJuKKyS0Ypww==",
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-3.0.6.tgz",
+      "integrity": "sha512-5SW7HcN+s3TyqpsxOujXhQDCRSCgsxdiU0peT/Y9CT5T0rAsGLwtpXcMyQ7OzOPQ4YUZ5XiGlrwuuQbszr2xtw==",
       "requires": {
         "sql-template-strings": "^2.2.2",
         "sqlite3": "^4.0.0"

package.json

@@ -49,6 +49,7 @@
     "imagemin-mozjpeg": "8.0.0",
     "imagemin-pngquant": "8.0.0",
     "ini": "1.3.5",
+    "is-svg": "^4.2.1",
     "jimp": "0.9.6",
     "mime-types": "2.1.26",
     "multer": "1.4.2",

src/public/javascripts/widgets/note_actions.js

@@ -6,6 +6,11 @@ const TPL = `
     .note-actions .dropdown-menu {
         width: 15em;
     }
+    
+    .note-actions .dropdown-item[disabled], .note-actions .dropdown-item[disabled]:hover {
+        color: var(--muted-text-color) !important;
+        background-color: transparent !important;
+    }
     </style>
 
     <button type="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" class="btn btn-sm dropdown-toggle">
@@ -46,8 +51,18 @@ export default class NoteActionsWidget extends TabAwareWidget {
     }
 
     refreshWithNote(note) {
-        this.$showSourceButton.prop('disabled', !['text', 'relation-map', 'search', 'code'].includes(note.type));
-        this.$exportNoteButton.prop('disabled', note.type !== 'text');
+        if (['text', 'relation-map', 'search', 'code'].includes(note.type)) {
+            this.$showSourceButton.removeAttr('disabled');
+        } else {
+            this.$showSourceButton.attr('disabled', 'disabled');
+        }
+
+        if (note.type === 'text') {
+            this.$exportNoteButton.removeAttr('disabled');
+        }
+        else {
+            this.$exportNoteButton.attr('disabled', 'disabled');
+        }
     }
 
     triggerEvent(e, eventName) {

src/routes/api/image.js

@@ -18,7 +18,7 @@ async function returnImage(req, res) {
         res.set('Content-Type', 'image/png');
         return res.send(fs.readFileSync(RESOURCE_DIR + '/db/image-deleted.png'));
     }
-
+    image.mime = image.mime.replace("image/svg", "image/svg+xml");
     res.set('Content-Type', image.mime);
 
     res.send(await image.getContent());

src/services/image.js

@@ -13,18 +13,19 @@ const jimp = require('jimp');
 const imageType = require('image-type');
 const sanitizeFilename = require('sanitize-filename');
 const noteRevisionService = require('./note_revisions.js');
+const isSvg = require('is-svg');
 
 async function processImage(uploadBuffer, originalName, shrinkImageSwitch) {
-    const origImageFormat = imageType(uploadBuffer);
+    const origImageFormat = getImageType(uploadBuffer);
 
-    if (origImageFormat.ext === "webp") {
+    if (origImageFormat && ["webp", "svg"].includes(origImageFormat.ext)) {
         // JIMP does not support webp at the moment: https://github.com/oliver-moran/jimp/issues/144
         shrinkImageSwitch = false;
     }
 
     const finalImageBuffer = shrinkImageSwitch ? await shrinkImage(uploadBuffer, originalName) : uploadBuffer;
 
-    const imageFormat = imageType(finalImageBuffer);
+    const imageFormat = getImageType(finalImageBuffer);
 
     return {
         buffer: finalImageBuffer,
@@ -32,6 +33,17 @@ async function processImage(uploadBuffer, originalName, shrinkImageSwitch) {
     };
 }
 
+function getImageType(buffer) {
+    if (isSvg(buffer)) {
+        return {
+            ext: 'svg'
+        }
+    }
+    else {
+        return imageType(buffer);
+    }
+}
+
 async function updateImage(noteId, uploadBuffer, originalName) {
     const {buffer, imageFormat} = await processImage(uploadBuffer, originalName, true);
 

src/services/notes.js

@@ -14,6 +14,10 @@ const hoistedNoteService = require('../services/hoisted_note');
 const protectedSessionService = require('../services/protected_session');
 const log = require('../services/log');
 const noteRevisionService = require('../services/note_revisions');
+const attributeService = require('../services/attributes');
+const request = require('./request');
+const path = require('path');
+const url = require('url');
 
 async function getNewNotePosition(parentNoteId) {
     const maxNotePos = await sql.getValue(`
@@ -161,31 +165,6 @@ async function createNewNoteWithTarget(target, targetBranchId, params) {
     }
 }
 
-// methods below should be probably just backend API methods
-async function createJsonNote(parentNoteId, title, content = {}, params = {}) {
-    params.parentNoteId = parentNoteId;
-    params.title = title;
-
-    params.type = "code";
-    params.mime = "application/json";
-
-    params.content = JSON.stringify(content, null, '\t');
-
-    return await createNewNote(params);
-}
-
-async function createTextNote(parentNoteId, title, content = "", params = {}) {
-    params.parentNoteId = parentNoteId;
-    params.title = title;
-
-    params.type = "text";
-    params.mime = "text/html";
-
-    params.content = content;
-
-    return await createNewNote(params);
-}
-
 async function protectNoteRecursively(note, protect, includingSubTree, taskContext) {
     await protectNote(note, protect);
 
@@ -283,6 +262,90 @@ function findRelationMapLinks(content, foundLinks) {
     }
 }
 
+const imageUrlToNoteIdMapping = {};
+
+async function downloadImage(noteId, imageUrl) {
+    const imageBuffer = await request.getImage(imageUrl);
+    const parsedUrl = url.parse(imageUrl);
+    const title = path.basename(parsedUrl.pathname);
+
+    const imageService = require('../services/image');
+    const {note} = await imageService.saveImage(noteId, imageBuffer, title, true);
+
+    await note.addLabel('imageUrl', imageUrl);
+
+    imageUrlToNoteIdMapping[imageUrl] = note.noteId;
+}
+
+const downloadImagePromises = {};
+
+function replaceUrl(content, url, imageNote) {
+    return content.replace(new RegExp(url, "g"), `api/images/${imageNote.noteId}/${imageNote.title}`);
+}
+
+async function downloadImages(noteId, content) {
+    const re = /<img\s.*?src=['"]([^'">]+)['"]/ig;
+    let match;
+
+    while (match = re.exec(content)) {
+        const url = match[1];
+
+        if (!url.startsWith('api/images/')) {
+            if (url in downloadImagePromises) {
+                continue;
+            }
+
+            if (url in imageUrlToNoteIdMapping) {
+                const imageNote = await repository.getNote(imageUrlToNoteIdMapping[url]);
+
+                if (imageNote || imageNote.isDeleted) {
+                    delete imageUrlToNoteIdMapping[url];
+                }
+                else {
+                    content = replaceUrl(content, url, imageNote);
+
+                    continue;
+                }
+            }
+
+            const existingImage = (await attributeService.getNotesWithLabel('imageUrl', url))
+                .find(note => note.type === 'image');
+
+            if (existingImage) {
+                imageUrlToNoteIdMapping[url] = existingImage.noteId;
+
+                content = replaceUrl(content, url, existingImage);
+
+                continue;
+            }
+
+            downloadImagePromises[url] = downloadImage(noteId, url);
+        }
+    }
+
+    await Promise.all(Object.values(downloadImagePromises)).then(() => {
+        setTimeout(async () => {
+            const imageNotes = await repository.getNotes(Object.values(imageUrlToNoteIdMapping));
+
+            const origNote = await repository.getNote(noteId);
+            const origContent = await origNote.getContent();
+            let updatedContent = origContent;
+
+            for (const url in imageUrlToNoteIdMapping) {
+                const imageNote = imageNotes.find(note => note.noteId === imageUrlToNoteIdMapping[url]);
+
+                if (imageNote) {
+                    updatedContent = replaceUrl(updatedContent, url, imageNote);
+                }
+            }
+
+            if (updatedContent !== origContent) {
+                await origNote.setContent(updatedContent);
+            }
+        }, 5000);
+    });
+}
+
 async function saveLinks(note, content) {
     if (note.type !== 'text' && note.type !== 'relation-map') {
         return content;
@@ -299,6 +362,8 @@ async function saveLinks(note, content) {
         content = findInternalLinks(content, foundLinks);
         content = findExternalLinks(content, foundLinks);
         content = findIncludeNoteLinks(content, foundLinks);
+
+        downloadImages(note.noteId, content);
     }
     else if (note.type === 'relation-map') {
         findRelationMapLinks(content, foundLinks);

src/services/request.js

@@ -3,6 +3,7 @@
 const utils = require('./utils');
 const log = require('./log');
 const url = require('url');
+const syncOptions = require('./sync_options');
 
 // this service provides abstraction over node's HTTP/HTTPS and electron net.client APIs
 // this allows to support system proxy
@@ -78,12 +79,60 @@ function exec(opts) {
         catch (e) {
             reject(generateError(opts, e.message));
         }
-    })
+    });
+}
+
+async function getImage(imageUrl) {
+    const opts = {
+        method: 'GET',
+        url: imageUrl,
+        proxy: await syncOptions.getSyncProxy()
+    };
+
+    const client = getClient(opts);
+    const proxyAgent = getProxyAgent(opts);
+    const parsedTargetUrl = url.parse(opts.url);
+
+    return await new Promise(async (resolve, reject) => {
+        try {
+            const request = client.request({
+                method: opts.method,
+                // url is used by electron net module
+                url: opts.url,
+                // 4 fields below are used by http and https node modules
+                protocol: parsedTargetUrl.protocol,
+                host: parsedTargetUrl.hostname,
+                port: parsedTargetUrl.port,
+                path: parsedTargetUrl.path,
+                timeout: opts.timeout,
+                headers: {},
+                agent: proxyAgent
+            });
+
+            request.on('error', err => reject(generateError(opts, err)));
+
+            request.on('response', response => {
+                if (![200, 201, 204].includes(response.statusCode)) {
+                    reject(generateError(opts, response.statusCode + ' ' + response.statusMessage));
+                }
+
+                const chunks = []
+
+                response.on('data', chunk => chunks.push(chunk));
+                response.on('end', () => resolve(Buffer.concat(chunks)));
+            });
+
+            request.end(undefined);
+        }
+        catch (e) {
+            reject(generateError(opts, e.message));
+        }
+    });
 }
 
 function getProxyAgent(opts) {
     if (!opts.proxy) {
-        return;
+        return null;
     }
 
     const {protocol} = url.parse(opts.url);
@@ -122,5 +171,6 @@ function generateError(opts, message) {
 }
 
 module.exports = {
-    exec
+    exec,
+    getImage
 };

src/services/utils.js

@@ -158,7 +158,7 @@ function getContentDisposition(filename) {
     return `file; filename="${sanitizedFilename}"; filename*=UTF-8''${sanitizedFilename}`;
 }
 
-const STRING_MIME_TYPES = ["application/x-javascript"];
+const STRING_MIME_TYPES = ["application/x-javascript", "image/svg"];
 
 function isStringNote(type, mime) {
     return ["text", "code", "relation-map", "search"].includes(type)