fixes for zip export/import in regard to attachments

src/becca/entities/abstract_becca_entity.js

@@ -252,7 +252,7 @@ class AbstractBeccaEntity {
         if (this.hasStringContent()) {
             return content === null
                 ? ""
-                : content.toString("UTF-8");
+                : content.toString("utf-8");
         } else {
             return content;
         }

src/becca/entities/bnote.js

@@ -1120,7 +1120,8 @@ class BNote extends AbstractBeccaEntity {
                 SELECT attachments.*
                 FROM attachments 
                 WHERE parentId = ? 
-                  AND isDeleted = 0`, [this.noteId])
+                  AND isDeleted = 0
+                ORDER BY position`, [this.noteId])
             .map(row => new BAttachment(row));
     }
 
@@ -1142,7 +1143,8 @@ class BNote extends AbstractBeccaEntity {
                 FROM attachments 
                 WHERE parentId = ? 
                   AND role = ?
-                  AND isDeleted = 0`, [this.noteId, role])
+                  AND isDeleted = 0
+                ORDER BY position`, [this.noteId, role])
             .map(row => new BAttachment(row));
     }
 

src/services/data_encryption.js

@@ -112,7 +112,7 @@ function decryptString(dataKey, cipherText) {
         throw new Error("Could not decrypt string.");
     }
 
-    return buffer.toString('UTF-8');
+    return buffer.toString('utf-8');
 }
 
 module.exports = {

src/services/etapi_tokens.js

@@ -34,7 +34,7 @@ function parseAuthToken(auth) {
         // allow also basic auth format for systems which allow this type of authentication
         // expect ETAPI token in the password field, require "etapi" username
         // https://github.com/zadam/trilium/issues/3181
-        const basicAuthStr = utils.fromBase64(auth.substring(6)).toString("UTF-8");
+        const basicAuthStr = utils.fromBase64(auth.substring(6)).toString("utf-8");
         const basicAuthChunks = basicAuthStr.split(":");
 
         if (basicAuthChunks.length !== 2) {

src/services/export/zip.js

@@ -122,7 +122,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
      * @param {Object.<string, integer>} existingFileNames
      * @returns {NoteMeta|null}
      */
-    function getNoteMeta(branch, parentMeta, existingFileNames) {
+    function createNoteMeta(branch, parentMeta, existingFileNames) {
         const note = branch.getNote();
 
         if (note.hasOwnedLabel('excludeFromExport')) {
@@ -200,6 +200,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
                 attMeta.title = attachment.title;
                 attMeta.role = attachment.role;
                 attMeta.mime = attachment.mime;
+                attMeta.position = attachment.position;
                 attMeta.dataFileName = getDataFileName(
                     null,
                     attachment.mime,
@@ -217,7 +218,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
             const childExistingNames = {};
 
             for (const childBranch of childBranches) {
-                const note = getNoteMeta(childBranch, meta, childExistingNames);
+                const note = createNoteMeta(childBranch, meta, childExistingNames);
 
                 // can be undefined if export is disabled for this note
                 if (note) {
@@ -234,7 +235,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
      * @param {NoteMeta} sourceMeta
      * @return {string|null}
      */
-    function getTargetUrl(targetNoteId, sourceMeta) {
+    function getNoteTargetUrl(targetNoteId, sourceMeta) {
         const targetMeta = noteIdToMeta[targetNoteId];
 
         if (!targetMeta) {
@@ -271,15 +272,29 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
      * @param {NoteMeta} noteMeta
      * @return {string}
      */
-    function findLinks(content, noteMeta) {
+    function rewriteLinks(content, noteMeta) {
         content = content.replace(/src="[^"]*api\/images\/([a-zA-Z0-9_]+)\/[^"]*"/g, (match, targetNoteId) => {
-            const url = getTargetUrl(targetNoteId, noteMeta);
+            const url = getNoteTargetUrl(targetNoteId, noteMeta);
+
+            return url ? `src="${url}"` : match;
+        });
+
+        content = content.replace(/src="[^"]*api\/attachments\/([a-zA-Z0-9_]+)\/image\/[^"]*"/g, (match, targetAttachmentId) => {
+            let url;
+
+            const attachmentMeta = noteMeta.attachments.find(attMeta => attMeta.attachmentId === targetAttachmentId);
+            if (attachmentMeta) {
+                // easy job here, because attachment will be in the same directory as the note's data file.
+                url = attachmentMeta.dataFileName;
+            } else {
+                log.info(`Could not find attachment meta object for attachmentId '${targetAttachmentId}'`);
+            }
 
             return url ? `src="${url}"` : match;
         });
 
         content = content.replace(/href="[^"]*#root[a-zA-Z0-9_\/]*\/([a-zA-Z0-9_]+)\/?"/g, (match, targetNoteId) => {
-            const url = getTargetUrl(targetNoteId, noteMeta);
+            const url = getNoteTargetUrl(targetNoteId, noteMeta);
 
             return url ? `href="${url}"` : match;
         });
@@ -297,7 +312,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
         if (['html', 'markdown'].includes(noteMeta.format)) {
             content = content.toString();
 
-            content = findLinks(content, noteMeta);
+            content = rewriteLinks(content, noteMeta);
         }
 
         if (noteMeta.format === 'html') {
@@ -347,7 +362,7 @@ ${markdownContent}`;
         log.info(`Exporting note '${noteMeta.noteId}'`);
 
         if (noteMeta.isClone) {
-            const targetUrl = getTargetUrl(noteMeta.noteId, noteMeta);
+            const targetUrl = getNoteTargetUrl(noteMeta.noteId, noteMeta);
 
             let content = `<p>This is a clone of a note. Go to its <a href="${targetUrl}">primary location</a>.</p>`;
 
@@ -404,7 +419,7 @@ ${markdownContent}`;
             const escapedTitle = utils.escapeHtml(`${meta.prefix ? `${meta.prefix} - ` : ''}${meta.title}`);
 
             if (meta.dataFileName) {
-                const targetUrl = getTargetUrl(meta.noteId, rootMeta);
+                const targetUrl = getNoteTargetUrl(meta.noteId, rootMeta);
 
                 html += `<a href="${targetUrl}" target="detail">${escapedTitle}</a>`;
             }
@@ -449,7 +464,7 @@ ${markdownContent}`;
 
         while (!firstNonEmptyNote) {
             if (curMeta.dataFileName) {
-                firstNonEmptyNote = getTargetUrl(curMeta.noteId, rootMeta);
+                firstNonEmptyNote = getNoteTargetUrl(curMeta.noteId, rootMeta);
             }
 
             if (curMeta.children && curMeta.children.length > 0) {
@@ -486,7 +501,7 @@ ${markdownContent}`;
     }
 
     const existingFileNames = format === 'html' ? ['navigation', 'index'] : [];
-    const rootMeta = getNoteMeta(branch, { notePath: [] }, existingFileNames);
+    const rootMeta = createNoteMeta(branch, { notePath: [] }, existingFileNames);
 
     const metaFile = {
         formatVersion: 1,

src/services/import/single.js

@@ -61,7 +61,7 @@ function importFile(taskContext, file, parentNote) {
 
 function importCodeNote(taskContext, file, parentNote) {
     const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
-    const content = file.buffer.toString("UTF-8");
+    const content = file.buffer.toString("utf-8");
     const detectedMime = mimeService.getMime(file.originalname) || file.mimetype;
     const mime = mimeService.normalizeMimeType(detectedMime);
 
@@ -81,7 +81,7 @@ function importCodeNote(taskContext, file, parentNote) {
 
 function importPlainText(taskContext, file, parentNote) {
     const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
-    const plainTextContent = file.buffer.toString("UTF-8");
+    const plainTextContent = file.buffer.toString("utf-8");
     const htmlContent = convertTextToHtml(plainTextContent);
 
     const {note} = noteService.createNewNote({
@@ -119,7 +119,7 @@ function convertTextToHtml(text) {
 function importMarkdown(taskContext, file, parentNote) {
     const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
 
-    const markdownContent = file.buffer.toString("UTF-8");
+    const markdownContent = file.buffer.toString("utf-8");
 
     const reader = new commonmark.Parser();
     const writer = new commonmark.HtmlRenderer();
@@ -158,7 +158,7 @@ function handleH1(content, title) {
 
 function importHtml(taskContext, file, parentNote) {
     const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
-    let content = file.buffer.toString("UTF-8");
+    let content = file.buffer.toString("utf-8");
 
     content = htmlSanitizer.sanitize(content);
 

src/services/import/zip.js

@@ -25,9 +25,11 @@ const BAttachment = require("../../becca/entities/battachment");
 async function importZip(taskContext, fileBuffer, importRootNote) {
     /** @type {Object.<string, string>} maps from original noteId (in ZIP file) to newly generated noteId */
     const noteIdMap = {};
+    /** @type {Object.<string, string>} maps from original attachmentId (in ZIP file) to newly generated attachmentId */
+    const attachmentIdMap = {};
     const attributes = [];
     // path => noteId, used only when meta file is not available
-    /** @type {Object.<string, string>} path => noteId */
+    /** @type {Object.<string, string>} path => noteId | attachmentId */
     const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId };
     const mdReader = new commonmark.Parser();
     const mdWriter = new commonmark.HtmlRenderer();
@@ -38,9 +40,9 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
     const createdNoteIds = new Set();
 
     function getNewNoteId(origNoteId) {
-        // in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution
         if (!origNoteId.trim()) {
-            return "";
+            // this probably shouldn't happen, but still good to have this precaution
+            return "empty_note_id";
         }
 
         if (origNoteId === 'root' || origNoteId.startsWith("_")) {
@@ -55,7 +57,40 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
         return noteIdMap[origNoteId];
     }
 
-    /** @returns {{noteMeta: NoteMeta, parentNoteMeta: NoteMeta, attachmentMeta: AttachmentMeta}} */
+    function getNewAttachmentId(origAttachmentId) {
+        if (!origAttachmentId.trim()) {
+            // this probably shouldn't happen, but still good to have this precaution
+            return "empty_attachment_id";
+        }
+
+        if (!attachmentIdMap[origAttachmentId]) {
+            attachmentIdMap[origAttachmentId] = utils.newEntityId();
+        }
+
+        return attachmentIdMap[origAttachmentId];
+    }
+
+    /**
+     * @param {NoteMeta} parentNoteMeta
+     * @param {string} dataFileName
+     */
+    function getAttachmentMeta(parentNoteMeta, dataFileName) {
+        for (const noteMeta of parentNoteMeta.children) {
+            for (const attachmentMeta of noteMeta.attachments || []) {
+                if (attachmentMeta.dataFileName === dataFileName) {
+                    return {
+                        parentNoteMeta,
+                        noteMeta,
+                        attachmentMeta
+                    };
+                }
+            }
+        }
+
+        return {};
+    }
+
+    /** @returns {{noteMeta: NoteMeta|undefined, parentNoteMeta: NoteMeta|undefined, attachmentMeta: AttachmentMeta|undefined}} */
     function getMeta(filePath) {
         if (!metaFile) {
             return {};
@@ -63,16 +98,17 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
 
         const pathSegments = filePath.split(/[\/\\]/g);
 
+        /** @type {NoteMeta} */
         let cursor = {
             isImportRoot: true,
             children: metaFile.files
         };
 
+        /** @type {NoteMeta} */
         let parent;
-        let attachmentMeta = false;
 
         for (const segment of pathSegments) {
-            if (!cursor || !cursor.children || cursor.children.length === 0) {
+            if (!cursor?.children?.length) {
                 return {};
             }
 
@@ -80,26 +116,13 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
             cursor = parent.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
 
             if (!cursor) {
-                for (const file of parent.children) {
+                return getAttachmentMeta(parent, segment);
-                    for (const attachment of file.attachments || []) {
-                        if (attachment.dataFileName === segment) {
-                            cursor = file;
-                            attachmentMeta = attachment;
-                            break;
-                        }
-                    }
-
-                    if (cursor) {
-                        break;
-                    }
-                }
             }
         }
 
         return {
             parentNoteMeta: parent,
-            noteMeta: cursor,
+            noteMeta: cursor
-            attachmentMeta
         };
     }
 
@@ -119,12 +142,10 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
 
             if (parentPath === '.') {
                 parentNoteId = importRootNote.noteId;
-            }
+            } else if (parentPath in createdPaths) {
-            else if (parentPath in createdPaths) {
                 parentNoteId = createdPaths[parentPath];
-            }
+            } else {
-            else {
+                // ZIP allows creating out of order records - i.e., file in a directory can appear in the ZIP stream before actual directory
-                // ZIP allows creating out of order records - i.e. file in a directory can appear in the ZIP stream before actual directory
                 parentNoteId = saveDirectory(parentPath);
             }
         }
@@ -142,6 +163,8 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
             return getNewNoteId(noteMeta.noteId);
         }
 
+        // in case we lack metadata, we treat e.g. "Programming.html" and "Programming" as the same note
+        // (one data file, the other directory for children)
         const filePathNoExt = utils.removeTextFileExtension(filePath);
 
         if (filePathNoExt in createdPaths) {
@@ -214,16 +237,15 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
         const { parentNoteMeta, noteMeta } = getMeta(filePath);
 
         const noteId = getNoteId(noteMeta, filePath);
-        const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
-        const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
 
-        let note = becca.getNote(noteId);
+        if (becca.getNote(noteId)) {
-
-        if (note) {
             return;
         }
 
-        ({note} = noteService.createNewNote({
+        const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
+        const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
+
+        const {note} = noteService.createNewNote({
             parentNoteId: parentNoteId,
             title: noteTitle,
             content: '',
@@ -234,20 +256,21 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
             isExpanded: noteMeta ? noteMeta.isExpanded : false,
             notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined,
             isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
-        }));
+        });
 
         createdNoteIds.add(note.noteId);
 
         saveAttributes(note, noteMeta);
 
-        if (!firstNote) {
+        firstNote = firstNote || note;
-            firstNote = note;
-        }
 
         return noteId;
     }
 
-    function getNoteIdFromRelativeUrl(url, filePath) {
+    /**
+     * @returns {{attachmentId: string}|{noteId: string}}
+     */
+    function getEntityIdFromRelativeUrl(url, filePath) {
         while (url.startsWith("./")) {
             url = url.substr(2);
         }
@@ -266,10 +289,17 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
 
         absUrl += `${absUrl.length > 0 ? '/' : ''}${url}`;
 
-        const {noteMeta} = getMeta(absUrl);
+        const { noteMeta, attachmentMeta } = getMeta(absUrl);
 
-        const targetNoteId = getNoteId(noteMeta, absUrl);
+        if (attachmentMeta) {
-        return targetNoteId;
+            return {
+                attachmentId: getNewAttachmentId(attachmentMeta.attachmentId)
+            };
+        } else { // don't check for noteMeta since it's not mandatory for notes
+            return {
+                noteId: getNoteId(noteMeta, absUrl)
+            };
+        }
     }
 
     /**
@@ -299,9 +329,9 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
 
         content = content.replace(/src="([^"]*)"/g, (match, url) => {
             try {
-                url = decodeURIComponent(url);
+                url = decodeURIComponent(url).trim();
             } catch (e) {
-                log.error(`Cannot parse image URL '${url}', keeping original (${e}).`);
+                log.error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`);
                 return `src="${url}"`;
             }
 
@@ -309,20 +339,22 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
                 return match;
             }
 
-            const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
+            const target = getEntityIdFromRelativeUrl(url, filePath);
 
-            if (!targetNoteId) {
+            if (target.noteId) {
+                return `src="api/images/${target.noteId}/${path.basename(url)}"`;
+            } else if (target.attachmentId) {
+                return `src="api/attachments/${target.attachmentId}/image/${path.basename(url)}"`;
+            } else {
                 return match;
             }
-
-            return `src="api/images/${targetNoteId}/${path.basename(url)}"`;
         });
 
         content = content.replace(/href="([^"]*)"/g, (match, url) => {
             try {
-                url = decodeURIComponent(url);
+                url = decodeURIComponent(url).trim();
             } catch (e) {
-                log.error(`Cannot parse link URL '${url}', keeping original (${e}).`);
+                log.error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`);
                 return `href="${url}"`;
             }
 
@@ -331,13 +363,15 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
                 return match;
             }
 
-            const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
+            const target = getEntityIdFromRelativeUrl(url, filePath);
 
-            if (!targetNoteId) {
+            if (!target.noteId) {
                 return match;
             }
 
-            return `href="#root/${targetNoteId}"`;
+            // FIXME for linking attachments
+
+            return `href="#root/${target.noteId}"`;
         });
 
         content = content.replace(/data-note-path="([^"]*)"/g, (match, notePath) => {
@@ -408,7 +442,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
      * @param {Buffer} content
      */
     function saveNote(filePath, content) {
-        const {parentNoteMeta, noteMeta, attachmentMeta} = getMeta(filePath);
+        const { parentNoteMeta, noteMeta, attachmentMeta } = getMeta(filePath);
 
         if (noteMeta?.noImport) {
             return;
@@ -418,10 +452,12 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
 
         if (attachmentMeta) {
             const attachment = new BAttachment({
+                attachmentId: getNewAttachmentId(attachmentMeta.attachmentId),
                 parentId: noteId,
                 title: attachmentMeta.title,
                 role: attachmentMeta.role,
-                mime: attachmentMeta.mime
+                mime: attachmentMeta.mime,
+                position: attachmentMeta.position
             });
 
             attachment.setContent(content);
@@ -448,11 +484,11 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
             return;
         }
 
-        let {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath);
+        let { type, mime } = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath);
         type = resolveNoteType(type);
 
         if (type !== 'file' && type !== 'image') {
-            content = content.toString("UTF-8");
+            content = content.toString("utf-8");
         }
 
         const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
@@ -496,7 +532,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
                 mime,
                 prefix: noteMeta ? noteMeta.prefix : '',
                 isExpanded: noteMeta ? noteMeta.isExpanded : false,
-                // root notePosition should be ignored since it relates to original document
+                // root notePosition should be ignored since it relates to the original document
                 // now import root should be placed after existing notes into new parent
                 notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined,
                 isProtected: isProtected,
@@ -506,13 +542,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
 
             saveAttributes(note, noteMeta);
 
-            if (!firstNote) {
+            firstNote = firstNote || note;
-                firstNote = note;
-            }
-
-            if (type === 'text') {
-                filePath = utils.removeTextFileExtension(filePath);
-            }
         }
 
         if (!noteMeta && (type === 'file' || type === 'image')) {
@@ -533,7 +563,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
         if (filePath === '!!!meta.json') {
             const content = await readContent(zipfile, entry);
 
-            metaFile = JSON.parse(content.toString("UTF-8"));
+            metaFile = JSON.parse(content.toString("utf-8"));
         }
 
         zipfile.readEntry();
@@ -597,6 +627,7 @@ function normalizeFilePath(filePath) {
     return filePath;
 }
 
+/** @returns {Promise<Buffer>} */
 function streamToBuffer(stream) {
     const chunks = [];
     stream.on('data', chunk => chunks.push(chunk));
@@ -604,7 +635,7 @@ function streamToBuffer(stream) {
     return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks))));
 }
 
-/** @returns {Buffer} */
+/** @returns {Promise<Buffer>} */
 function readContent(zipfile, entry) {
     return new Promise((res, rej) => {
         zipfile.openReadStream(entry, function(err, readStream) {
@@ -627,8 +658,6 @@ function readZipFile(buffer, processEntryCallback) {
 }
 
 function resolveNoteType(type) {
-    type = type || 'text';
-
     // BC for ZIPs created in Triliun 0.57 and older
     if (type === 'relation-map') {
         type = 'relationMap';
@@ -638,7 +667,7 @@ function resolveNoteType(type) {
         type = 'webView';
     }
 
-    return type;
+    return type || "text";
 }
 
 

src/services/meta/attachment_meta.js

@@ -7,6 +7,8 @@ class AttachmentMeta {
     role;
     /** @type {string} */
     mime;
+    /** @type {integer} */
+    position;
     /** @type {string} */
     dataFileName;
 }

src/services/meta/attribute_meta.js

@@ -3,7 +3,7 @@ class AttributeMeta {
     type;
     /** @type {string} */
     name;
-    /** @type {boolean} */
+    /** @type {string} */
     value;
     /** @type {boolean} */
     isInheritable;

src/services/sync.js

@@ -323,7 +323,7 @@ function getEntityChangeRow(entityName, entityId) {
 
         if (entityName === 'blobs' && entity.content !== null) {
             if (typeof entity.content === 'string') {
-                entity.content = Buffer.from(entity.content, 'UTF-8');
+                entity.content = Buffer.from(entity.content, 'utf-8');
             }
 
             entity.content = entity.content.toString("base64");

src/share/shaca/entities/snote.js

@@ -110,7 +110,7 @@ class SNote extends AbstractShacaEntity {
         if (this.hasStringContent()) {
             return content === null
                 ? ""
-                : content.toString("UTF-8");
+                : content.toString("utf-8");
         }
         else {
             return content;