fixes for zip export/import in regard to attachments

This commit is contained in:
zadam 2023-05-06 22:50:28 +02:00
parent a06ddc439d
commit 2aa987c072
11 changed files with 139 additions and 91 deletions

View File

@ -252,7 +252,7 @@ class AbstractBeccaEntity {
if (this.hasStringContent()) {
return content === null
? ""
: content.toString("UTF-8");
: content.toString("utf-8");
} else {
return content;
}

View File

@ -1120,7 +1120,8 @@ class BNote extends AbstractBeccaEntity {
SELECT attachments.*
FROM attachments
WHERE parentId = ?
AND isDeleted = 0`, [this.noteId])
AND isDeleted = 0
ORDER BY position`, [this.noteId])
.map(row => new BAttachment(row));
}
@ -1142,7 +1143,8 @@ class BNote extends AbstractBeccaEntity {
FROM attachments
WHERE parentId = ?
AND role = ?
AND isDeleted = 0`, [this.noteId, role])
AND isDeleted = 0
ORDER BY position`, [this.noteId, role])
.map(row => new BAttachment(row));
}

View File

@ -112,7 +112,7 @@ function decryptString(dataKey, cipherText) {
throw new Error("Could not decrypt string.");
}
return buffer.toString('UTF-8');
return buffer.toString('utf-8');
}
module.exports = {

View File

@ -34,7 +34,7 @@ function parseAuthToken(auth) {
// allow also basic auth format for systems which allow this type of authentication
// expect ETAPI token in the password field, require "etapi" username
// https://github.com/zadam/trilium/issues/3181
const basicAuthStr = utils.fromBase64(auth.substring(6)).toString("UTF-8");
const basicAuthStr = utils.fromBase64(auth.substring(6)).toString("utf-8");
const basicAuthChunks = basicAuthStr.split(":");
if (basicAuthChunks.length !== 2) {

View File

@ -122,7 +122,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
* @param {Object.<string, integer>} existingFileNames
* @returns {NoteMeta|null}
*/
function getNoteMeta(branch, parentMeta, existingFileNames) {
function createNoteMeta(branch, parentMeta, existingFileNames) {
const note = branch.getNote();
if (note.hasOwnedLabel('excludeFromExport')) {
@ -200,6 +200,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
attMeta.title = attachment.title;
attMeta.role = attachment.role;
attMeta.mime = attachment.mime;
attMeta.position = attachment.position;
attMeta.dataFileName = getDataFileName(
null,
attachment.mime,
@ -217,7 +218,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
const childExistingNames = {};
for (const childBranch of childBranches) {
const note = getNoteMeta(childBranch, meta, childExistingNames);
const note = createNoteMeta(childBranch, meta, childExistingNames);
// can be undefined if export is disabled for this note
if (note) {
@ -234,7 +235,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
* @param {NoteMeta} sourceMeta
* @return {string|null}
*/
function getTargetUrl(targetNoteId, sourceMeta) {
function getNoteTargetUrl(targetNoteId, sourceMeta) {
const targetMeta = noteIdToMeta[targetNoteId];
if (!targetMeta) {
@ -271,15 +272,29 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
* @param {NoteMeta} noteMeta
* @return {string}
*/
function findLinks(content, noteMeta) {
function rewriteLinks(content, noteMeta) {
content = content.replace(/src="[^"]*api\/images\/([a-zA-Z0-9_]+)\/[^"]*"/g, (match, targetNoteId) => {
const url = getTargetUrl(targetNoteId, noteMeta);
const url = getNoteTargetUrl(targetNoteId, noteMeta);
return url ? `src="${url}"` : match;
});
content = content.replace(/src="[^"]*api\/attachments\/([a-zA-Z0-9_]+)\/image\/[^"]*"/g, (match, targetAttachmentId) => {
let url;
const attachmentMeta = noteMeta.attachments.find(attMeta => attMeta.attachmentId === targetAttachmentId);
if (attachmentMeta) {
// easy job here, because attachment will be in the same directory as the note's data file.
url = attachmentMeta.dataFileName;
} else {
log.info(`Could not find attachment meta object for attachmentId '${targetAttachmentId}'`);
}
return url ? `src="${url}"` : match;
});
content = content.replace(/href="[^"]*#root[a-zA-Z0-9_\/]*\/([a-zA-Z0-9_]+)\/?"/g, (match, targetNoteId) => {
const url = getTargetUrl(targetNoteId, noteMeta);
const url = getNoteTargetUrl(targetNoteId, noteMeta);
return url ? `href="${url}"` : match;
});
@ -297,7 +312,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
if (['html', 'markdown'].includes(noteMeta.format)) {
content = content.toString();
content = findLinks(content, noteMeta);
content = rewriteLinks(content, noteMeta);
}
if (noteMeta.format === 'html') {
@ -347,7 +362,7 @@ ${markdownContent}`;
log.info(`Exporting note '${noteMeta.noteId}'`);
if (noteMeta.isClone) {
const targetUrl = getTargetUrl(noteMeta.noteId, noteMeta);
const targetUrl = getNoteTargetUrl(noteMeta.noteId, noteMeta);
let content = `<p>This is a clone of a note. Go to its <a href="${targetUrl}">primary location</a>.</p>`;
@ -404,7 +419,7 @@ ${markdownContent}`;
const escapedTitle = utils.escapeHtml(`${meta.prefix ? `${meta.prefix} - ` : ''}${meta.title}`);
if (meta.dataFileName) {
const targetUrl = getTargetUrl(meta.noteId, rootMeta);
const targetUrl = getNoteTargetUrl(meta.noteId, rootMeta);
html += `<a href="${targetUrl}" target="detail">${escapedTitle}</a>`;
}
@ -449,7 +464,7 @@ ${markdownContent}`;
while (!firstNonEmptyNote) {
if (curMeta.dataFileName) {
firstNonEmptyNote = getTargetUrl(curMeta.noteId, rootMeta);
firstNonEmptyNote = getNoteTargetUrl(curMeta.noteId, rootMeta);
}
if (curMeta.children && curMeta.children.length > 0) {
@ -486,7 +501,7 @@ ${markdownContent}`;
}
const existingFileNames = format === 'html' ? ['navigation', 'index'] : [];
const rootMeta = getNoteMeta(branch, { notePath: [] }, existingFileNames);
const rootMeta = createNoteMeta(branch, { notePath: [] }, existingFileNames);
const metaFile = {
formatVersion: 1,

View File

@ -61,7 +61,7 @@ function importFile(taskContext, file, parentNote) {
function importCodeNote(taskContext, file, parentNote) {
const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
const content = file.buffer.toString("UTF-8");
const content = file.buffer.toString("utf-8");
const detectedMime = mimeService.getMime(file.originalname) || file.mimetype;
const mime = mimeService.normalizeMimeType(detectedMime);
@ -81,7 +81,7 @@ function importCodeNote(taskContext, file, parentNote) {
function importPlainText(taskContext, file, parentNote) {
const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
const plainTextContent = file.buffer.toString("UTF-8");
const plainTextContent = file.buffer.toString("utf-8");
const htmlContent = convertTextToHtml(plainTextContent);
const {note} = noteService.createNewNote({
@ -119,7 +119,7 @@ function convertTextToHtml(text) {
function importMarkdown(taskContext, file, parentNote) {
const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
const markdownContent = file.buffer.toString("UTF-8");
const markdownContent = file.buffer.toString("utf-8");
const reader = new commonmark.Parser();
const writer = new commonmark.HtmlRenderer();
@ -158,7 +158,7 @@ function handleH1(content, title) {
function importHtml(taskContext, file, parentNote) {
const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces);
let content = file.buffer.toString("UTF-8");
let content = file.buffer.toString("utf-8");
content = htmlSanitizer.sanitize(content);

View File

@ -25,9 +25,11 @@ const BAttachment = require("../../becca/entities/battachment");
async function importZip(taskContext, fileBuffer, importRootNote) {
/** @type {Object.<string, string>} maps from original noteId (in ZIP file) to newly generated noteId */
const noteIdMap = {};
/** @type {Object.<string, string>} maps from original attachmentId (in ZIP file) to newly generated attachmentId */
const attachmentIdMap = {};
const attributes = [];
// path => noteId, used only when meta file is not available
/** @type {Object.<string, string>} path => noteId */
/** @type {Object.<string, string>} path => noteId | attachmentId */
const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId };
const mdReader = new commonmark.Parser();
const mdWriter = new commonmark.HtmlRenderer();
@ -38,9 +40,9 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
const createdNoteIds = new Set();
function getNewNoteId(origNoteId) {
// in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution
if (!origNoteId.trim()) {
return "";
// this probably shouldn't happen, but still good to have this precaution
return "empty_note_id";
}
if (origNoteId === 'root' || origNoteId.startsWith("_")) {
@ -55,7 +57,40 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
return noteIdMap[origNoteId];
}
/** @returns {{noteMeta: NoteMeta, parentNoteMeta: NoteMeta, attachmentMeta: AttachmentMeta}} */
function getNewAttachmentId(origAttachmentId) {
if (!origAttachmentId.trim()) {
// this probably shouldn't happen, but still good to have this precaution
return "empty_attachment_id";
}
if (!attachmentIdMap[origAttachmentId]) {
attachmentIdMap[origAttachmentId] = utils.newEntityId();
}
return attachmentIdMap[origAttachmentId];
}
/**
* @param {NoteMeta} parentNoteMeta
* @param {string} dataFileName
*/
function getAttachmentMeta(parentNoteMeta, dataFileName) {
for (const noteMeta of parentNoteMeta.children) {
for (const attachmentMeta of noteMeta.attachments || []) {
if (attachmentMeta.dataFileName === dataFileName) {
return {
parentNoteMeta,
noteMeta,
attachmentMeta
};
}
}
}
return {};
}
/** @returns {{noteMeta: NoteMeta|undefined, parentNoteMeta: NoteMeta|undefined, attachmentMeta: AttachmentMeta|undefined}} */
function getMeta(filePath) {
if (!metaFile) {
return {};
@ -63,16 +98,17 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
const pathSegments = filePath.split(/[\/\\]/g);
/** @type {NoteMeta} */
let cursor = {
isImportRoot: true,
children: metaFile.files
};
/** @type {NoteMeta} */
let parent;
let attachmentMeta = false;
for (const segment of pathSegments) {
if (!cursor || !cursor.children || cursor.children.length === 0) {
if (!cursor?.children?.length) {
return {};
}
@ -80,26 +116,13 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
cursor = parent.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
if (!cursor) {
for (const file of parent.children) {
for (const attachment of file.attachments || []) {
if (attachment.dataFileName === segment) {
cursor = file;
attachmentMeta = attachment;
break;
}
}
if (cursor) {
break;
}
}
return getAttachmentMeta(parent, segment);
}
}
return {
parentNoteMeta: parent,
noteMeta: cursor,
attachmentMeta
noteMeta: cursor
};
}
@ -119,12 +142,10 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
if (parentPath === '.') {
parentNoteId = importRootNote.noteId;
}
else if (parentPath in createdPaths) {
} else if (parentPath in createdPaths) {
parentNoteId = createdPaths[parentPath];
}
else {
// ZIP allows creating out of order records - i.e. file in a directory can appear in the ZIP stream before actual directory
} else {
// ZIP allows creating out of order records - i.e., file in a directory can appear in the ZIP stream before actual directory
parentNoteId = saveDirectory(parentPath);
}
}
@ -142,6 +163,8 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
return getNewNoteId(noteMeta.noteId);
}
// in case we lack metadata, we treat e.g. "Programming.html" and "Programming" as the same note
// (one data file, the other directory for children)
const filePathNoExt = utils.removeTextFileExtension(filePath);
if (filePathNoExt in createdPaths) {
@ -214,16 +237,15 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
const { parentNoteMeta, noteMeta } = getMeta(filePath);
const noteId = getNoteId(noteMeta, filePath);
const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
let note = becca.getNote(noteId);
if (note) {
if (becca.getNote(noteId)) {
return;
}
({note} = noteService.createNewNote({
const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
const {note} = noteService.createNewNote({
parentNoteId: parentNoteId,
title: noteTitle,
content: '',
@ -234,20 +256,21 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
isExpanded: noteMeta ? noteMeta.isExpanded : false,
notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined,
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
}));
});
createdNoteIds.add(note.noteId);
saveAttributes(note, noteMeta);
if (!firstNote) {
firstNote = note;
}
firstNote = firstNote || note;
return noteId;
}
function getNoteIdFromRelativeUrl(url, filePath) {
/**
* @returns {{attachmentId: string}|{noteId: string}}
*/
function getEntityIdFromRelativeUrl(url, filePath) {
while (url.startsWith("./")) {
url = url.substr(2);
}
@ -266,10 +289,17 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
absUrl += `${absUrl.length > 0 ? '/' : ''}${url}`;
const {noteMeta} = getMeta(absUrl);
const { noteMeta, attachmentMeta } = getMeta(absUrl);
const targetNoteId = getNoteId(noteMeta, absUrl);
return targetNoteId;
if (attachmentMeta) {
return {
attachmentId: getNewAttachmentId(attachmentMeta.attachmentId)
};
} else { // don't check for noteMeta since it's not mandatory for notes
return {
noteId: getNoteId(noteMeta, absUrl)
};
}
}
/**
@ -299,9 +329,9 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
content = content.replace(/src="([^"]*)"/g, (match, url) => {
try {
url = decodeURIComponent(url);
url = decodeURIComponent(url).trim();
} catch (e) {
log.error(`Cannot parse image URL '${url}', keeping original (${e}).`);
log.error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`);
return `src="${url}"`;
}
@ -309,20 +339,22 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
return match;
}
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
const target = getEntityIdFromRelativeUrl(url, filePath);
if (!targetNoteId) {
if (target.noteId) {
return `src="api/images/${target.noteId}/${path.basename(url)}"`;
} else if (target.attachmentId) {
return `src="api/attachments/${target.attachmentId}/image/${path.basename(url)}"`;
} else {
return match;
}
return `src="api/images/${targetNoteId}/${path.basename(url)}"`;
});
content = content.replace(/href="([^"]*)"/g, (match, url) => {
try {
url = decodeURIComponent(url);
url = decodeURIComponent(url).trim();
} catch (e) {
log.error(`Cannot parse link URL '${url}', keeping original (${e}).`);
log.error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`);
return `href="${url}"`;
}
@ -331,13 +363,15 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
return match;
}
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
const target = getEntityIdFromRelativeUrl(url, filePath);
if (!targetNoteId) {
if (!target.noteId) {
return match;
}
return `href="#root/${targetNoteId}"`;
// FIXME for linking attachments
return `href="#root/${target.noteId}"`;
});
content = content.replace(/data-note-path="([^"]*)"/g, (match, notePath) => {
@ -408,7 +442,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
* @param {Buffer} content
*/
function saveNote(filePath, content) {
const {parentNoteMeta, noteMeta, attachmentMeta} = getMeta(filePath);
const { parentNoteMeta, noteMeta, attachmentMeta } = getMeta(filePath);
if (noteMeta?.noImport) {
return;
@ -418,10 +452,12 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
if (attachmentMeta) {
const attachment = new BAttachment({
attachmentId: getNewAttachmentId(attachmentMeta.attachmentId),
parentId: noteId,
title: attachmentMeta.title,
role: attachmentMeta.role,
mime: attachmentMeta.mime
mime: attachmentMeta.mime,
position: attachmentMeta.position
});
attachment.setContent(content);
@ -448,11 +484,11 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
return;
}
let {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath);
let { type, mime } = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath);
type = resolveNoteType(type);
if (type !== 'file' && type !== 'image') {
content = content.toString("UTF-8");
content = content.toString("utf-8");
}
const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
@ -496,7 +532,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
mime,
prefix: noteMeta ? noteMeta.prefix : '',
isExpanded: noteMeta ? noteMeta.isExpanded : false,
// root notePosition should be ignored since it relates to original document
// root notePosition should be ignored since it relates to the original document
// now import root should be placed after existing notes into new parent
notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined,
isProtected: isProtected,
@ -506,13 +542,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
saveAttributes(note, noteMeta);
if (!firstNote) {
firstNote = note;
}
if (type === 'text') {
filePath = utils.removeTextFileExtension(filePath);
}
firstNote = firstNote || note;
}
if (!noteMeta && (type === 'file' || type === 'image')) {
@ -533,7 +563,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
if (filePath === '!!!meta.json') {
const content = await readContent(zipfile, entry);
metaFile = JSON.parse(content.toString("UTF-8"));
metaFile = JSON.parse(content.toString("utf-8"));
}
zipfile.readEntry();
@ -597,6 +627,7 @@ function normalizeFilePath(filePath) {
return filePath;
}
/** @returns {Promise<Buffer>} */
function streamToBuffer(stream) {
const chunks = [];
stream.on('data', chunk => chunks.push(chunk));
@ -604,7 +635,7 @@ function streamToBuffer(stream) {
return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks))));
}
/** @returns {Buffer} */
/** @returns {Promise<Buffer>} */
function readContent(zipfile, entry) {
return new Promise((res, rej) => {
zipfile.openReadStream(entry, function(err, readStream) {
@ -627,8 +658,6 @@ function readZipFile(buffer, processEntryCallback) {
}
function resolveNoteType(type) {
type = type || 'text';
// BC for ZIPs created in Triliun 0.57 and older
if (type === 'relation-map') {
type = 'relationMap';
@ -638,7 +667,7 @@ function resolveNoteType(type) {
type = 'webView';
}
return type;
return type || "text";
}

View File

@ -7,6 +7,8 @@ class AttachmentMeta {
role;
/** @type {string} */
mime;
/** @type {integer} */
position;
/** @type {string} */
dataFileName;
}

View File

@ -3,7 +3,7 @@ class AttributeMeta {
type;
/** @type {string} */
name;
/** @type {boolean} */
/** @type {string} */
value;
/** @type {boolean} */
isInheritable;

View File

@ -323,7 +323,7 @@ function getEntityChangeRow(entityName, entityId) {
if (entityName === 'blobs' && entity.content !== null) {
if (typeof entity.content === 'string') {
entity.content = Buffer.from(entity.content, 'UTF-8');
entity.content = Buffer.from(entity.content, 'utf-8');
}
entity.content = entity.content.toString("base64");

View File

@ -110,7 +110,7 @@ class SNote extends AbstractShacaEntity {
if (this.hasStringContent()) {
return content === null
? ""
: content.toString("UTF-8");
: content.toString("utf-8");
}
else {
return content;