introducing meta types for import/export, fixes for attachments export

This commit is contained in:
zadam 2023-05-06 14:38:45 +02:00
parent fa3cbb4645
commit f6944b8219
16 changed files with 183 additions and 75 deletions

View File

@ -45,7 +45,7 @@ class BAttachment extends AbstractBeccaEntity {
this.mime = row.mime; this.mime = row.mime;
/** @type {string} */ /** @type {string} */
this.title = row.title; this.title = row.title;
/** @type {number} */ /** @type {integer} */
this.position = row.position; this.position = row.position;
/** @type {string} */ /** @type {string} */
this.blobId = row.blobId; this.blobId = row.blobId;

View File

@ -51,7 +51,7 @@ class BAttribute extends AbstractBeccaEntity {
this.type = type; this.type = type;
/** @type {string} */ /** @type {string} */
this.name = name; this.name = name;
/** @type {int} */ /** @type {integer} */
this.position = position; this.position = position;
/** @type {string} */ /** @type {string} */
this.value = value || ""; this.value = value || "";

View File

@ -4,7 +4,7 @@ class BBlob {
this.blobId = row.blobId; this.blobId = row.blobId;
/** @type {string|Buffer} */ /** @type {string|Buffer} */
this.content = row.content; this.content = row.content;
/** @type {number} */ /** @type {integer} */
this.contentLength = row.contentLength; this.contentLength = row.contentLength;
/** @type {string} */ /** @type {string} */
this.dateModified = row.dateModified; this.dateModified = row.dateModified;

View File

@ -55,7 +55,7 @@ class BBranch extends AbstractBeccaEntity {
this.parentNoteId = parentNoteId; this.parentNoteId = parentNoteId;
/** @type {string|null} */ /** @type {string|null} */
this.prefix = prefix; this.prefix = prefix;
/** @type {int} */ /** @type {integer} */
this.notePosition = notePosition; this.notePosition = notePosition;
/** @type {boolean} */ /** @type {boolean} */
this.isExpanded = !!isExpanded; this.isExpanded = !!isExpanded;

View File

@ -46,7 +46,7 @@ class BNoteRevision extends AbstractBeccaEntity {
this.utcDateCreated = row.utcDateCreated; this.utcDateCreated = row.utcDateCreated;
/** @type {string} */ /** @type {string} */
this.utcDateModified = row.utcDateModified; this.utcDateModified = row.utcDateModified;
/** @type {number} */ /** @type {integer} */
this.contentLength = row.contentLength; this.contentLength = row.contentLength;
if (this.isProtected && !titleDecrypted) { if (this.isProtected && !titleDecrypted) {

View File

@ -22,7 +22,7 @@ class FAttribute {
this.name = row.name; this.name = row.name;
/** @type {string} */ /** @type {string} */
this.value = row.value; this.value = row.value;
/** @type {int} */ /** @type {integer} */
this.position = row.position; this.position = row.position;
/** @type {boolean} */ /** @type {boolean} */
this.isInheritable = !!row.isInheritable; this.isInheritable = !!row.isInheritable;

View File

@ -19,7 +19,7 @@ class FBranch {
this.noteId = row.noteId; this.noteId = row.noteId;
/** @type {string} */ /** @type {string} */
this.parentNoteId = row.parentNoteId; this.parentNoteId = row.parentNoteId;
/** @type {int} */ /** @type {integer} */
this.notePosition = row.notePosition; this.notePosition = row.notePosition;
/** @type {string} */ /** @type {string} */
this.prefix = row.prefix; this.prefix = row.prefix;

View File

@ -478,7 +478,7 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
* Return randomly generated string of given length. This random string generation is NOT cryptographically secure. * Return randomly generated string of given length. This random string generation is NOT cryptographically secure.
* *
* @method * @method
* @param {number} length of the string * @param {integer} length of the string
* @returns {string} random string * @returns {string} random string
*/ */
this.randomString = utils.randomString; this.randomString = utils.randomString;

View File

@ -35,7 +35,7 @@ function buildDescendantCountMap(noteIdsToCount) {
} }
/** /**
* @param {BNote} note * @param {BNote} note
* @param {int} depth * @param {integer} depth
* @returns {string[]} noteIds * @returns {string[]} noteIds
*/ */
function getNeighbors(note, depth) { function getNeighbors(note, depth) {

View File

@ -215,7 +215,7 @@ function BackendScriptApi(currentNote, apiParams) {
* @property {boolean} [params.isProtected=false] * @property {boolean} [params.isProtected=false]
* @property {boolean} [params.isExpanded=false] * @property {boolean} [params.isExpanded=false]
* @property {string} [params.prefix=''] * @property {string} [params.prefix='']
* @property {int} [params.notePosition] - default is last existing notePosition in a parent + 10 * @property {integer} [params.notePosition] - default is last existing notePosition in a parent + 10
* @returns {{note: BNote, branch: BBranch}} object contains newly created entities note and branch * @returns {{note: BNote, branch: BBranch}} object contains newly created entities note and branch
*/ */
this.createNewNote = noteService.createNewNote; this.createNewNote = noteService.createNewNote;
@ -412,7 +412,7 @@ function BackendScriptApi(currentNote, apiParams) {
* Return randomly generated string of given length. This random string generation is NOT cryptographically secure. * Return randomly generated string of given length. This random string generation is NOT cryptographically secure.
* *
* @method * @method
* @param {number} length of the string * @param {integer} length of the string
* @returns {string} random string * @returns {string} random string
*/ */
this.randomString = utils.randomString; this.randomString = utils.randomString;

View File

@ -16,6 +16,9 @@ const archiver = require('archiver');
const log = require("../log"); const log = require("../log");
const TaskContext = require("../task_context"); const TaskContext = require("../task_context");
const ValidationError = require("../../errors/validation_error"); const ValidationError = require("../../errors/validation_error");
const NoteMeta = require("../meta/note_meta");
const AttachmentMeta = require("../meta/attachment_meta");
const AttributeMeta = require("../meta/attribute_meta");
/** /**
* @param {TaskContext} taskContext * @param {TaskContext} taskContext
@ -33,8 +36,14 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
zlib: { level: 9 } // Sets the compression level. zlib: { level: 9 } // Sets the compression level.
}); });
/** @type {Object.<string, NoteMeta>} */
const noteIdToMeta = {}; const noteIdToMeta = {};
/**
* @param {Object.<string, integer>} existingFileNames
* @param {string} fileName
* @returns {string}
*/
function getUniqueFilename(existingFileNames, fileName) { function getUniqueFilename(existingFileNames, fileName) {
const lcFileName = fileName.toLowerCase(); const lcFileName = fileName.toLowerCase();
@ -51,13 +60,20 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
return `${index}_${fileName}`; return `${index}_${fileName}`;
} }
else { else {[]
existingFileNames[lcFileName] = 1; existingFileNames[lcFileName] = 1;
return fileName; return fileName;
} }
} }
/**
* @param {string|null} type
* @param {string} mime
* @param {string} baseFileName
* @param {Object.<string, integer>} existingFileNames
* @return {string}
*/
function getDataFileName(type, mime, baseFileName, existingFileNames) { function getDataFileName(type, mime, baseFileName, existingFileNames) {
let fileName = baseFileName; let fileName = baseFileName;
@ -68,7 +84,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
fileName = fileName.substr(0, 30); fileName = fileName.substr(0, 30);
} }
// following two are handled specifically since we always want to have these extensions no matter the automatic detection // the following two are handled specifically since we always want to have these extensions no matter the automatic detection
// and/or existing detected extensions in the note name // and/or existing detected extensions in the note name
if (type === 'text' && format === 'markdown') { if (type === 'text' && format === 'markdown') {
newExtension = 'md'; newExtension = 'md';
@ -100,18 +116,24 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
return getUniqueFilename(existingFileNames, fileName); return getUniqueFilename(existingFileNames, fileName);
} }
/**
* @param {BBranch} branch
* @param {NoteMeta} parentMeta
* @param {Object.<string, integer>} existingFileNames
* @returns {NoteMeta|null}
*/
function getNoteMeta(branch, parentMeta, existingFileNames) { function getNoteMeta(branch, parentMeta, existingFileNames) {
const note = branch.getNote(); const note = branch.getNote();
if (note.hasOwnedLabel('excludeFromExport')) { if (note.hasOwnedLabel('excludeFromExport')) {
return; return null;
} }
const title = note.getTitleOrProtected(); const title = note.getTitleOrProtected();
const completeTitle = branch.prefix ? (`${branch.prefix} - ${title}`) : title; const completeTitle = branch.prefix ? (`${branch.prefix} - ${title}`) : title;
let baseFileName = sanitize(completeTitle); let baseFileName = sanitize(completeTitle);
if (baseFileName.length > 200) { // actual limit is 256 bytes(!) but let's be conservative if (baseFileName.length > 200) { // the actual limit is 256 bytes(!) but let's be conservative
baseFileName = baseFileName.substr(0, 200); baseFileName = baseFileName.substr(0, 200);
} }
@ -120,37 +142,37 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
if (note.noteId in noteIdToMeta) { if (note.noteId in noteIdToMeta) {
const fileName = getUniqueFilename(existingFileNames, `${baseFileName}.clone.${format === 'html' ? 'html' : 'md'}`); const fileName = getUniqueFilename(existingFileNames, `${baseFileName}.clone.${format === 'html' ? 'html' : 'md'}`);
return { const meta = new NoteMeta();
isClone: true, meta.isClone = true;
noteId: note.noteId, meta.noteId = note.noteId;
notePath: notePath, meta.notePath = notePath;
title: note.getTitleOrProtected(), meta.title = note.getTitleOrProtected();
prefix: branch.prefix, meta.prefix = branch.prefix;
dataFileName: fileName, meta.dataFileName = fileName;
type: 'text', // export will have text description, meta.type = 'text'; // export will have text description
format: format meta.format = format;
}; return meta;
} }
const meta = { const meta = new NoteMeta();
isClone: false, meta.isClone = false;
noteId: note.noteId, meta.noteId = note.noteId;
notePath: notePath, meta.notePath = notePath;
title: note.getTitleOrProtected(), meta.title = note.getTitleOrProtected();
notePosition: branch.notePosition, meta.notePosition = branch.notePosition;
prefix: branch.prefix, meta.prefix = branch.prefix;
isExpanded: branch.isExpanded, meta.isExpanded = branch.isExpanded;
type: note.type, meta.type = note.type;
mime: note.mime, meta.mime = note.mime;
// we don't export utcDateCreated and utcDateModified of any entity since that would be a bit misleading meta.attributes = note.getOwnedAttributes().map(attribute => {
attributes: note.getOwnedAttributes().map(attribute => ({ const attrMeta = new AttributeMeta();
type: attribute.type, attrMeta.type = attribute.type;
name: attribute.name, attrMeta.name = attribute.name;
value: attribute.value, attrMeta.value = attribute.value;
isInheritable: attribute.isInheritable, attrMeta.isInheritable = attribute.isInheritable;
position: attribute.position attrMeta.position = attribute.position;
})) return attrMeta;
}; });
taskContext.increaseProgressCount(); taskContext.increaseProgressCount();
@ -165,27 +187,27 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
const available = !note.isProtected || protectedSessionService.isProtectedSessionAvailable(); const available = !note.isProtected || protectedSessionService.isProtectedSessionAvailable();
// if it's a leaf then we'll export it even if it's empty // if it's a leaf, then we'll export it even if it's empty
if (available && (note.getContent().length > 0 || childBranches.length === 0)) { if (available && (note.getContent().length > 0 || childBranches.length === 0)) {
meta.dataFileName = getDataFileName(note.type, note.mime, baseFileName, existingFileNames); meta.dataFileName = getDataFileName(note.type, note.mime, baseFileName, existingFileNames);
} }
const attachments = note.getAttachments(); const attachments = note.getAttachments();
meta.attachments = attachments
if (attachments.length > 0) { .map(attachment => {
meta.attachments = attachments const attMeta = new AttachmentMeta();
.map(attachment => ({ attMeta.attachmentId = attachment.attachmentId;
title: attachment.title, attMeta.title = attachment.title;
role: attachment.role, attMeta.role = attachment.role;
mime: attachment.mime, attMeta.mime = attachment.mime;
dataFileName: getDataFileName( attMeta.dataFileName = getDataFileName(
null, null,
attachment.mime, attachment.mime,
baseFileName + "_" + attachment.title, baseFileName + "_" + attachment.title,
existingFileNames existingFileNames
) );
})); return attMeta;
} });
if (childBranches.length > 0) { if (childBranches.length > 0) {
meta.dirFileName = getUniqueFilename(existingFileNames, baseFileName); meta.dirFileName = getUniqueFilename(existingFileNames, baseFileName);
@ -207,6 +229,11 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
return meta; return meta;
} }
/**
* @param {string} targetNoteId
* @param {NoteMeta} sourceMeta
* @return {string|null}
*/
function getTargetUrl(targetNoteId, sourceMeta) { function getTargetUrl(targetNoteId, sourceMeta) {
const targetMeta = noteIdToMeta[targetNoteId]; const targetMeta = noteIdToMeta[targetNoteId];
@ -217,7 +244,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
const targetPath = targetMeta.notePath.slice(); const targetPath = targetMeta.notePath.slice();
const sourcePath = sourceMeta.notePath.slice(); const sourcePath = sourceMeta.notePath.slice();
// > 1 for edge case that targetPath and sourcePath are exact same (link to itself) // > 1 for the edge case that targetPath and sourcePath are exact same (a link to itself)
while (targetPath.length > 1 && sourcePath.length > 1 && targetPath[0] === sourcePath[0]) { while (targetPath.length > 1 && sourcePath.length > 1 && targetPath[0] === sourcePath[0]) {
targetPath.shift(); targetPath.shift();
sourcePath.shift(); sourcePath.shift();
@ -233,12 +260,17 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
const meta = noteIdToMeta[targetPath[targetPath.length - 1]]; const meta = noteIdToMeta[targetPath[targetPath.length - 1]];
// link can target note which is only "folder-note" and as such will not have a file in an export // link can target note which is only "folder-note" and as such, will not have a file in an export
url += encodeURIComponent(meta.dataFileName || meta.dirFileName); url += encodeURIComponent(meta.dataFileName || meta.dirFileName);
return url; return url;
} }
/**
* @param {string} content
* @param {NoteMeta} noteMeta
* @return {string}
*/
function findLinks(content, noteMeta) { function findLinks(content, noteMeta) {
content = content.replace(/src="[^"]*api\/images\/([a-zA-Z0-9_]+)\/[^"]*"/g, (match, targetNoteId) => { content = content.replace(/src="[^"]*api\/images\/([a-zA-Z0-9_]+)\/[^"]*"/g, (match, targetNoteId) => {
const url = getTargetUrl(targetNoteId, noteMeta); const url = getTargetUrl(targetNoteId, noteMeta);
@ -255,6 +287,12 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
return content; return content;
} }
/**
* @param {string} title
* @param {string|Buffer} content
* @param {NoteMeta} noteMeta
* @return {string|Buffer}
*/
function prepareContent(title, content, noteMeta) { function prepareContent(title, content, noteMeta) {
if (['html', 'markdown'].includes(noteMeta.format)) { if (['html', 'markdown'].includes(noteMeta.format)) {
content = content.toString(); content = content.toString();
@ -287,8 +325,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true)
return content.length < 100000 return content.length < 100000
? html.prettyPrint(content, {indent_size: 2}) ? html.prettyPrint(content, {indent_size: 2})
: content; : content;
} } else if (noteMeta.format === 'markdown') {
else if (noteMeta.format === 'markdown') {
let markdownContent = mdService.toMarkdown(content); let markdownContent = mdService.toMarkdown(content);
if (markdownContent.trim().length > 0 && !markdownContent.startsWith("# ")) { if (markdownContent.trim().length > 0 && !markdownContent.startsWith("# ")) {
@ -297,17 +334,17 @@ ${markdownContent}`;
} }
return markdownContent; return markdownContent;
} } else {
else {
return content; return content;
} }
} }
// noteId => file path /**
const notePaths = {}; * @param {NoteMeta} noteMeta
* @param {string} filePathPrefix
*/
function saveNote(noteMeta, filePathPrefix) { function saveNote(noteMeta, filePathPrefix) {
log.info(`Exporting note ${noteMeta.noteId}`); log.info(`Exporting note '${noteMeta.noteId}'`);
if (noteMeta.isClone) { if (noteMeta.isClone) {
const targetUrl = getTargetUrl(noteMeta.noteId, noteMeta); const targetUrl = getTargetUrl(noteMeta.noteId, noteMeta);
@ -323,8 +360,6 @@ ${markdownContent}`;
const note = becca.getNote(noteMeta.noteId); const note = becca.getNote(noteMeta.noteId);
notePaths[note.noteId] = filePathPrefix + (noteMeta.dataFileName || noteMeta.dirFileName);
if (noteMeta.dataFileName) { if (noteMeta.dataFileName) {
const content = prepareContent(noteMeta.title, note.getContent(), noteMeta); const content = prepareContent(noteMeta.title, note.getContent(), noteMeta);
@ -336,9 +371,8 @@ ${markdownContent}`;
taskContext.increaseProgressCount(); taskContext.increaseProgressCount();
for (const attachmentMeta of noteMeta.attachments || []) { for (const attachmentMeta of noteMeta.attachments) {
// FIXME const attachment = note.getAttachmentById(attachmentMeta.attachmentId);
const attachment = note.getAttachmentByName(attachmentMeta.name);
const content = attachment.getContent(); const content = attachment.getContent();
archive.append(content, { archive.append(content, {
@ -347,7 +381,7 @@ ${markdownContent}`;
}); });
} }
if (noteMeta.children && noteMeta.children.length > 0) { if (noteMeta.children?.length > 0) {
const directoryPath = filePathPrefix + noteMeta.dirFileName; const directoryPath = filePathPrefix + noteMeta.dirFileName;
// create directory // create directory
@ -359,6 +393,10 @@ ${markdownContent}`;
} }
} }
/**
* @param {NoteMeta} rootMeta
* @param {NoteMeta} navigationMeta
*/
function saveNavigation(rootMeta, navigationMeta) { function saveNavigation(rootMeta, navigationMeta) {
function saveNavigationInner(meta) { function saveNavigationInner(meta) {
let html = '<li>'; let html = '<li>';
@ -401,6 +439,10 @@ ${markdownContent}`;
archive.append(prettyHtml, { name: navigationMeta.dataFileName }); archive.append(prettyHtml, { name: navigationMeta.dataFileName });
} }
/**
* @param {NoteMeta} rootMeta
* @param {NoteMeta} indexMeta
*/
function saveIndex(rootMeta, indexMeta) { function saveIndex(rootMeta, indexMeta) {
let firstNonEmptyNote; let firstNonEmptyNote;
let curMeta = rootMeta; let curMeta = rootMeta;
@ -433,6 +475,10 @@ ${markdownContent}`;
archive.append(fullHtml, { name: indexMeta.dataFileName }); archive.append(fullHtml, { name: indexMeta.dataFileName });
} }
/**
* @param {NoteMeta} rootMeta
* @param {NoteMeta} cssMeta
*/
function saveCss(rootMeta, cssMeta) { function saveCss(rootMeta, cssMeta) {
const cssContent = fs.readFileSync(`${RESOURCE_DIR}/libraries/ckeditor/ckeditor-content.css`); const cssContent = fs.readFileSync(`${RESOURCE_DIR}/libraries/ckeditor/ckeditor-content.css`);

View File

@ -0,0 +1,14 @@
class AttachmentMeta {
/** @type {string} */
attachmentId;
/** @type {string} */
title;
/** @type {string} */
role;
/** @type {string} */
mime;
/** @type {string} */
dataFileName;
}
module.exports = AttachmentMeta;

View File

@ -0,0 +1,14 @@
class AttributeMeta {
/** @type {string} */
type;
/** @type {string} */
name;
/** @type {boolean} */
value;
/** @type {boolean} */
isInheritable;
/** @type {integer} */
position;
}
module.exports = AttributeMeta;

View File

@ -0,0 +1,34 @@
class NoteMeta {
/** @type {string} */
noteId;
/** @type {string} */
notePath;
/** @type {boolean} */
isClone;
/** @type {string} */
title;
/** @type {integer} */
notePosition;
/** @type {string} */
prefix;
/** @type {boolean} */
isExpanded;
/** @type {string} */
type;
/** @type {string} */
mime;
/** @type {string} - 'html' or 'markdown', applicable to text notes only */
format;
/** @type {string} */
dataFileName;
/** @type {string} */
dirFileName;
/** @type {AttributeMeta[]} */
attributes;
/** @type {AttachmentMeta[]} */
attachments;
/** @type {NoteMeta[]|undefined} */
children;
}
module.exports = NoteMeta;

View File

@ -27,7 +27,7 @@ function getOption(name) {
} }
/** /**
* @returns {number} * @returns {integer}
*/ */
function getOptionInt(name) { function getOptionInt(name) {
const val = getOption(name); const val = getOption(name);

View File

@ -14,7 +14,7 @@ class SAttribute extends AbstractShacaEntity {
this.type = type; this.type = type;
/** @param {string} */ /** @param {string} */
this.name = name; this.name = name;
/** @param {int} */ /** @param {integer} */
this.position = position; this.position = position;
/** @param {string} */ /** @param {string} */
this.value = value; this.value = value;