From 5b9a1ef0e90d6dcb9458d4fec5bcec705698172d Mon Sep 17 00:00:00 2001 From: azivner Date: Fri, 16 Nov 2018 12:12:04 +0100 Subject: [PATCH] import/export logic refactored into separate files per format, closes #237 --- .../javascripts/dialogs/export_subtree.js | 3 +- src/routes/api/export.js | 260 +------------- src/routes/api/import.js | 339 +----------------- src/services/export/markdown_single.js | 19 + src/services/export/markdown_tar.js | 82 +++++ src/services/export/native_tar.js | 103 ++++++ src/services/export/opml.js | 67 ++++ src/services/import/markdown.js | 30 ++ src/services/import/opml.js | 56 +++ src/services/import/tar.js | 265 ++++++++++++++ src/views/dialogs/export_subtree.ejs | 4 +- 11 files changed, 645 insertions(+), 583 deletions(-) create mode 100644 src/services/export/markdown_single.js create mode 100644 src/services/export/markdown_tar.js create mode 100644 src/services/export/native_tar.js create mode 100644 src/services/export/opml.js create mode 100644 src/services/import/markdown.js create mode 100644 src/services/import/opml.js create mode 100644 src/services/import/tar.js diff --git a/src/public/javascripts/dialogs/export_subtree.js b/src/public/javascripts/dialogs/export_subtree.js index 2a5d18b93..3d6058fa8 100644 --- a/src/public/javascripts/dialogs/export_subtree.js +++ b/src/public/javascripts/dialogs/export_subtree.js @@ -6,7 +6,6 @@ import exportService from "../services/export.js"; const $dialog = $("#export-subtree-dialog"); const $form = $("#export-subtree-form"); const $noteTitle = $dialog.find(".note-title"); -const $exportFormat = $dialog.find("input[name='export-format']:checked"); async function showDialog() { glob.activeDialog = $dialog; @@ -20,7 +19,7 @@ async function showDialog() { } $form.submit(() => { - const exportFormat = $exportFormat.val(); + const exportFormat = $dialog.find("input[name='export-format']:checked").val(); const currentNode = treeService.getCurrentNode(); diff --git a/src/routes/api/export.js b/src/routes/api/export.js index 460c9a003..2336584a4 100644 --- a/src/routes/api/export.js +++ b/src/routes/api/export.js @@ -1,270 +1,34 @@ "use strict"; -const html = require('html'); -const tar = require('tar-stream'); -const sanitize = require("sanitize-filename"); +const nativeTarExportService = require('../../services/export/native_tar'); +const markdownTarExportService = require('../../services/export/markdown_tar'); +const markdownSingleExportService = require('../../services/export/markdown_single'); +const opmlExportService = require('../../services/export/opml'); const repository = require("../../services/repository"); -const utils = require('../../services/utils'); -const TurndownService = require('turndown'); async function exportNote(req, res) { // entityId maybe either noteId or branchId depending on format const entityId = req.params.entityId; const format = req.params.format; - if (format === 'tar') { - await exportToTar(await repository.getBranch(entityId), res); + if (format === 'native-tar') { + await nativeTarExportService.exportToTar(await repository.getBranch(entityId), res); } - else if (format === 'opml') { - await exportToOpml(await repository.getBranch(entityId), res); - } - else if (format === 'markdown') { - await exportToMarkdown(await repository.getBranch(entityId), res); + else if (format === 'markdown-tar') { + await markdownTarExportService.exportToMarkdown(await repository.getBranch(entityId), res); } // export single note without subtree else if (format === 'markdown-single') { - await exportSingleMarkdown(await repository.getNote(entityId), res); + await markdownSingleExportService.exportSingleMarkdown(await repository.getNote(entityId), res); + } + else if (format === 'opml') { + await opmlExportService.exportToOpml(await repository.getBranch(entityId), res); } else { return [404, "Unrecognized export format " + format]; } } -function escapeXmlAttribute(text) { - return text.replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); -} - -function prepareText(text) { - const newLines = text.replace(/(]*>|)/g, '\n') - .replace(/ /g, ' '); // nbsp isn't in XML standard (only HTML) - - const stripped = utils.stripTags(newLines); - - const escaped = escapeXmlAttribute(stripped); - - return escaped.replace(/\n/g, ' '); -} - -async function exportToOpml(branch, res) { - const note = await branch.getNote(); - const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title; - const sanitizedTitle = sanitize(title); - - async function exportNoteInner(branchId) { - const branch = await repository.getBranch(branchId); - const note = await branch.getNote(); - const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title; - - const preparedTitle = prepareText(title); - const preparedContent = prepareText(note.content); - - res.write(`\n`); - - for (const child of await note.getChildBranches()) { - await exportNoteInner(child.branchId); - } - - res.write(''); - } - - res.setHeader('Content-Disposition', 'file; filename="' + sanitizedTitle + '.opml"'); - res.setHeader('Content-Type', 'text/x-opml'); - - res.write(` - - -Trilium export - -`); - - await exportNoteInner(branch.branchId); - - res.write(` -`); - res.end(); -} - -async function exportToTar(branch, res) { - const pack = tar.pack(); - - const exportedNoteIds = []; - const name = await exportNoteInner(branch, ''); - - async function exportNoteInner(branch, directory) { - const note = await branch.getNote(); - const childFileName = directory + sanitize(note.title); - - if (exportedNoteIds.includes(note.noteId)) { - saveMetadataFile(childFileName, { - version: 1, - clone: true, - noteId: note.noteId, - prefix: branch.prefix - }); - - return; - } - - const metadata = { - version: 1, - clone: false, - noteId: note.noteId, - title: note.title, - prefix: branch.prefix, - isExpanded: branch.isExpanded, - type: note.type, - mime: note.mime, - // we don't export dateCreated and dateModified of any entity since that would be a bit misleading - attributes: (await note.getOwnedAttributes()).map(attribute => { - return { - type: attribute.type, - name: attribute.name, - value: attribute.value, - isInheritable: attribute.isInheritable, - position: attribute.position - }; - }), - links: (await note.getLinks()).map(link => { - return { - type: link.type, - targetNoteId: link.targetNoteId - } - }) - }; - - if (await note.hasLabel('excludeFromExport')) { - return; - } - - saveMetadataFile(childFileName, metadata); - saveDataFile(childFileName, note); - - exportedNoteIds.push(note.noteId); - - const childBranches = await note.getChildBranches(); - - if (childBranches.length > 0) { - saveDirectory(childFileName); - } - - for (const childBranch of childBranches) { - await exportNoteInner(childBranch, childFileName + "/"); - } - - return childFileName; - } - - function saveDataFile(childFileName, note) { - const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content; - - pack.entry({name: childFileName + ".dat", size: content.length}, content); - } - - function saveMetadataFile(childFileName, metadata) { - const metadataJson = JSON.stringify(metadata, null, '\t'); - - pack.entry({name: childFileName + ".meta", size: metadataJson.length}, metadataJson); - } - - function saveDirectory(childFileName) { - pack.entry({name: childFileName, type: 'directory'}); - } - - pack.finalize(); - - res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"'); - res.setHeader('Content-Type', 'application/tar'); - - pack.pipe(res); -} - -async function exportToMarkdown(branch, res) { - const note = await branch.getNote(); - - if (!await note.hasChildren()) { - await exportSingleMarkdown(note, res); - - return; - } - - const turndownService = new TurndownService(); - const pack = tar.pack(); - const name = await exportNoteInner(note, ''); - - async function exportNoteInner(note, directory) { - const childFileName = directory + sanitize(note.title); - - if (await note.hasLabel('excludeFromExport')) { - return; - } - - saveDataFile(childFileName, note); - - const childNotes = await note.getChildNotes(); - - if (childNotes.length > 0) { - saveDirectory(childFileName); - } - - for (const childNote of childNotes) { - await exportNoteInner(childNote, childFileName + "/"); - } - - return childFileName; - } - - function saveDataFile(childFileName, note) { - if (note.type !== 'text' && note.type !== 'code') { - return; - } - - if (note.content.trim().length === 0) { - return; - } - - let markdown; - - if (note.type === 'code') { - markdown = '```\n' + note.content + "\n```"; - } - else if (note.type === 'text') { - markdown = turndownService.turndown(note.content); - } - else { - // other note types are not supported - return; - } - - pack.entry({name: childFileName + ".md", size: markdown.length}, markdown); - } - - function saveDirectory(childFileName) { - pack.entry({name: childFileName, type: 'directory'}); - } - - pack.finalize(); - - res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"'); - res.setHeader('Content-Type', 'application/tar'); - - pack.pipe(res); -} - -async function exportSingleMarkdown(note, res) { - const turndownService = new TurndownService(); - const markdown = turndownService.turndown(note.content); - const name = sanitize(note.title); - - res.setHeader('Content-Disposition', 'file; filename="' + name + '.md"'); - res.setHeader('Content-Type', 'text/markdown; charset=UTF-8'); - - res.send(markdown); -} - module.exports = { exportNote }; \ No newline at end of file diff --git a/src/routes/api/import.js b/src/routes/api/import.js index 60475925d..02bbc1ae7 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -1,18 +1,11 @@ "use strict"; -const Attribute = require('../../entities/attribute'); -const Link = require('../../entities/link'); const repository = require('../../services/repository'); -const log = require('../../services/log'); -const utils = require('../../services/utils'); -const enex = require('../../services/import/enex'); -const noteService = require('../../services/notes'); -const Branch = require('../../entities/branch'); -const tar = require('tar-stream'); -const stream = require('stream'); +const enexImportService = require('../../services/import/enex'); +const opmlImportService = require('../../services/import/opml'); +const tarImportService = require('../../services/import/tar'); +const markdownImportService = require('../../services/import/markdown'); const path = require('path'); -const parseString = require('xml2js').parseString; -const commonmark = require('commonmark'); async function importToBranch(req) { const parentNoteId = req.params.parentNoteId; @@ -31,338 +24,22 @@ async function importToBranch(req) { const extension = path.extname(file.originalname).toLowerCase(); if (extension === '.tar') { - return await importTar(file, parentNote); + return await tarImportService.importTar(file, parentNote); } else if (extension === '.opml') { - return await importOpml(file, parentNote); + return await opmlImportService.importOpml(file, parentNote); } else if (extension === '.md') { - return await importMarkdown(file, parentNote); + return await markdownImportService.importMarkdown(file, parentNote); } else if (extension === '.enex') { - return await enex.importEnex(file, parentNote); + return await enexImportService.importEnex(file, parentNote); } else { return [400, `Unrecognized extension ${extension}, must be .tar or .opml`]; } } -function toHtml(text) { - if (!text) { - return ''; - } - - return '

' + text.replace(/(?:\r\n|\r|\n)/g, '

') + '

'; -} - -async function importOutline(outline, parentNoteId) { - const {note} = await noteService.createNote(parentNoteId, outline.$.title, toHtml(outline.$.text)); - - for (const childOutline of (outline.outline || [])) { - await importOutline(childOutline, note.noteId); - } - - return note; -} - -async function importOpml(file, parentNote) { - const xml = await new Promise(function(resolve, reject) - { - parseString(file.buffer, function (err, result) { - if (err) { - reject(err); - } - else { - resolve(result); - } - }); - }); - - if (xml.opml.$.version !== '1.0' && xml.opml.$.version !== '1.1') { - return [400, 'Unsupported OPML version ' + xml.opml.$.version + ', 1.0 or 1.1 expected instead.']; - } - - const outlines = xml.opml.body[0].outline || []; - let returnNote = null; - - for (const outline of outlines) { - const note = await importOutline(outline, parentNote.noteId); - - // first created note will be activated after import - returnNote = returnNote || note; - } - - return returnNote; -} - -/** - * Complication of this export is the need to balance two needs: - * - - */ -async function importTar(file, parentNote) { - const files = await parseImportFile(file); - - const ctx = { - // maps from original noteId (in tar file) to newly generated noteId - noteIdMap: {}, - // new noteIds of notes which were actually created (not just referenced) - createdNoteIds: [], - attributes: [], - links: [], - reader: new commonmark.Parser(), - writer: new commonmark.HtmlRenderer() - }; - - ctx.getNewNoteId = function(origNoteId) { - // in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution - if (!origNoteId.trim()) { - return ""; - } - - if (!ctx.noteIdMap[origNoteId]) { - ctx.noteIdMap[origNoteId] = utils.newEntityId(); - } - - return ctx.noteIdMap[origNoteId]; - }; - - const note = await importNotes(ctx, files, parentNote.noteId); - - // we save attributes and links after importing notes because we need to check that target noteIds - // have been really created (relation/links with targets outside of the export are not created) - - for (const attr of ctx.attributes) { - if (attr.type === 'relation') { - attr.value = ctx.getNewNoteId(attr.value); - - if (!ctx.createdNoteIds.includes(attr.value)) { - // relation targets note outside of the export - continue; - } - } - - await new Attribute(attr).save(); - } - - for (const link of ctx.links) { - link.targetNoteId = ctx.getNewNoteId(link.targetNoteId); - - if (!ctx.createdNoteIds.includes(link.targetNoteId)) { - // link targets note outside of the export - continue; - } - - await new Link(link).save(); - } - - return note; -} - -function getFileName(name) { - let key; - - if (name.endsWith(".dat")) { - key = "data"; - name = name.substr(0, name.length - 4); - } - else if (name.endsWith(".md")) { - key = "markdown"; - name = name.substr(0, name.length - 3); - } - else if (name.endsWith((".meta"))) { - key = "meta"; - name = name.substr(0, name.length - 5); - } - else { - log.error("Unknown file type in import: " + name); - } - - return {name, key}; -} - -async function parseImportFile(file) { - const fileMap = {}; - const files = []; - - const extract = tar.extract(); - - extract.on('entry', function(header, stream, next) { - let name, key; - - if (header.type === 'file') { - ({name, key} = getFileName(header.name)); - } - else if (header.type === 'directory') { - // directory entries in tar often end with directory separator - name = (header.name.endsWith("/") || header.name.endsWith("\\")) ? header.name.substr(0, header.name.length - 1) : header.name; - key = 'directory'; - } - else { - log.error("Unrecognized tar entry: " + JSON.stringify(header)); - return; - } - - let file = fileMap[name]; - - if (!file) { - file = fileMap[name] = { - name: path.basename(name), - children: [] - }; - - let parentFileName = path.dirname(header.name); - - if (parentFileName && parentFileName !== '.') { - fileMap[parentFileName].children.push(file); - } - else { - files.push(file); - } - } - - const chunks = []; - - stream.on("data", function (chunk) { - chunks.push(chunk); - }); - - // header is the tar header - // stream is the content body (might be an empty stream) - // call next when you are done with this entry - - stream.on('end', function() { - file[key] = Buffer.concat(chunks); - - if (key === "meta") { - file[key] = JSON.parse(file[key].toString("UTF-8")); - } - - next(); // ready for next entry - }); - - stream.resume(); // just auto drain the stream - }); - - return new Promise(resolve => { - extract.on('finish', function() { - resolve(files); - }); - - const bufferStream = new stream.PassThrough(); - bufferStream.end(file.buffer); - - bufferStream.pipe(extract); - }); -} - -async function importNotes(ctx, files, parentNoteId) { - let returnNote = null; - - for (const file of files) { - let note; - - if (!file.meta) { - let content = ''; - - if (file.data) { - content = file.data.toString("UTF-8"); - } - else if (file.markdown) { - const parsed = ctx.reader.parse(file.markdown.toString("UTF-8")); - content = ctx.writer.render(parsed); - } - - note = (await noteService.createNote(parentNoteId, file.name, content, { - type: 'text', - mime: 'text/html' - })).note; - } - else { - if (file.meta.version !== 1) { - throw new Error("Can't read meta data version " + file.meta.version); - } - - if (file.meta.clone) { - await new Branch({ - parentNoteId: parentNoteId, - noteId: ctx.getNewNoteId(file.meta.noteId), - prefix: file.meta.prefix, - isExpanded: !!file.meta.isExpanded - }).save(); - - return; - } - - if (file.meta.type !== 'file' && file.meta.type !== 'image') { - file.data = file.data.toString("UTF-8"); - - // this will replace all internal links ( and ) inside the body - // links pointing outside the export will be broken and changed (ctx.getNewNoteId() will still assign new noteId) - for (const link of file.meta.links || []) { - // no need to escape the regexp find string since it's a noteId which doesn't contain any special characters - file.data = file.data.replace(new RegExp(link.targetNoteId, "g"), ctx.getNewNoteId(link.targetNoteId)); - } - } - - note = (await noteService.createNote(parentNoteId, file.meta.title, file.data, { - noteId: ctx.getNewNoteId(file.meta.noteId), - type: file.meta.type, - mime: file.meta.mime, - prefix: file.meta.prefix - })).note; - - ctx.createdNoteIds.push(note.noteId); - - for (const attribute of file.meta.attributes || []) { - ctx.attributes.push({ - noteId: note.noteId, - type: attribute.type, - name: attribute.name, - value: attribute.value, - isInheritable: attribute.isInheritable, - position: attribute.position - }); - } - - for (const link of file.meta.links || []) { - ctx.links.push({ - noteId: note.noteId, - type: link.type, - targetNoteId: link.targetNoteId - }); - } - } - - // first created note will be activated after import - returnNote = returnNote || note; - - if (file.children.length > 0) { - await importNotes(ctx, file.children, note.noteId); - } - } - - return returnNote; -} - -async function importMarkdown(file, parentNote) { - const markdownContent = file.buffer.toString("UTF-8"); - - const reader = new commonmark.Parser(); - const writer = new commonmark.HtmlRenderer(); - - const parsed = reader.parse(markdownContent); - const htmlContent = writer.render(parsed); - - const title = file.originalname.substr(0, file.originalname.length - 3); // strip .md extension - - const {note} = await noteService.createNote(parentNote.noteId, title, htmlContent, { - type: 'text', - mime: 'text/html' - }); - - return note; -} - module.exports = { importToBranch }; \ No newline at end of file diff --git a/src/services/export/markdown_single.js b/src/services/export/markdown_single.js new file mode 100644 index 000000000..ceb14b553 --- /dev/null +++ b/src/services/export/markdown_single.js @@ -0,0 +1,19 @@ +"use strict"; + +const sanitize = require("sanitize-filename"); +const TurndownService = require('turndown'); + +async function exportSingleMarkdown(note, res) { + const turndownService = new TurndownService(); + const markdown = turndownService.turndown(note.content); + const name = sanitize(note.title); + + res.setHeader('Content-Disposition', 'file; filename="' + name + '.md"'); + res.setHeader('Content-Type', 'text/markdown; charset=UTF-8'); + + res.send(markdown); +} + +module.exports = { + exportSingleMarkdown +}; \ No newline at end of file diff --git a/src/services/export/markdown_tar.js b/src/services/export/markdown_tar.js new file mode 100644 index 000000000..0cec971e8 --- /dev/null +++ b/src/services/export/markdown_tar.js @@ -0,0 +1,82 @@ +"use strict"; + +const tar = require('tar-stream'); +const TurndownService = require('turndown'); +const sanitize = require("sanitize-filename"); +const markdownSingleExportService = require('../../services/export/markdown_single'); + +async function exportToMarkdown(branch, res) { + const note = await branch.getNote(); + + if (!await note.hasChildren()) { + await markdownSingleExportService.exportSingleMarkdown(note, res); + + return; + } + + const turndownService = new TurndownService(); + const pack = tar.pack(); + const name = await exportNoteInner(note, ''); + + async function exportNoteInner(note, directory) { + const childFileName = directory + sanitize(note.title); + + if (await note.hasLabel('excludeFromExport')) { + return; + } + + saveDataFile(childFileName, note); + + const childNotes = await note.getChildNotes(); + + if (childNotes.length > 0) { + saveDirectory(childFileName); + } + + for (const childNote of childNotes) { + await exportNoteInner(childNote, childFileName + "/"); + } + + return childFileName; + } + + function saveDataFile(childFileName, note) { + if (note.type !== 'text' && note.type !== 'code') { + return; + } + + if (note.content.trim().length === 0) { + return; + } + + let markdown; + + if (note.type === 'code') { + markdown = '```\n' + note.content + "\n```"; + } + else if (note.type === 'text') { + markdown = turndownService.turndown(note.content); + } + else { + // other note types are not supported + return; + } + + pack.entry({name: childFileName + ".md", size: markdown.length}, markdown); + } + + function saveDirectory(childFileName) { + pack.entry({name: childFileName, type: 'directory'}); + } + + pack.finalize(); + + res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"'); + res.setHeader('Content-Type', 'application/tar'); + + pack.pipe(res); +} + +module.exports = { + exportToMarkdown +}; \ No newline at end of file diff --git a/src/services/export/native_tar.js b/src/services/export/native_tar.js new file mode 100644 index 000000000..323998837 --- /dev/null +++ b/src/services/export/native_tar.js @@ -0,0 +1,103 @@ +"use strict"; + +const html = require('html'); +const native_tar = require('tar-stream'); +const sanitize = require("sanitize-filename"); + +async function exportToTar(branch, res) { + const pack = native_tar.pack(); + + const exportedNoteIds = []; + const name = await exportNoteInner(branch, ''); + + async function exportNoteInner(branch, directory) { + const note = await branch.getNote(); + const childFileName = directory + sanitize(note.title); + + if (exportedNoteIds.includes(note.noteId)) { + saveMetadataFile(childFileName, { + version: 1, + clone: true, + noteId: note.noteId, + prefix: branch.prefix + }); + + return; + } + + const metadata = { + version: 1, + clone: false, + noteId: note.noteId, + title: note.title, + prefix: branch.prefix, + isExpanded: branch.isExpanded, + type: note.type, + mime: note.mime, + // we don't export dateCreated and dateModified of any entity since that would be a bit misleading + attributes: (await note.getOwnedAttributes()).map(attribute => { + return { + type: attribute.type, + name: attribute.name, + value: attribute.value, + isInheritable: attribute.isInheritable, + position: attribute.position + }; + }), + links: (await note.getLinks()).map(link => { + return { + type: link.type, + targetNoteId: link.targetNoteId + } + }) + }; + + if (await note.hasLabel('excludeFromExport')) { + return; + } + + saveMetadataFile(childFileName, metadata); + saveDataFile(childFileName, note); + + exportedNoteIds.push(note.noteId); + + const childBranches = await note.getChildBranches(); + + if (childBranches.length > 0) { + saveDirectory(childFileName); + } + + for (const childBranch of childBranches) { + await exportNoteInner(childBranch, childFileName + "/"); + } + + return childFileName; + } + + function saveDataFile(childFileName, note) { + const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content; + + pack.entry({name: childFileName + ".dat", size: content.length}, content); + } + + function saveMetadataFile(childFileName, metadata) { + const metadataJson = JSON.stringify(metadata, null, '\t'); + + pack.entry({name: childFileName + ".meta", size: metadataJson.length}, metadataJson); + } + + function saveDirectory(childFileName) { + pack.entry({name: childFileName, type: 'directory'}); + } + + pack.finalize(); + + res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"'); + res.setHeader('Content-Type', 'application/tar'); + + pack.pipe(res); +} + +module.exports = { + exportToTar +}; \ No newline at end of file diff --git a/src/services/export/opml.js b/src/services/export/opml.js new file mode 100644 index 000000000..04a02b4d5 --- /dev/null +++ b/src/services/export/opml.js @@ -0,0 +1,67 @@ +"use strict"; + +const sanitize = require("sanitize-filename"); +const repository = require("../../services/repository"); +const utils = require('../../services/utils'); + +async function exportToOpml(branch, res) { + const note = await branch.getNote(); + const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title; + const sanitizedTitle = sanitize(title); + + async function exportNoteInner(branchId) { + const branch = await repository.getBranch(branchId); + const note = await branch.getNote(); + const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title; + + const preparedTitle = prepareText(title); + const preparedContent = prepareText(note.content); + + res.write(`\n`); + + for (const child of await note.getChildBranches()) { + await exportNoteInner(child.branchId); + } + + res.write(''); + } + + res.setHeader('Content-Disposition', 'file; filename="' + sanitizedTitle + '.opml"'); + res.setHeader('Content-Type', 'text/x-opml'); + + res.write(` + + +Trilium export + +`); + + await exportNoteInner(branch.branchId); + + res.write(` +`); + res.end(); +} + +function prepareText(text) { + const newLines = text.replace(/(]*>|)/g, '\n') + .replace(/ /g, ' '); // nbsp isn't in XML standard (only HTML) + + const stripped = utils.stripTags(newLines); + + const escaped = escapeXmlAttribute(stripped); + + return escaped.replace(/\n/g, ' '); +} + +function escapeXmlAttribute(text) { + return text.replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +module.exports = { + exportToOpml +}; \ No newline at end of file diff --git a/src/services/import/markdown.js b/src/services/import/markdown.js new file mode 100644 index 000000000..8ed911baf --- /dev/null +++ b/src/services/import/markdown.js @@ -0,0 +1,30 @@ +"use strict"; + +// note that this is for import of single markdown file only - for archive/structure of markdown files +// see tar export/import + +const noteService = require('../../services/notes'); +const commonmark = require('commonmark'); + +async function importMarkdown(file, parentNote) { + const markdownContent = file.buffer.toString("UTF-8"); + + const reader = new commonmark.Parser(); + const writer = new commonmark.HtmlRenderer(); + + const parsed = reader.parse(markdownContent); + const htmlContent = writer.render(parsed); + + const title = file.originalname.substr(0, file.originalname.length - 3); // strip .md extension + + const {note} = await noteService.createNote(parentNote.noteId, title, htmlContent, { + type: 'text', + mime: 'text/html' + }); + + return note; +} + +module.exports = { + importMarkdown +}; \ No newline at end of file diff --git a/src/services/import/opml.js b/src/services/import/opml.js new file mode 100644 index 000000000..4bc19fdbe --- /dev/null +++ b/src/services/import/opml.js @@ -0,0 +1,56 @@ +"use strict"; + +const noteService = require('../../services/notes'); +const parseString = require('xml2js').parseString; + +async function importOpml(file, parentNote) { + const xml = await new Promise(function(resolve, reject) + { + parseString(file.buffer, function (err, result) { + if (err) { + reject(err); + } + else { + resolve(result); + } + }); + }); + + if (xml.opml.$.version !== '1.0' && xml.opml.$.version !== '1.1') { + return [400, 'Unsupported OPML version ' + xml.opml.$.version + ', 1.0 or 1.1 expected instead.']; + } + + const outlines = xml.opml.body[0].outline || []; + let returnNote = null; + + for (const outline of outlines) { + const note = await importOutline(outline, parentNote.noteId); + + // first created note will be activated after import + returnNote = returnNote || note; + } + + return returnNote; +} + +function toHtml(text) { + if (!text) { + return ''; + } + + return '

' + text.replace(/(?:\r\n|\r|\n)/g, '

') + '

'; +} + +async function importOutline(outline, parentNoteId) { + const {note} = await noteService.createNote(parentNoteId, outline.$.title, toHtml(outline.$.text)); + + for (const childOutline of (outline.outline || [])) { + await importOutline(childOutline, note.noteId); + } + + return note; +} + +module.exports = { + importOpml +}; \ No newline at end of file diff --git a/src/services/import/tar.js b/src/services/import/tar.js new file mode 100644 index 000000000..dfbd717d8 --- /dev/null +++ b/src/services/import/tar.js @@ -0,0 +1,265 @@ +"use strict"; + +const Attribute = require('../../entities/attribute'); +const Link = require('../../entities/link'); +const log = require('../../services/log'); +const utils = require('../../services/utils'); +const noteService = require('../../services/notes'); +const Branch = require('../../entities/branch'); +const tar = require('tar-stream'); +const stream = require('stream'); +const path = require('path'); +const commonmark = require('commonmark'); + +/** + * Complication of this export is the need to balance two needs: + * - + */ +async function importTar(file, parentNote) { + const files = await parseImportFile(file); + + const ctx = { + // maps from original noteId (in tar file) to newly generated noteId + noteIdMap: {}, + // new noteIds of notes which were actually created (not just referenced) + createdNoteIds: [], + attributes: [], + links: [], + reader: new commonmark.Parser(), + writer: new commonmark.HtmlRenderer() + }; + + ctx.getNewNoteId = function(origNoteId) { + // in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution + if (!origNoteId.trim()) { + return ""; + } + + if (!ctx.noteIdMap[origNoteId]) { + ctx.noteIdMap[origNoteId] = utils.newEntityId(); + } + + return ctx.noteIdMap[origNoteId]; + }; + + const note = await importNotes(ctx, files, parentNote.noteId); + + // we save attributes and links after importing notes because we need to check that target noteIds + // have been really created (relation/links with targets outside of the export are not created) + + for (const attr of ctx.attributes) { + if (attr.type === 'relation') { + attr.value = ctx.getNewNoteId(attr.value); + + if (!ctx.createdNoteIds.includes(attr.value)) { + // relation targets note outside of the export + continue; + } + } + + await new Attribute(attr).save(); + } + + for (const link of ctx.links) { + link.targetNoteId = ctx.getNewNoteId(link.targetNoteId); + + if (!ctx.createdNoteIds.includes(link.targetNoteId)) { + // link targets note outside of the export + continue; + } + + await new Link(link).save(); + } + + return note; +} + +function getFileName(name) { + let key; + + if (name.endsWith(".dat")) { + key = "data"; + name = name.substr(0, name.length - 4); + } + else if (name.endsWith(".md")) { + key = "markdown"; + name = name.substr(0, name.length - 3); + } + else if (name.endsWith((".meta"))) { + key = "meta"; + name = name.substr(0, name.length - 5); + } + else { + log.error("Unknown file type in import: " + name); + } + + return {name, key}; +} + +async function parseImportFile(file) { + const fileMap = {}; + const files = []; + + const extract = tar.extract(); + + extract.on('entry', function(header, stream, next) { + let name, key; + + if (header.type === 'file') { + ({name, key} = getFileName(header.name)); + } + else if (header.type === 'directory') { + // directory entries in tar often end with directory separator + name = (header.name.endsWith("/") || header.name.endsWith("\\")) ? header.name.substr(0, header.name.length - 1) : header.name; + key = 'directory'; + } + else { + log.error("Unrecognized tar entry: " + JSON.stringify(header)); + return; + } + + let file = fileMap[name]; + + if (!file) { + file = fileMap[name] = { + name: path.basename(name), + children: [] + }; + + let parentFileName = path.dirname(header.name); + + if (parentFileName && parentFileName !== '.') { + fileMap[parentFileName].children.push(file); + } + else { + files.push(file); + } + } + + const chunks = []; + + stream.on("data", function (chunk) { + chunks.push(chunk); + }); + + // header is the tar header + // stream is the content body (might be an empty stream) + // call next when you are done with this entry + + stream.on('end', function() { + file[key] = Buffer.concat(chunks); + + if (key === "meta") { + file[key] = JSON.parse(file[key].toString("UTF-8")); + } + + next(); // ready for next entry + }); + + stream.resume(); // just auto drain the stream + }); + + return new Promise(resolve => { + extract.on('finish', function() { + resolve(files); + }); + + const bufferStream = new stream.PassThrough(); + bufferStream.end(file.buffer); + + bufferStream.pipe(extract); + }); +} + +async function importNotes(ctx, files, parentNoteId) { + let returnNote = null; + + for (const file of files) { + let note; + + if (!file.meta) { + let content = ''; + + if (file.data) { + content = file.data.toString("UTF-8"); + } + else if (file.markdown) { + const parsed = ctx.reader.parse(file.markdown.toString("UTF-8")); + content = ctx.writer.render(parsed); + } + + note = (await noteService.createNote(parentNoteId, file.name, content, { + type: 'text', + mime: 'text/html' + })).note; + } + else { + if (file.meta.version !== 1) { + throw new Error("Can't read meta data version " + file.meta.version); + } + + if (file.meta.clone) { + await new Branch({ + parentNoteId: parentNoteId, + noteId: ctx.getNewNoteId(file.meta.noteId), + prefix: file.meta.prefix, + isExpanded: !!file.meta.isExpanded + }).save(); + + return; + } + + if (file.meta.type !== 'file' && file.meta.type !== 'image') { + file.data = file.data.toString("UTF-8"); + + // this will replace all internal links ( and ) inside the body + // links pointing outside the export will be broken and changed (ctx.getNewNoteId() will still assign new noteId) + for (const link of file.meta.links || []) { + // no need to escape the regexp find string since it's a noteId which doesn't contain any special characters + file.data = file.data.replace(new RegExp(link.targetNoteId, "g"), ctx.getNewNoteId(link.targetNoteId)); + } + } + + note = (await noteService.createNote(parentNoteId, file.meta.title, file.data, { + noteId: ctx.getNewNoteId(file.meta.noteId), + type: file.meta.type, + mime: file.meta.mime, + prefix: file.meta.prefix + })).note; + + ctx.createdNoteIds.push(note.noteId); + + for (const attribute of file.meta.attributes || []) { + ctx.attributes.push({ + noteId: note.noteId, + type: attribute.type, + name: attribute.name, + value: attribute.value, + isInheritable: attribute.isInheritable, + position: attribute.position + }); + } + + for (const link of file.meta.links || []) { + ctx.links.push({ + noteId: note.noteId, + type: link.type, + targetNoteId: link.targetNoteId + }); + } + } + + // first created note will be activated after import + returnNote = returnNote || note; + + if (file.children.length > 0) { + await importNotes(ctx, file.children, note.noteId); + } + } + + return returnNote; +} + +module.exports = { + importTar +}; \ No newline at end of file diff --git a/src/views/dialogs/export_subtree.ejs b/src/views/dialogs/export_subtree.ejs index 56c984d29..339b6b55e 100644 --- a/src/views/dialogs/export_subtree.ejs +++ b/src/views/dialogs/export_subtree.ejs @@ -14,7 +14,7 @@
- +
@@ -31,7 +31,7 @@
+ value="markdown-tar">