From 5e318c624243fc91e413ae6310bcf6186f0b90ec Mon Sep 17 00:00:00 2001 From: azivner Date: Mon, 5 Nov 2018 00:06:17 +0100 Subject: [PATCH] basic enex import, closes #194 --- package.json | 1 + src/public/javascripts/services/export.js | 6 +- .../javascripts/services/tree_context_menu.js | 2 +- src/routes/api/file_upload.js | 2 +- src/routes/api/import.js | 22 +- src/services/enex.js | 247 + src/services/notes.js | 3 +- src/test/enex/Export-stack.enex | 7 + src/test/enex/Export-test.enex | 5488 +++++++++++++++++ 9 files changed, 5765 insertions(+), 13 deletions(-) create mode 100644 src/services/enex.js create mode 100644 src/test/enex/Export-stack.enex create mode 100644 src/test/enex/Export-test.enex diff --git a/package.json b/package.json index 5bbe3c2fa..b947a3320 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "request-promise": "4.2.2", "rimraf": "2.6.2", "sanitize-filename": "1.6.1", + "sax": "^1.2.4", "serve-favicon": "2.5.0", "session-file-store": "1.2.0", "simple-node-logger": "0.93.40", diff --git a/src/public/javascripts/services/export.js b/src/public/javascripts/services/export.js index 89c9d6eaa..bba4167f4 100644 --- a/src/public/javascripts/services/export.js +++ b/src/public/javascripts/services/export.js @@ -38,7 +38,11 @@ $("#import-upload").change(async function() { .done(async note => { await treeService.reload(); - await treeService.activateNote(note.noteId); + if (note) { + const node = await treeService.activateNote(note.noteId); + + node.setExpanded(true); + } }); }); diff --git a/src/public/javascripts/services/tree_context_menu.js b/src/public/javascripts/services/tree_context_menu.js index cfb1ba77a..a05c579f4 100644 --- a/src/public/javascripts/services/tree_context_menu.js +++ b/src/public/javascripts/services/tree_context_menu.js @@ -99,7 +99,7 @@ const contextMenuOptions = { {title: "OPML", cmd: "exportSubtreeToOpml"}, {title: "Markdown", cmd: "exportSubtreeToMarkdown"} ]}, - {title: "Import into note (tar, opml, md)", cmd: "importIntoNote", uiIcon: "ui-icon-arrowthick-1-sw"}, + {title: "Import into note (tar, opml, md, enex)", cmd: "importIntoNote", uiIcon: "ui-icon-arrowthick-1-sw"}, {title: "----"}, {title: "Collapse subtree Alt+-", cmd: "collapseSubtree", uiIcon: "ui-icon-minus"}, {title: "Force note sync", cmd: "forceNoteSync", uiIcon: "ui-icon-refresh"}, diff --git a/src/routes/api/file_upload.js b/src/routes/api/file_upload.js index 164c9e939..dec0fc208 100644 --- a/src/routes/api/file_upload.js +++ b/src/routes/api/file_upload.js @@ -48,7 +48,7 @@ async function downloadFile(req, res) { } const originalFileName = await note.getLabel('originalFileName'); - const fileName = originalFileName.value || note.title; + const fileName = originalFileName ? originalFileName.value : note.title; res.setHeader('Content-Disposition', 'file; filename="' + fileName + '"'); res.setHeader('Content-Type', note.mime); diff --git a/src/routes/api/import.js b/src/routes/api/import.js index f424211fa..c0da827c6 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -2,6 +2,7 @@ const repository = require('../../services/repository'); const log = require('../../services/log'); +const enex = require('../../services/enex'); const attributeService = require('../../services/attributes'); const noteService = require('../../services/notes'); const Branch = require('../../entities/branch'); @@ -28,13 +29,16 @@ async function importToBranch(req) { const extension = path.extname(file.originalname).toLowerCase(); if (extension === '.tar') { - return await importTar(file, parentNoteId); + return await importTar(file, parentNote); } else if (extension === '.opml') { - return await importOpml(file, parentNoteId); + return await importOpml(file, parentNote); } else if (extension === '.md') { - return await importMarkdown(file, parentNoteId); + return await importMarkdown(file, parentNote); + } + else if (extension === '.enex') { + return await enex.importEnex(file, parentNote); } else { return [400, `Unrecognized extension ${extension}, must be .tar or .opml`]; @@ -59,7 +63,7 @@ async function importOutline(outline, parentNoteId) { return note; } -async function importOpml(file, parentNoteId) { +async function importOpml(file, parentNote) { const xml = await new Promise(function(resolve, reject) { parseString(file.buffer, function (err, result) { @@ -80,7 +84,7 @@ async function importOpml(file, parentNoteId) { let returnNote = null; for (const outline of outlines) { - const note = await importOutline(outline, parentNoteId); + const note = await importOutline(outline, parentNote.noteId); // first created note will be activated after import returnNote = returnNote || note; @@ -89,7 +93,7 @@ async function importOpml(file, parentNoteId) { return returnNote; } -async function importTar(file, parentNoteId) { +async function importTar(file, parentNote) { const files = await parseImportFile(file); const ctx = { @@ -100,7 +104,7 @@ async function importTar(file, parentNoteId) { writer: new commonmark.HtmlRenderer() }; - const note = await importNotes(ctx, files, parentNoteId); + const note = await importNotes(ctx, files, parentNote.noteId); // we save attributes after importing notes because we need to have all the relation // targets already existing @@ -290,7 +294,7 @@ async function importNotes(ctx, files, parentNoteId) { return returnNote; } -async function importMarkdown(file, parentNoteId) { +async function importMarkdown(file, parentNote) { const markdownContent = file.buffer.toString("UTF-8"); const reader = new commonmark.Parser(); @@ -301,7 +305,7 @@ async function importMarkdown(file, parentNoteId) { const title = file.originalname.substr(0, file.originalname.length - 3); // strip .md extension - const {note} = await noteService.createNote(parentNoteId, title, htmlContent, { + const {note} = await noteService.createNote(parentNote.noteId, title, htmlContent, { type: 'text', mime: 'text/html' }); diff --git a/src/services/enex.js b/src/services/enex.js new file mode 100644 index 000000000..41335990c --- /dev/null +++ b/src/services/enex.js @@ -0,0 +1,247 @@ +const sax = require("sax"); +const stream = require('stream'); +const xml2js = require('xml2js'); +const log = require("./log"); +const utils = require("./utils"); +const noteService = require("./notes"); + +// date format is e.g. 20181121T193703Z +function parseDate(text) { + // insert - and : to make it ISO format + text = text.substr(0, 4) + "-" + text.substr(4, 2) + "-" + text.substr(6, 2) + + "T" + text.substr(9, 2) + ":" + text.substr(11, 2) + ":" + text.substr(13, 2) + "Z"; + + return text; +} + +let note = {}; +let resource; + +async function importEnex(file, parentNote) { + const saxStream = sax.createStream(true); + const xmlBuilder = new xml2js.Builder({ headless: true }); + const parser = new xml2js.Parser({ explicitArray: true }); + + // we're persisting notes as we parse the document, but these are run asynchronously and may not be finished + // when we finish parsing. We use this to be sure that all saving has been finished before returning successfully. + const saveNotePromises = []; + + async function parseXml(text) { + return new Promise(function(resolve, reject) + { + parser.parseString(text, function (err, result) { + if (err) { + reject(err); + } + else { + resolve(result); + } + }); + }); + } + + function extractContent(enNote) { + // [] thing is workaround for https://github.com/Leonidas-from-XIV/node-xml2js/issues/484 + let content = xmlBuilder.buildObject([enNote]); + content = content.substr(3, content.length - 7).trim(); + + // workaround for https://github.com/ckeditor/ckeditor5-list/issues/116 + content = content.replace(/
  • \s+
    /g, "
  • "); + content = content.replace(/<\/div>\s+<\/li>/g, "
    • "); + + // workaround for https://github.com/ckeditor/ckeditor5-list/issues/115 + content = content.replace(/