From af5c4b58597b79912f86ce6a4c94a3a7185c5817 Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 20 Mar 2020 21:57:16 +0100 Subject: [PATCH] zip import using yazl --- package-lock.json | 236 ----------- package.json | 1 + src/public/javascripts/services/import.js | 2 +- src/routes/api/import.js | 3 + src/services/export/zip.js | 2 - src/services/import/zip.js | 470 ++++++++++++++++++++++ src/views/dialogs/import.ejs | 8 +- 7 files changed, 479 insertions(+), 243 deletions(-) create mode 100644 src/services/import/zip.js diff --git a/package-lock.json b/package-lock.json index bd6c49ff8..a0eebcc35 100644 --- a/package-lock.json +++ b/package-lock.json @@ -825,43 +825,6 @@ } } }, - "archiver-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", - "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", - "requires": { - "glob": "^7.1.4", - "graceful-fs": "^4.2.0", - "lazystream": "^1.0.0", - "lodash.defaults": "^4.2.0", - "lodash.difference": "^4.5.0", - "lodash.flatten": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.union": "^4.6.0", - "normalize-path": "^3.0.0", - "readable-stream": "^2.0.0" - }, - "dependencies": { - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" - } - } - }, "are-we-there-yet": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", @@ -1072,11 +1035,6 @@ "resolved": "https://registry.npmjs.org/beeper/-/beeper-1.1.1.tgz", "integrity": "sha1-5tXqjF2tABMEpwsiY4RH9pyy+Ak=" }, - "big-integer": { - "version": "1.6.48", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", - "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==" - }, "bin-build": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/bin-build/-/bin-build-2.2.0.tgz", @@ -1175,15 +1133,6 @@ "os-filter-obj": "^1.0.0" } }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, "bl": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", @@ -1433,11 +1382,6 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, - "buffer-indexof-polyfill": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz", - "integrity": "sha1-qfuAbOgUXVQoUQznLyeLs2OmOL8=" - }, "buffer-to-vinyl": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/buffer-to-vinyl/-/buffer-to-vinyl-1.1.0.tgz", @@ -1461,11 +1405,6 @@ } } }, - "buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" - }, "builder-util": { "version": "22.4.1", "resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.4.1.tgz", @@ -1675,14 +1614,6 @@ } } }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", - "requires": { - "traverse": ">=0.3.0 <0.4" - } - }, "chalk": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", @@ -1887,24 +1818,6 @@ "integrity": "sha1-AWLsLZNR9d3VmpICy6k1NmpyUIA=", "dev": true }, - "compress-commons": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-2.1.1.tgz", - "integrity": "sha512-eVw6n7CnEMFzc3duyFVrQEuY1BlHR3rYsSztyG32ibGMW722i3C6IizEGMFmfMU+A+fALvBIwxN3czffTcdA+Q==", - "requires": { - "buffer-crc32": "^0.2.13", - "crc32-stream": "^3.0.1", - "normalize-path": "^3.0.0", - "readable-stream": "^2.3.6" - }, - "dependencies": { - "normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" - } - } - }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2023,35 +1936,6 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, - "crc": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz", - "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==", - "requires": { - "buffer": "^5.1.0" - } - }, - "crc32-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-3.0.1.tgz", - "integrity": "sha512-mctvpXlbzsvK+6z8kJwSJ5crm7yBwrQMTybJzMw1O4lLGJqjlDCXY2Zw7KheiA6XBEcBmfLx1D88mjRGVJtY9w==", - "requires": { - "crc": "^3.4.4", - "readable-stream": "^3.4.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, "create-error-class": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", @@ -4067,40 +3951,6 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, - "fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "requires": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - }, - "dependencies": { - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "requires": { - "glob": "^7.1.3" - } - } - } - }, "galactus": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/galactus/-/galactus-0.2.1.tgz", @@ -5810,11 +5660,6 @@ "uc.micro": "^1.0.1" } }, - "listenercount": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", - "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" - }, "load-bmfont": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.0.tgz", @@ -5914,16 +5759,6 @@ "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=", "dev": true }, - "lodash.defaults": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" - }, - "lodash.difference": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", - "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" - }, "lodash.escape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/lodash.escape/-/lodash.escape-3.2.0.tgz", @@ -5932,11 +5767,6 @@ "lodash._root": "^3.0.0" } }, - "lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" - }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -5963,11 +5793,6 @@ "resolved": "https://registry.npmjs.org/lodash.isfinite/-/lodash.isfinite-3.3.2.tgz", "integrity": "sha1-+4m2WpqAKBgz8LdHizpRBPiY67M=" }, - "lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" - }, "lodash.keys": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz", @@ -6013,11 +5838,6 @@ "lodash.escape": "^3.0.0" } }, - "lodash.union": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", - "integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=" - }, "log-symbols": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", @@ -9126,11 +8946,6 @@ "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=" }, - "setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" - }, "setprototypeof": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", @@ -9769,11 +9584,6 @@ "punycode": "^2.1.1" } }, - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" - }, "trim-newlines": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", @@ -9977,30 +9787,6 @@ "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz", "integrity": "sha1-uYTwh3/AqJwsdzzB73tbIytbBv4=" }, - "unzipper": { - "version": "0.10.10", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.10.tgz", - "integrity": "sha512-wEgtqtrnJ/9zIBsQb8UIxOhAH1eTHfi7D/xvmrUoMEePeI6u24nq1wigazbIFtHt6ANYXdEVTvc8XYNlTurs7A==", - "requires": { - "big-integer": "^1.6.17", - "binary": "~0.3.0", - "bluebird": "~3.4.1", - "buffer-indexof-polyfill": "~1.0.0", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2", - "listenercount": "~1.0.1", - "readable-stream": "~2.3.6", - "setimmediate": "~1.0.4" - }, - "dependencies": { - "bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" - } - } - }, "update-notifier": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.0.tgz", @@ -10713,28 +10499,6 @@ "requires": { "buffer-crc32": "~0.2.3" } - }, - "zip-stream": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-2.1.3.tgz", - "integrity": "sha512-EkXc2JGcKhO5N5aZ7TmuNo45budRaFGHOmz24wtJR7znbNqDPmdZtUauKX6et8KAVseAMBOyWJqEpXcHTBsh7Q==", - "requires": { - "archiver-utils": "^2.1.0", - "compress-commons": "^2.1.1", - "readable-stream": "^3.4.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } } } } diff --git a/package.json b/package.json index 6bff5ab3e..8d3a292d1 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "turndown-plugin-gfm": "1.0.2", "unescape": "1.0.1", "ws": "7.2.3", + "yauzl": "^2.10.0", "yazl": "^2.5.1" }, "devDependencies": { diff --git a/src/public/javascripts/services/import.js b/src/public/javascripts/services/import.js index c3085a314..f2339bf0d 100644 --- a/src/public/javascripts/services/import.js +++ b/src/public/javascripts/services/import.js @@ -64,7 +64,7 @@ ws.subscribeToMessages(async message => { toastService.showPersistent(toast); if (message.result.importedNoteId) { - await appContext.tabManager.getActiveTabContext.setNote(message.result.importedNoteId); + await appContext.tabManager.getActiveTabContext().setNote(message.result.importedNoteId); } } }); diff --git a/src/routes/api/import.js b/src/routes/api/import.js index 81833bf17..7237450d3 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -4,6 +4,7 @@ const repository = require('../../services/repository'); const enexImportService = require('../../services/import/enex'); const opmlImportService = require('../../services/import/opml'); const tarImportService = require('../../services/import/tar'); +const zipImportService = require('../../services/import/zip'); const singleImportService = require('../../services/import/single'); const cls = require('../../services/cls'); const path = require('path'); @@ -48,6 +49,8 @@ async function importToBranch(req) { try { if (extension === '.tar' && options.explodeArchives) { note = await tarImportService.importTar(taskContext, file.buffer, parentNote); + } else if (extension === '.zip' && options.explodeArchives) { + note = await zipImportService.importZip(taskContext, file.buffer, parentNote); } else if (extension === '.opml' && options.explodeArchives) { note = await opmlImportService.importOpml(taskContext, file.buffer, parentNote); } else if (extension === '.enex' && options.explodeArchives) { diff --git a/src/services/export/zip.js b/src/services/export/zip.js index 9673cf761..63883b18c 100644 --- a/src/services/export/zip.js +++ b/src/services/export/zip.js @@ -13,8 +13,6 @@ const protectedSessionService = require('../protected_session'); const sanitize = require("sanitize-filename"); const fs = require("fs"); const RESOURCE_DIR = require('../../services/resource_dir').RESOURCE_DIR; -const ZipStream = require('zip-stream'); -const {Readable} = require('stream'); const yazl = require("yazl"); /** diff --git a/src/services/import/zip.js b/src/services/import/zip.js new file mode 100644 index 000000000..f7717a70b --- /dev/null +++ b/src/services/import/zip.js @@ -0,0 +1,470 @@ +"use strict"; + +const Attribute = require('../../entities/attribute'); +const utils = require('../../services/utils'); +const log = require('../../services/log'); +const repository = require('../../services/repository'); +const noteService = require('../../services/notes'); +const attributeService = require('../../services/attributes'); +const Branch = require('../../entities/branch'); +const path = require('path'); +const commonmark = require('commonmark'); +const TaskContext = require('../task_context.js'); +const protectedSessionService = require('../protected_session'); +const mimeService = require("./mime"); +const treeService = require("../tree"); +const yauzl = require("yauzl"); + +/** + * @param {TaskContext} taskContext + * @param {Buffer} fileBuffer + * @param {Note} importRootNote + * @return {Promise<*>} + */ +async function importZip(taskContext, fileBuffer, importRootNote) { + // maps from original noteId (in tar file) to newly generated noteId + const noteIdMap = {}; + const attributes = []; + // path => noteId + const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId }; + const mdReader = new commonmark.Parser(); + const mdWriter = new commonmark.HtmlRenderer(); + let metaFile = null; + let firstNote = null; + + function getNewNoteId(origNoteId) { + // in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution + if (!origNoteId.trim()) { + return ""; + } + + if (!noteIdMap[origNoteId]) { + noteIdMap[origNoteId] = utils.newEntityId(); + } + + return noteIdMap[origNoteId]; + } + + function getMeta(filePath) { + if (!metaFile) { + return {}; + } + + const pathSegments = filePath.split(/[\/\\]/g); + + let cursor = { + isImportRoot: true, + children: metaFile.files + }; + + let parent; + + for (const segment of pathSegments) { + if (!cursor || !cursor.children || cursor.children.length === 0) { + return {}; + } + + parent = cursor; + cursor = cursor.children.find(file => file.dataFileName === segment || file.dirFileName === segment); + } + + return { + parentNoteMeta: parent, + noteMeta: cursor + }; + } + + async function getParentNoteId(filePath, parentNoteMeta) { + let parentNoteId; + + if (parentNoteMeta) { + parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId); + } + else { + const parentPath = path.dirname(filePath); + + if (parentPath === '.') { + parentNoteId = importRootNote.noteId; + } + else if (parentPath in createdPaths) { + parentNoteId = createdPaths[parentPath]; + } + else { + // tar allows creating out of order records - i.e. file in a directory can appear in the tar stream before actual directory + // (out-of-order-directory-records.tar in test set) + parentNoteId = await saveDirectory(parentPath); + } + } + + return parentNoteId; + } + + function getNoteTitle(filePath, noteMeta) { + if (noteMeta) { + return noteMeta.title; + } + else { + const basename = path.basename(filePath); + + return getTextFileWithoutExtension(basename); + } + } + + function getNoteId(noteMeta, filePath) { + const filePathNoExt = getTextFileWithoutExtension(filePath); + + if (filePathNoExt in createdPaths) { + return createdPaths[filePathNoExt]; + } + + const noteId = noteMeta ? getNewNoteId(noteMeta.noteId) : utils.newEntityId(); + + createdPaths[filePathNoExt] = noteId; + + return noteId; + } + + function detectFileTypeAndMime(taskContext, filePath) { + const mime = mimeService.getMime(filePath) || "application/octet-stream"; + const type = mimeService.getType(taskContext.data, mime); + + return { mime, type }; + } + + async function saveAttributes(note, noteMeta) { + if (!noteMeta) { + return; + } + + for (const attr of noteMeta.attributes) { + attr.noteId = note.noteId; + + if (!attributeService.isAttributeType(attr.type)) { + log.error("Unrecognized attribute type " + attr.type); + continue; + } + + if (attr.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(attr.name)) { + // these relations are created automatically and as such don't need to be duplicated in the import + continue; + } + + if (attr.type === 'label' && attr.name === 'externalLink') { + // also created automatically + continue; + } + + if (attr.type === 'relation') { + attr.value = getNewNoteId(attr.value); + } + + if (taskContext.data.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) { + attr.name = 'disabled-' + attr.name; + } + + attributes.push(attr); + } + } + + async function saveDirectory(filePath) { + const { parentNoteMeta, noteMeta } = getMeta(filePath); + + const noteId = getNoteId(noteMeta, filePath); + const noteTitle = getNoteTitle(filePath, noteMeta); + const parentNoteId = await getParentNoteId(filePath, parentNoteMeta); + + let note = await repository.getNote(noteId); + + if (note) { + return; + } + + ({note} = await noteService.createNewNote({ + parentNoteId: parentNoteId, + title: noteTitle, + content: '', + noteId: noteId, + type: noteMeta ? noteMeta.type : 'text', + mime: noteMeta ? noteMeta.mime : 'text/html', + prefix: noteMeta ? noteMeta.prefix : '', + isExpanded: noteMeta ? noteMeta.isExpanded : false, + isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), + })); + + await saveAttributes(note, noteMeta); + + if (!firstNote) { + firstNote = note; + } + + return noteId; + } + + function getTextFileWithoutExtension(filePath) { + const extension = path.extname(filePath).toLowerCase(); + + if (extension === '.md' || extension === '.html') { + return filePath.substr(0, filePath.length - extension.length); + } + else { + return filePath; + } + } + + function getNoteIdFromRelativeUrl(url, filePath) { + while (url.startsWith("./")) { + url = url.substr(2); + } + + let absUrl = path.dirname(filePath); + + while (url.startsWith("../")) { + absUrl = path.dirname(absUrl); + + url = url.substr(3); + } + + if (absUrl === '.') { + absUrl = ''; + } + + absUrl += (absUrl.length > 0 ? '/' : '') + url; + + const {noteMeta} = getMeta(absUrl); + const targetNoteId = getNoteId(noteMeta, absUrl); + return targetNoteId; + } + + async function saveNote(filePath, content) { + const {parentNoteMeta, noteMeta} = getMeta(filePath); + + if (noteMeta && noteMeta.noImport) { + return; + } + + const noteId = getNoteId(noteMeta, filePath); + const parentNoteId = await getParentNoteId(filePath, parentNoteMeta); + + if (noteMeta && noteMeta.isClone) { + await new Branch({ + noteId, + parentNoteId, + isExpanded: noteMeta.isExpanded, + prefix: noteMeta.prefix, + notePosition: noteMeta.notePosition + }).save(); + + return; + } + + const {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath); + + if (type !== 'file' && type !== 'image') { + content = content.toString("UTF-8"); + } + + if ((noteMeta && noteMeta.format === 'markdown') + || (!noteMeta && taskContext.data.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime))) { + const parsed = mdReader.parse(content); + content = mdWriter.render(parsed); + } + + const noteTitle = getNoteTitle(filePath, noteMeta); + + if (type === 'text') { + function isUrlAbsolute(url) { + return /^(?:[a-z]+:)?\/\//i.test(url); + } + + content = content.replace(/]*>/gis, ""); + content = content.replace(/<\/body>.*<\/html>/gis, ""); + + content = content.replace(/src="([^"]*)"/g, (match, url) => { + url = decodeURIComponent(url); + + if (isUrlAbsolute(url) || url.startsWith("/")) { + return match; + } + + const targetNoteId = getNoteIdFromRelativeUrl(url, filePath); + + return `src="api/images/${targetNoteId}/${path.basename(url)}"`; + }); + + content = content.replace(/href="([^"]*)"/g, (match, url) => { + url = decodeURIComponent(url); + + if (isUrlAbsolute(url)) { + return match; + } + + const targetNoteId = getNoteIdFromRelativeUrl(url, filePath); + + return `href="#root/${targetNoteId}"`; + }); + + content = content.replace(/

([^<]*)<\/h1>/gi, (match, text) => { + if (noteTitle.trim() === text.trim()) { + return ""; // remove whole H1 tag + } + else { + return match; + } + }); + } + + if (type === 'relation-map' && noteMeta) { + const relationMapLinks = (noteMeta.attributes || []) + .filter(attr => attr.type === 'relation' && attr.name === 'relationMapLink'); + + // this will replace relation map links + for (const link of relationMapLinks) { + // no need to escape the regexp find string since it's a noteId which doesn't contain any special characters + content = content.replace(new RegExp(link.value, "g"), getNewNoteId(link.value)); + } + } + + let note = await repository.getNote(noteId); + + if (note) { + await note.setContent(content); + } + else { + ({note} = await noteService.createNewNote({ + parentNoteId: parentNoteId, + title: noteTitle, + content: content, + noteId, + type, + mime, + prefix: noteMeta ? noteMeta.prefix : '', + isExpanded: noteMeta ? noteMeta.isExpanded : false, + notePosition: noteMeta ? noteMeta.notePosition : false, + isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), + })); + + await saveAttributes(note, noteMeta); + + if (!firstNote) { + firstNote = note; + } + + if (type === 'text') { + filePath = getTextFileWithoutExtension(filePath); + } + } + + if (!noteMeta && (type === 'file' || type === 'image')) { + attributes.push({ + noteId, + type: 'label', + name: 'originalFileName', + value: path.basename(filePath) + }); + } + } + + /** @return {string} path without leading or trailing slash and backslashes converted to forward ones*/ + function normalizeFilePath(filePath) { + filePath = filePath.replace(/\\/g, "/"); + + if (filePath.startsWith("/")) { + filePath = filePath.substr(1); + } + + if (filePath.endsWith("/")) { + filePath = filePath.substr(0, filePath.length - 1); + } + + return filePath; + } + + function streamToBuffer(stream) { + const chunks = []; + stream.on('data', chunk => chunks.push(chunk)); + + return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks)))); + } + + function readZipFile(buffer) { + return new Promise((res, rej) => { + yauzl.fromBuffer(buffer, {lazyEntries: true, validateEntrySizes: false}, function(err, zipfile) { + function readContent(entry) { + return new Promise((res, rej) => { + zipfile.openReadStream(entry, function(err, readStream) { + if (err) rej(err); + + streamToBuffer(readStream).then(res); + }); + }); + } + + async function saveEntry(entry) { + const filePath = normalizeFilePath(entry.fileName); +console.log(filePath); + if (/\/$/.test(entry.fileName)) { + await saveDirectory(filePath); + } + else { + const content = await readContent(entry); + + if (filePath === '!!!meta.json') { + metaFile = JSON.parse(content.toString("UTF-8")); + } + else { + await saveNote(filePath, content); + } + } + + taskContext.increaseProgressCount(); + + zipfile.readEntry(); + } + + if (err) throw err; + zipfile.readEntry(); + zipfile.on("entry", saveEntry); + zipfile.on("end", res); + }); + }); + } + + await readZipFile(fileBuffer); + + const createdNoteIds = {}; + + for (const path in createdPaths) { + const noteId = createdPaths[path]; + + createdNoteIds[noteId] = true; + } + + for (const noteId in createdNoteIds) { // now the noteIds are unique + await noteService.scanForLinks(noteId); + + if (!metaFile) { + // if there's no meta file then the notes are created based on the order in that tar file but that + // is usually quite random so we sort the notes in the way they would appear in the file manager + await treeService.sortNotesAlphabetically(noteId, true); + } + + taskContext.increaseProgressCount(); + } + + // we're saving attributes and links only now so that all relation and link target notes + // are already in the database (we don't want to have "broken" relations, not even transitionally) + for (const attr of attributes) { + if (attr.type !== 'relation' || attr.value in createdNoteIds) { + await new Attribute(attr).save(); + } + else { + log.info("Relation not imported since target note doesn't exist: " + JSON.stringify(attr)); + } + } + + return firstNote; +} + +module.exports = { + importZip +}; \ No newline at end of file diff --git a/src/views/dialogs/import.ejs b/src/views/dialogs/import.ejs index 1cd6637ae..3da631026 100644 --- a/src/views/dialogs/import.ejs +++ b/src/views/dialogs/import.ejs @@ -21,21 +21,21 @@ Options:
-
-
-