mirror of
https://github.com/zadam/trilium.git
synced 2025-03-01 14:22:32 +01:00
zip import using yazl
This commit is contained in:
zadam
parent
04360381b6
commit
af5c4b5859
236
package-lock.json
generated
236
package-lock.json
generated
@ -825,43 +825,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"archiver-utils": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz",
|
||||
"integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==",
|
||||
"requires": {
|
||||
"glob": "^7.1.4",
|
||||
"graceful-fs": "^4.2.0",
|
||||
"lazystream": "^1.0.0",
|
||||
"lodash.defaults": "^4.2.0",
|
||||
"lodash.difference": "^4.5.0",
|
||||
"lodash.flatten": "^4.4.0",
|
||||
"lodash.isplainobject": "^4.0.6",
|
||||
"lodash.union": "^4.6.0",
|
||||
"normalize-path": "^3.0.0",
|
||||
"readable-stream": "^2.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"glob": {
|
||||
"version": "7.1.6",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
|
||||
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
|
||||
"requires": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"normalize-path": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
|
||||
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"are-we-there-yet": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
|
||||
@ -1072,11 +1035,6 @@
|
||||
"resolved": "https://registry.npmjs.org/beeper/-/beeper-1.1.1.tgz",
|
||||
"integrity": "sha1-5tXqjF2tABMEpwsiY4RH9pyy+Ak="
|
||||
},
|
||||
"big-integer": {
|
||||
"version": "1.6.48",
|
||||
"resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz",
|
||||
"integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w=="
|
||||
},
|
||||
"bin-build": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/bin-build/-/bin-build-2.2.0.tgz",
|
||||
@ -1175,15 +1133,6 @@
|
||||
"os-filter-obj": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"binary": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
|
||||
"integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
|
||||
"requires": {
|
||||
"buffers": "~0.1.1",
|
||||
"chainsaw": "~0.1.0"
|
||||
}
|
||||
},
|
||||
"bl": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
|
||||
@ -1433,11 +1382,6 @@
|
||||
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
|
||||
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
|
||||
},
|
||||
"buffer-indexof-polyfill": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz",
|
||||
"integrity": "sha1-qfuAbOgUXVQoUQznLyeLs2OmOL8="
|
||||
},
|
||||
"buffer-to-vinyl": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-to-vinyl/-/buffer-to-vinyl-1.1.0.tgz",
|
||||
@ -1461,11 +1405,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"buffers": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
|
||||
"integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
|
||||
},
|
||||
"builder-util": {
|
||||
"version": "22.4.1",
|
||||
"resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.4.1.tgz",
|
||||
@ -1675,14 +1614,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"chainsaw": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
|
||||
"integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
|
||||
"requires": {
|
||||
"traverse": ">=0.3.0 <0.4"
|
||||
}
|
||||
},
|
||||
"chalk": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
|
||||
@ -1887,24 +1818,6 @@
|
||||
"integrity": "sha1-AWLsLZNR9d3VmpICy6k1NmpyUIA=",
|
||||
"dev": true
|
||||
},
|
||||
"compress-commons": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-2.1.1.tgz",
|
||||
"integrity": "sha512-eVw6n7CnEMFzc3duyFVrQEuY1BlHR3rYsSztyG32ibGMW722i3C6IizEGMFmfMU+A+fALvBIwxN3czffTcdA+Q==",
|
||||
"requires": {
|
||||
"buffer-crc32": "^0.2.13",
|
||||
"crc32-stream": "^3.0.1",
|
||||
"normalize-path": "^3.0.0",
|
||||
"readable-stream": "^2.3.6"
|
||||
},
|
||||
"dependencies": {
|
||||
"normalize-path": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
|
||||
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
@ -2023,35 +1936,6 @@
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
|
||||
},
|
||||
"crc": {
|
||||
"version": "3.8.0",
|
||||
"resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz",
|
||||
"integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==",
|
||||
"requires": {
|
||||
"buffer": "^5.1.0"
|
||||
}
|
||||
},
|
||||
"crc32-stream": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-3.0.1.tgz",
|
||||
"integrity": "sha512-mctvpXlbzsvK+6z8kJwSJ5crm7yBwrQMTybJzMw1O4lLGJqjlDCXY2Zw7KheiA6XBEcBmfLx1D88mjRGVJtY9w==",
|
||||
"requires": {
|
||||
"crc": "^3.4.4",
|
||||
"readable-stream": "^3.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"readable-stream": {
|
||||
"version": "3.6.0",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
|
||||
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
|
||||
"requires": {
|
||||
"inherits": "^2.0.3",
|
||||
"string_decoder": "^1.1.1",
|
||||
"util-deprecate": "^1.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"create-error-class": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
|
||||
@ -4067,40 +3951,6 @@
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
|
||||
},
|
||||
"fstream": {
|
||||
"version": "1.0.12",
|
||||
"resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
|
||||
"integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
|
||||
"requires": {
|
||||
"graceful-fs": "^4.1.2",
|
||||
"inherits": "~2.0.0",
|
||||
"mkdirp": ">=0.5 0",
|
||||
"rimraf": "2"
|
||||
},
|
||||
"dependencies": {
|
||||
"glob": {
|
||||
"version": "7.1.6",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
|
||||
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
|
||||
"requires": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "2.7.1",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
|
||||
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
|
||||
"requires": {
|
||||
"glob": "^7.1.3"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"galactus": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/galactus/-/galactus-0.2.1.tgz",
|
||||
@ -5810,11 +5660,6 @@
|
||||
"uc.micro": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"listenercount": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz",
|
||||
"integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc="
|
||||
},
|
||||
"load-bmfont": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.0.tgz",
|
||||
@ -5914,16 +5759,6 @@
|
||||
"integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=",
|
||||
"dev": true
|
||||
},
|
||||
"lodash.defaults": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
|
||||
"integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw="
|
||||
},
|
||||
"lodash.difference": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz",
|
||||
"integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw="
|
||||
},
|
||||
"lodash.escape": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.escape/-/lodash.escape-3.2.0.tgz",
|
||||
@ -5932,11 +5767,6 @@
|
||||
"lodash._root": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"lodash.flatten": {
|
||||
"version": "4.4.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
|
||||
"integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8="
|
||||
},
|
||||
"lodash.get": {
|
||||
"version": "4.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
|
||||
@ -5963,11 +5793,6 @@
|
||||
"resolved": "https://registry.npmjs.org/lodash.isfinite/-/lodash.isfinite-3.3.2.tgz",
|
||||
"integrity": "sha1-+4m2WpqAKBgz8LdHizpRBPiY67M="
|
||||
},
|
||||
"lodash.isplainobject": {
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
|
||||
"integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
|
||||
},
|
||||
"lodash.keys": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz",
|
||||
@ -6013,11 +5838,6 @@
|
||||
"lodash.escape": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"lodash.union": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz",
|
||||
"integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg="
|
||||
},
|
||||
"log-symbols": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
|
||||
@ -9126,11 +8946,6 @@
|
||||
"resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz",
|
||||
"integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E="
|
||||
},
|
||||
"setimmediate": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
|
||||
"integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU="
|
||||
},
|
||||
"setprototypeof": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
|
||||
@ -9769,11 +9584,6 @@
|
||||
"punycode": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"traverse": {
|
||||
"version": "0.3.9",
|
||||
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
|
||||
"integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
|
||||
},
|
||||
"trim-newlines": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
|
||||
@ -9977,30 +9787,6 @@
|
||||
"resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz",
|
||||
"integrity": "sha1-uYTwh3/AqJwsdzzB73tbIytbBv4="
|
||||
},
|
||||
"unzipper": {
|
||||
"version": "0.10.10",
|
||||
"resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.10.tgz",
|
||||
"integrity": "sha512-wEgtqtrnJ/9zIBsQb8UIxOhAH1eTHfi7D/xvmrUoMEePeI6u24nq1wigazbIFtHt6ANYXdEVTvc8XYNlTurs7A==",
|
||||
"requires": {
|
||||
"big-integer": "^1.6.17",
|
||||
"binary": "~0.3.0",
|
||||
"bluebird": "~3.4.1",
|
||||
"buffer-indexof-polyfill": "~1.0.0",
|
||||
"duplexer2": "~0.1.4",
|
||||
"fstream": "^1.0.12",
|
||||
"graceful-fs": "^4.2.2",
|
||||
"listenercount": "~1.0.1",
|
||||
"readable-stream": "~2.3.6",
|
||||
"setimmediate": "~1.0.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"bluebird": {
|
||||
"version": "3.4.7",
|
||||
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
|
||||
"integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM="
|
||||
}
|
||||
}
|
||||
},
|
||||
"update-notifier": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.0.tgz",
|
||||
@ -10713,28 +10499,6 @@
|
||||
"requires": {
|
||||
"buffer-crc32": "~0.2.3"
|
||||
}
|
||||
},
|
||||
"zip-stream": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-2.1.3.tgz",
|
||||
"integrity": "sha512-EkXc2JGcKhO5N5aZ7TmuNo45budRaFGHOmz24wtJR7znbNqDPmdZtUauKX6et8KAVseAMBOyWJqEpXcHTBsh7Q==",
|
||||
"requires": {
|
||||
"archiver-utils": "^2.1.0",
|
||||
"compress-commons": "^2.1.1",
|
||||
"readable-stream": "^3.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"readable-stream": {
|
||||
"version": "3.6.0",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
|
||||
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
|
||||
"requires": {
|
||||
"inherits": "^2.0.3",
|
||||
"string_decoder": "^1.1.1",
|
||||
"util-deprecate": "^1.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,6 +72,7 @@
|
||||
"turndown-plugin-gfm": "1.0.2",
|
||||
"unescape": "1.0.1",
|
||||
"ws": "7.2.3",
|
||||
"yauzl": "^2.10.0",
|
||||
"yazl": "^2.5.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@ -64,7 +64,7 @@ ws.subscribeToMessages(async message => {
|
||||
toastService.showPersistent(toast);
|
||||
|
||||
if (message.result.importedNoteId) {
|
||||
await appContext.tabManager.getActiveTabContext.setNote(message.result.importedNoteId);
|
||||
await appContext.tabManager.getActiveTabContext().setNote(message.result.importedNoteId);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@ -4,6 +4,7 @@ const repository = require('../../services/repository');
|
||||
const enexImportService = require('../../services/import/enex');
|
||||
const opmlImportService = require('../../services/import/opml');
|
||||
const tarImportService = require('../../services/import/tar');
|
||||
const zipImportService = require('../../services/import/zip');
|
||||
const singleImportService = require('../../services/import/single');
|
||||
const cls = require('../../services/cls');
|
||||
const path = require('path');
|
||||
@ -48,6 +49,8 @@ async function importToBranch(req) {
|
||||
try {
|
||||
if (extension === '.tar' && options.explodeArchives) {
|
||||
note = await tarImportService.importTar(taskContext, file.buffer, parentNote);
|
||||
} else if (extension === '.zip' && options.explodeArchives) {
|
||||
note = await zipImportService.importZip(taskContext, file.buffer, parentNote);
|
||||
} else if (extension === '.opml' && options.explodeArchives) {
|
||||
note = await opmlImportService.importOpml(taskContext, file.buffer, parentNote);
|
||||
} else if (extension === '.enex' && options.explodeArchives) {
|
||||
|
||||
@ -13,8 +13,6 @@ const protectedSessionService = require('../protected_session');
|
||||
const sanitize = require("sanitize-filename");
|
||||
const fs = require("fs");
|
||||
const RESOURCE_DIR = require('../../services/resource_dir').RESOURCE_DIR;
|
||||
const ZipStream = require('zip-stream');
|
||||
const {Readable} = require('stream');
|
||||
const yazl = require("yazl");
|
||||
|
||||
/**
|
||||
|
||||
470
src/services/import/zip.js
Normal file
470
src/services/import/zip.js
Normal file
@ -0,0 +1,470 @@
|
||||
"use strict";
|
||||
|
||||
const Attribute = require('../../entities/attribute');
|
||||
const utils = require('../../services/utils');
|
||||
const log = require('../../services/log');
|
||||
const repository = require('../../services/repository');
|
||||
const noteService = require('../../services/notes');
|
||||
const attributeService = require('../../services/attributes');
|
||||
const Branch = require('../../entities/branch');
|
||||
const path = require('path');
|
||||
const commonmark = require('commonmark');
|
||||
const TaskContext = require('../task_context.js');
|
||||
const protectedSessionService = require('../protected_session');
|
||||
const mimeService = require("./mime");
|
||||
const treeService = require("../tree");
|
||||
const yauzl = require("yauzl");
|
||||
|
||||
/**
|
||||
* @param {TaskContext} taskContext
|
||||
* @param {Buffer} fileBuffer
|
||||
* @param {Note} importRootNote
|
||||
* @return {Promise<*>}
|
||||
*/
|
||||
async function importZip(taskContext, fileBuffer, importRootNote) {
|
||||
// maps from original noteId (in tar file) to newly generated noteId
|
||||
const noteIdMap = {};
|
||||
const attributes = [];
|
||||
// path => noteId
|
||||
const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId };
|
||||
const mdReader = new commonmark.Parser();
|
||||
const mdWriter = new commonmark.HtmlRenderer();
|
||||
let metaFile = null;
|
||||
let firstNote = null;
|
||||
|
||||
function getNewNoteId(origNoteId) {
|
||||
// in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution
|
||||
if (!origNoteId.trim()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!noteIdMap[origNoteId]) {
|
||||
noteIdMap[origNoteId] = utils.newEntityId();
|
||||
}
|
||||
|
||||
return noteIdMap[origNoteId];
|
||||
}
|
||||
|
||||
function getMeta(filePath) {
|
||||
if (!metaFile) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const pathSegments = filePath.split(/[\/\\]/g);
|
||||
|
||||
let cursor = {
|
||||
isImportRoot: true,
|
||||
children: metaFile.files
|
||||
};
|
||||
|
||||
let parent;
|
||||
|
||||
for (const segment of pathSegments) {
|
||||
if (!cursor || !cursor.children || cursor.children.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
parent = cursor;
|
||||
cursor = cursor.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
|
||||
}
|
||||
|
||||
return {
|
||||
parentNoteMeta: parent,
|
||||
noteMeta: cursor
|
||||
};
|
||||
}
|
||||
|
||||
async function getParentNoteId(filePath, parentNoteMeta) {
|
||||
let parentNoteId;
|
||||
|
||||
if (parentNoteMeta) {
|
||||
parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId);
|
||||
}
|
||||
else {
|
||||
const parentPath = path.dirname(filePath);
|
||||
|
||||
if (parentPath === '.') {
|
||||
parentNoteId = importRootNote.noteId;
|
||||
}
|
||||
else if (parentPath in createdPaths) {
|
||||
parentNoteId = createdPaths[parentPath];
|
||||
}
|
||||
else {
|
||||
// tar allows creating out of order records - i.e. file in a directory can appear in the tar stream before actual directory
|
||||
// (out-of-order-directory-records.tar in test set)
|
||||
parentNoteId = await saveDirectory(parentPath);
|
||||
}
|
||||
}
|
||||
|
||||
return parentNoteId;
|
||||
}
|
||||
|
||||
function getNoteTitle(filePath, noteMeta) {
|
||||
if (noteMeta) {
|
||||
return noteMeta.title;
|
||||
}
|
||||
else {
|
||||
const basename = path.basename(filePath);
|
||||
|
||||
return getTextFileWithoutExtension(basename);
|
||||
}
|
||||
}
|
||||
|
||||
function getNoteId(noteMeta, filePath) {
|
||||
const filePathNoExt = getTextFileWithoutExtension(filePath);
|
||||
|
||||
if (filePathNoExt in createdPaths) {
|
||||
return createdPaths[filePathNoExt];
|
||||
}
|
||||
|
||||
const noteId = noteMeta ? getNewNoteId(noteMeta.noteId) : utils.newEntityId();
|
||||
|
||||
createdPaths[filePathNoExt] = noteId;
|
||||
|
||||
return noteId;
|
||||
}
|
||||
|
||||
function detectFileTypeAndMime(taskContext, filePath) {
|
||||
const mime = mimeService.getMime(filePath) || "application/octet-stream";
|
||||
const type = mimeService.getType(taskContext.data, mime);
|
||||
|
||||
return { mime, type };
|
||||
}
|
||||
|
||||
async function saveAttributes(note, noteMeta) {
|
||||
if (!noteMeta) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const attr of noteMeta.attributes) {
|
||||
attr.noteId = note.noteId;
|
||||
|
||||
if (!attributeService.isAttributeType(attr.type)) {
|
||||
log.error("Unrecognized attribute type " + attr.type);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attr.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(attr.name)) {
|
||||
// these relations are created automatically and as such don't need to be duplicated in the import
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attr.type === 'label' && attr.name === 'externalLink') {
|
||||
// also created automatically
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attr.type === 'relation') {
|
||||
attr.value = getNewNoteId(attr.value);
|
||||
}
|
||||
|
||||
if (taskContext.data.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) {
|
||||
attr.name = 'disabled-' + attr.name;
|
||||
}
|
||||
|
||||
attributes.push(attr);
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDirectory(filePath) {
|
||||
const { parentNoteMeta, noteMeta } = getMeta(filePath);
|
||||
|
||||
const noteId = getNoteId(noteMeta, filePath);
|
||||
const noteTitle = getNoteTitle(filePath, noteMeta);
|
||||
const parentNoteId = await getParentNoteId(filePath, parentNoteMeta);
|
||||
|
||||
let note = await repository.getNote(noteId);
|
||||
|
||||
if (note) {
|
||||
return;
|
||||
}
|
||||
|
||||
({note} = await noteService.createNewNote({
|
||||
parentNoteId: parentNoteId,
|
||||
title: noteTitle,
|
||||
content: '',
|
||||
noteId: noteId,
|
||||
type: noteMeta ? noteMeta.type : 'text',
|
||||
mime: noteMeta ? noteMeta.mime : 'text/html',
|
||||
prefix: noteMeta ? noteMeta.prefix : '',
|
||||
isExpanded: noteMeta ? noteMeta.isExpanded : false,
|
||||
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
|
||||
}));
|
||||
|
||||
await saveAttributes(note, noteMeta);
|
||||
|
||||
if (!firstNote) {
|
||||
firstNote = note;
|
||||
}
|
||||
|
||||
return noteId;
|
||||
}
|
||||
|
||||
function getTextFileWithoutExtension(filePath) {
|
||||
const extension = path.extname(filePath).toLowerCase();
|
||||
|
||||
if (extension === '.md' || extension === '.html') {
|
||||
return filePath.substr(0, filePath.length - extension.length);
|
||||
}
|
||||
else {
|
||||
return filePath;
|
||||
}
|
||||
}
|
||||
|
||||
function getNoteIdFromRelativeUrl(url, filePath) {
|
||||
while (url.startsWith("./")) {
|
||||
url = url.substr(2);
|
||||
}
|
||||
|
||||
let absUrl = path.dirname(filePath);
|
||||
|
||||
while (url.startsWith("../")) {
|
||||
absUrl = path.dirname(absUrl);
|
||||
|
||||
url = url.substr(3);
|
||||
}
|
||||
|
||||
if (absUrl === '.') {
|
||||
absUrl = '';
|
||||
}
|
||||
|
||||
absUrl += (absUrl.length > 0 ? '/' : '') + url;
|
||||
|
||||
const {noteMeta} = getMeta(absUrl);
|
||||
const targetNoteId = getNoteId(noteMeta, absUrl);
|
||||
return targetNoteId;
|
||||
}
|
||||
|
||||
async function saveNote(filePath, content) {
|
||||
const {parentNoteMeta, noteMeta} = getMeta(filePath);
|
||||
|
||||
if (noteMeta && noteMeta.noImport) {
|
||||
return;
|
||||
}
|
||||
|
||||
const noteId = getNoteId(noteMeta, filePath);
|
||||
const parentNoteId = await getParentNoteId(filePath, parentNoteMeta);
|
||||
|
||||
if (noteMeta && noteMeta.isClone) {
|
||||
await new Branch({
|
||||
noteId,
|
||||
parentNoteId,
|
||||
isExpanded: noteMeta.isExpanded,
|
||||
prefix: noteMeta.prefix,
|
||||
notePosition: noteMeta.notePosition
|
||||
}).save();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath);
|
||||
|
||||
if (type !== 'file' && type !== 'image') {
|
||||
content = content.toString("UTF-8");
|
||||
}
|
||||
|
||||
if ((noteMeta && noteMeta.format === 'markdown')
|
||||
|| (!noteMeta && taskContext.data.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime))) {
|
||||
const parsed = mdReader.parse(content);
|
||||
content = mdWriter.render(parsed);
|
||||
}
|
||||
|
||||
const noteTitle = getNoteTitle(filePath, noteMeta);
|
||||
|
||||
if (type === 'text') {
|
||||
function isUrlAbsolute(url) {
|
||||
return /^(?:[a-z]+:)?\/\//i.test(url);
|
||||
}
|
||||
|
||||
content = content.replace(/<html.*<body[^>]*>/gis, "");
|
||||
content = content.replace(/<\/body>.*<\/html>/gis, "");
|
||||
|
||||
content = content.replace(/src="([^"]*)"/g, (match, url) => {
|
||||
url = decodeURIComponent(url);
|
||||
|
||||
if (isUrlAbsolute(url) || url.startsWith("/")) {
|
||||
return match;
|
||||
}
|
||||
|
||||
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
|
||||
|
||||
return `src="api/images/${targetNoteId}/${path.basename(url)}"`;
|
||||
});
|
||||
|
||||
content = content.replace(/href="([^"]*)"/g, (match, url) => {
|
||||
url = decodeURIComponent(url);
|
||||
|
||||
if (isUrlAbsolute(url)) {
|
||||
return match;
|
||||
}
|
||||
|
||||
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
|
||||
|
||||
return `href="#root/${targetNoteId}"`;
|
||||
});
|
||||
|
||||
content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
|
||||
if (noteTitle.trim() === text.trim()) {
|
||||
return ""; // remove whole H1 tag
|
||||
}
|
||||
else {
|
||||
return match;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (type === 'relation-map' && noteMeta) {
|
||||
const relationMapLinks = (noteMeta.attributes || [])
|
||||
.filter(attr => attr.type === 'relation' && attr.name === 'relationMapLink');
|
||||
|
||||
// this will replace relation map links
|
||||
for (const link of relationMapLinks) {
|
||||
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
|
||||
content = content.replace(new RegExp(link.value, "g"), getNewNoteId(link.value));
|
||||
}
|
||||
}
|
||||
|
||||
let note = await repository.getNote(noteId);
|
||||
|
||||
if (note) {
|
||||
await note.setContent(content);
|
||||
}
|
||||
else {
|
||||
({note} = await noteService.createNewNote({
|
||||
parentNoteId: parentNoteId,
|
||||
title: noteTitle,
|
||||
content: content,
|
||||
noteId,
|
||||
type,
|
||||
mime,
|
||||
prefix: noteMeta ? noteMeta.prefix : '',
|
||||
isExpanded: noteMeta ? noteMeta.isExpanded : false,
|
||||
notePosition: noteMeta ? noteMeta.notePosition : false,
|
||||
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
|
||||
}));
|
||||
|
||||
await saveAttributes(note, noteMeta);
|
||||
|
||||
if (!firstNote) {
|
||||
firstNote = note;
|
||||
}
|
||||
|
||||
if (type === 'text') {
|
||||
filePath = getTextFileWithoutExtension(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (!noteMeta && (type === 'file' || type === 'image')) {
|
||||
attributes.push({
|
||||
noteId,
|
||||
type: 'label',
|
||||
name: 'originalFileName',
|
||||
value: path.basename(filePath)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** @return {string} path without leading or trailing slash and backslashes converted to forward ones*/
|
||||
function normalizeFilePath(filePath) {
|
||||
filePath = filePath.replace(/\\/g, "/");
|
||||
|
||||
if (filePath.startsWith("/")) {
|
||||
filePath = filePath.substr(1);
|
||||
}
|
||||
|
||||
if (filePath.endsWith("/")) {
|
||||
filePath = filePath.substr(0, filePath.length - 1);
|
||||
}
|
||||
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function streamToBuffer(stream) {
|
||||
const chunks = [];
|
||||
stream.on('data', chunk => chunks.push(chunk));
|
||||
|
||||
return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks))));
|
||||
}
|
||||
|
||||
function readZipFile(buffer) {
|
||||
return new Promise((res, rej) => {
|
||||
yauzl.fromBuffer(buffer, {lazyEntries: true, validateEntrySizes: false}, function(err, zipfile) {
|
||||
function readContent(entry) {
|
||||
return new Promise((res, rej) => {
|
||||
zipfile.openReadStream(entry, function(err, readStream) {
|
||||
if (err) rej(err);
|
||||
|
||||
streamToBuffer(readStream).then(res);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function saveEntry(entry) {
|
||||
const filePath = normalizeFilePath(entry.fileName);
|
||||
console.log(filePath);
|
||||
if (/\/$/.test(entry.fileName)) {
|
||||
await saveDirectory(filePath);
|
||||
}
|
||||
else {
|
||||
const content = await readContent(entry);
|
||||
|
||||
if (filePath === '!!!meta.json') {
|
||||
metaFile = JSON.parse(content.toString("UTF-8"));
|
||||
}
|
||||
else {
|
||||
await saveNote(filePath, content);
|
||||
}
|
||||
}
|
||||
|
||||
taskContext.increaseProgressCount();
|
||||
|
||||
zipfile.readEntry();
|
||||
}
|
||||
|
||||
if (err) throw err;
|
||||
zipfile.readEntry();
|
||||
zipfile.on("entry", saveEntry);
|
||||
zipfile.on("end", res);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
await readZipFile(fileBuffer);
|
||||
|
||||
const createdNoteIds = {};
|
||||
|
||||
for (const path in createdPaths) {
|
||||
const noteId = createdPaths[path];
|
||||
|
||||
createdNoteIds[noteId] = true;
|
||||
}
|
||||
|
||||
for (const noteId in createdNoteIds) { // now the noteIds are unique
|
||||
await noteService.scanForLinks(noteId);
|
||||
|
||||
if (!metaFile) {
|
||||
// if there's no meta file then the notes are created based on the order in that tar file but that
|
||||
// is usually quite random so we sort the notes in the way they would appear in the file manager
|
||||
await treeService.sortNotesAlphabetically(noteId, true);
|
||||
}
|
||||
|
||||
taskContext.increaseProgressCount();
|
||||
}
|
||||
|
||||
// we're saving attributes and links only now so that all relation and link target notes
|
||||
// are already in the database (we don't want to have "broken" relations, not even transitionally)
|
||||
for (const attr of attributes) {
|
||||
if (attr.type !== 'relation' || attr.value in createdNoteIds) {
|
||||
await new Attribute(attr).save();
|
||||
}
|
||||
else {
|
||||
log.info("Relation not imported since target note doesn't exist: " + JSON.stringify(attr));
|
||||
}
|
||||
}
|
||||
|
||||
return firstNote;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
importZip
|
||||
};
|
||||
@ -21,21 +21,21 @@
|
||||
<strong>Options:</strong>
|
||||
|
||||
<div class="checkbox">
|
||||
<label data-toggle="tooltip" title="Trilium <code>.tar</code> export files can contain executable scripts which may contain harmful behavior. Safe import will deactivate automatic execution of all imported scripts. Uncheck "Safe import" only if the imported tar archive is supposed to contain executable scripts and you completely trust the contents of the import file.">
|
||||
<label data-toggle="tooltip" title="Trilium <code>.zip</code>, <code>.tar</code> export files can contain executable scripts which may contain harmful behavior. Safe import will deactivate automatic execution of all imported scripts. Uncheck "Safe import" only if the imported tar archive is supposed to contain executable scripts and you completely trust the contents of the import file.">
|
||||
<input id="safe-import-checkbox" value="1" type="checkbox" checked>
|
||||
<span>Safe import</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox">
|
||||
<label data-toggle="tooltip" title="If this is checked then Trilium will read <code>.tar</code>, <code>.enex</code> and <code>.opml</code> files and create notes from files insides those archives. If unchecked, then Trilium will attach the archives themselves to the note.">
|
||||
<label data-toggle="tooltip" title="If this is checked then Trilium will read <code>.zip</code>, <code>.tar</code>, <code>.enex</code> and <code>.opml</code> files and create notes from files insides those archives. If unchecked, then Trilium will attach the archives themselves to the note.">
|
||||
<input id="explode-archives-checkbox" value="1" type="checkbox" checked>
|
||||
<span>Read contents of <code>.tar</code>, <code>.enex</code> and <code>.opml</code> archives.</span>
|
||||
<span>Read contents of <code>.zip</code>, <code>.tar</code>, <code>.enex</code> and <code>.opml</code> archives.</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox">
|
||||
<label data-toggle="tooltip" title="<p>If you check this option, Trilium will attempt to shrink the imported images by scaling and optimization which may affect the perceived image quality. If unchecked, images will be imported without changes.</p><p>This doesn't apply to <code>.tar</code> imports with metadata since it is assumed these files are already optimized.</p>">
|
||||
<label data-toggle="tooltip" title="<p>If you check this option, Trilium will attempt to shrink the imported images by scaling and optimization which may affect the perceived image quality. If unchecked, images will be imported without changes.</p><p>This doesn't apply to <code>.zip</code>, <code>.tar</code> imports with metadata since it is assumed these files are already optimized.</p>">
|
||||
<input id="shrink-images-checkbox" value="1" type="checkbox" checked> <span>Shrink images</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user