mirror of
https://github.com/zadam/trilium.git
synced 2025-06-06 18:08:33 +02:00
removed tar import
This commit is contained in:
zadam
parent
054420225f
commit
055db32510
@ -23,7 +23,7 @@
|
||||
"test-es6": "node -r esm spec-es6/attribute_parser.spec.js "
|
||||
},
|
||||
"dependencies": {
|
||||
"async-mutex": "0.2.3",
|
||||
"async-mutex": "0.2.4",
|
||||
"axios": "0.19.2",
|
||||
"better-sqlite3": "^7.1.0",
|
||||
"body-parser": "1.19.0",
|
||||
@ -66,7 +66,6 @@
|
||||
"serve-favicon": "2.5.0",
|
||||
"session-file-store": "1.4.0",
|
||||
"string-similarity": "4.0.1",
|
||||
"tar-stream": "2.1.2",
|
||||
"turndown": "6.0.0",
|
||||
"turndown-plugin-gfm": "1.0.2",
|
||||
"unescape": "1.0.1",
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
const repository = require('../../services/repository');
|
||||
const enexImportService = require('../../services/import/enex');
|
||||
const opmlImportService = require('../../services/import/opml');
|
||||
const tarImportService = require('../../services/import/tar');
|
||||
const zipImportService = require('../../services/import/zip');
|
||||
const singleImportService = require('../../services/import/single');
|
||||
const cls = require('../../services/cls');
|
||||
@ -48,9 +47,7 @@ async function importToBranch(req) {
|
||||
const taskContext = TaskContext.getInstance(taskId, 'import', options);
|
||||
|
||||
try {
|
||||
if (extension === '.tar' && options.explodeArchives) {
|
||||
note = await tarImportService.importTar(taskContext, file.buffer, parentNote);
|
||||
} else if (extension === '.zip' && options.explodeArchives) {
|
||||
if (extension === '.zip' && options.explodeArchives) {
|
||||
note = await zipImportService.importZip(taskContext, file.buffer, parentNote);
|
||||
} else if (extension === '.opml' && options.explodeArchives) {
|
||||
note = await opmlImportService.importOpml(taskContext, file.buffer, parentNote);
|
||||
|
||||
@ -1,446 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
const Attribute = require('../../entities/attribute');
|
||||
const utils = require('../../services/utils');
|
||||
const log = require('../../services/log');
|
||||
const repository = require('../../services/repository');
|
||||
const noteService = require('../../services/notes');
|
||||
const attributeService = require('../../services/attributes');
|
||||
const Branch = require('../../entities/branch');
|
||||
const tar = require('tar-stream');
|
||||
const stream = require('stream');
|
||||
const path = require('path');
|
||||
const commonmark = require('commonmark');
|
||||
const TaskContext = require('../task_context.js');
|
||||
const protectedSessionService = require('../protected_session');
|
||||
const mimeService = require("./mime");
|
||||
const sql = require("../sql");
|
||||
const treeService = require("../tree");
|
||||
const htmlSanitizer = require("../html_sanitizer");
|
||||
|
||||
/**
|
||||
* @param {TaskContext} taskContext
|
||||
* @param {Buffer} fileBuffer
|
||||
* @param {Note} importRootNote
|
||||
* @return {Promise<*>}
|
||||
*/
|
||||
async function importTar(taskContext, fileBuffer, importRootNote) {
|
||||
// maps from original noteId (in tar file) to newly generated noteId
|
||||
const noteIdMap = {};
|
||||
const attributes = [];
|
||||
// path => noteId
|
||||
const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId };
|
||||
const mdReader = new commonmark.Parser();
|
||||
const mdWriter = new commonmark.HtmlRenderer();
|
||||
let metaFile = null;
|
||||
let firstNote = null;
|
||||
|
||||
const extract = tar.extract();
|
||||
|
||||
function getNewNoteId(origNoteId) {
|
||||
// in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution
|
||||
if (!origNoteId.trim()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!noteIdMap[origNoteId]) {
|
||||
noteIdMap[origNoteId] = utils.newEntityId();
|
||||
}
|
||||
|
||||
return noteIdMap[origNoteId];
|
||||
}
|
||||
|
||||
function getMeta(filePath) {
|
||||
if (!metaFile) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const pathSegments = filePath.split(/[\/\\]/g);
|
||||
|
||||
let cursor = {
|
||||
isImportRoot: true,
|
||||
children: metaFile.files
|
||||
};
|
||||
|
||||
let parent;
|
||||
|
||||
for (const segment of pathSegments) {
|
||||
if (!cursor || !cursor.children || cursor.children.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
parent = cursor;
|
||||
cursor = cursor.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
|
||||
}
|
||||
|
||||
return {
|
||||
parentNoteMeta: parent,
|
||||
noteMeta: cursor
|
||||
};
|
||||
}
|
||||
|
||||
function getParentNoteId(filePath, parentNoteMeta) {
|
||||
let parentNoteId;
|
||||
|
||||
if (parentNoteMeta) {
|
||||
parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId);
|
||||
}
|
||||
else {
|
||||
const parentPath = path.dirname(filePath);
|
||||
|
||||
if (parentPath === '.') {
|
||||
parentNoteId = importRootNote.noteId;
|
||||
}
|
||||
else if (parentPath in createdPaths) {
|
||||
parentNoteId = createdPaths[parentPath];
|
||||
}
|
||||
else {
|
||||
// tar allows creating out of order records - i.e. file in a directory can appear in the tar stream before actual directory
|
||||
// (out-of-order-directory-records.tar in test set)
|
||||
parentNoteId = saveDirectory(parentPath);
|
||||
}
|
||||
}
|
||||
|
||||
return parentNoteId;
|
||||
}
|
||||
|
||||
function getNoteId(noteMeta, filePath) {
|
||||
const filePathNoExt = utils.removeTextFileExtension(filePath);
|
||||
|
||||
if (filePathNoExt in createdPaths) {
|
||||
return createdPaths[filePathNoExt];
|
||||
}
|
||||
|
||||
const noteId = noteMeta ? getNewNoteId(noteMeta.noteId) : utils.newEntityId();
|
||||
|
||||
createdPaths[filePathNoExt] = noteId;
|
||||
|
||||
return noteId;
|
||||
}
|
||||
|
||||
function detectFileTypeAndMime(taskContext, filePath) {
|
||||
const mime = mimeService.getMime(filePath) || "application/octet-stream";
|
||||
const type = mimeService.getType(taskContext.data, mime);
|
||||
|
||||
return { mime, type };
|
||||
}
|
||||
|
||||
function saveAttributes(note, noteMeta) {
|
||||
if (!noteMeta) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const attr of noteMeta.attributes) {
|
||||
attr.noteId = note.noteId;
|
||||
|
||||
if (!attributeService.isAttributeType(attr.type)) {
|
||||
log.error("Unrecognized attribute type " + attr.type);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attr.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(attr.name)) {
|
||||
// these relations are created automatically and as such don't need to be duplicated in the import
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attr.type === 'relation') {
|
||||
attr.value = getNewNoteId(attr.value);
|
||||
}
|
||||
|
||||
if (taskContext.data.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) {
|
||||
attr.name = 'disabled-' + attr.name;
|
||||
}
|
||||
|
||||
attributes.push(attr);
|
||||
}
|
||||
}
|
||||
|
||||
function saveDirectory(filePath) {
|
||||
const { parentNoteMeta, noteMeta } = getMeta(filePath);
|
||||
|
||||
const noteId = getNoteId(noteMeta, filePath);
|
||||
const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
|
||||
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
|
||||
|
||||
let note = repository.getNote(noteId);
|
||||
|
||||
if (note) {
|
||||
return;
|
||||
}
|
||||
|
||||
sql.transactional(() => {
|
||||
({note} = noteService.createNewNote({
|
||||
parentNoteId: parentNoteId,
|
||||
title: noteTitle,
|
||||
content: '',
|
||||
noteId: noteId,
|
||||
type: noteMeta ? noteMeta.type : 'text',
|
||||
mime: noteMeta ? noteMeta.mime : 'text/html',
|
||||
prefix: noteMeta ? noteMeta.prefix : '',
|
||||
isExpanded: noteMeta ? noteMeta.isExpanded : false,
|
||||
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
|
||||
}));
|
||||
|
||||
saveAttributes(note, noteMeta);
|
||||
});
|
||||
|
||||
if (!firstNote) {
|
||||
firstNote = note;
|
||||
}
|
||||
|
||||
return noteId;
|
||||
}
|
||||
|
||||
function getNoteIdFromRelativeUrl(url, filePath) {
|
||||
while (url.startsWith("./")) {
|
||||
url = url.substr(2);
|
||||
}
|
||||
|
||||
let absUrl = path.dirname(filePath);
|
||||
|
||||
while (url.startsWith("../")) {
|
||||
absUrl = path.dirname(absUrl);
|
||||
|
||||
url = url.substr(3);
|
||||
}
|
||||
|
||||
if (absUrl === '.') {
|
||||
absUrl = '';
|
||||
}
|
||||
|
||||
absUrl += (absUrl.length > 0 ? '/' : '') + url;
|
||||
|
||||
const {noteMeta} = getMeta(absUrl);
|
||||
const targetNoteId = getNoteId(noteMeta, absUrl);
|
||||
return targetNoteId;
|
||||
}
|
||||
|
||||
function saveNote(filePath, content) {
|
||||
const {parentNoteMeta, noteMeta} = getMeta(filePath);
|
||||
|
||||
if (noteMeta && noteMeta.noImport) {
|
||||
return;
|
||||
}
|
||||
|
||||
const noteId = getNoteId(noteMeta, filePath);
|
||||
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
|
||||
|
||||
if (noteMeta && noteMeta.isClone) {
|
||||
new Branch({
|
||||
noteId,
|
||||
parentNoteId,
|
||||
isExpanded: noteMeta.isExpanded,
|
||||
prefix: noteMeta.prefix,
|
||||
notePosition: noteMeta.notePosition
|
||||
}).save();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath);
|
||||
|
||||
if (type !== 'file' && type !== 'image') {
|
||||
content = content.toString("UTF-8");
|
||||
}
|
||||
|
||||
if ((noteMeta && noteMeta.format === 'markdown')
|
||||
|| (!noteMeta && taskContext.data.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime))) {
|
||||
const parsed = mdReader.parse(content);
|
||||
content = mdWriter.render(parsed);
|
||||
}
|
||||
|
||||
const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta);
|
||||
|
||||
if (type === 'text') {
|
||||
function isUrlAbsolute(url) {
|
||||
return /^(?:[a-z]+:)?\/\//i.test(url);
|
||||
}
|
||||
|
||||
content = htmlSanitizer.sanitize(content);
|
||||
|
||||
content = content.replace(/<html.*<body[^>]*>/gis, "");
|
||||
content = content.replace(/<\/body>.*<\/html>/gis, "");
|
||||
|
||||
content = content.replace(/src="([^"]*)"/g, (match, url) => {
|
||||
url = decodeURIComponent(url);
|
||||
|
||||
if (isUrlAbsolute(url) || url.startsWith("/")) {
|
||||
return match;
|
||||
}
|
||||
|
||||
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
|
||||
|
||||
return `src="api/images/${targetNoteId}/${path.basename(url)}"`;
|
||||
});
|
||||
|
||||
content = content.replace(/href="([^"]*)"/g, (match, url) => {
|
||||
url = decodeURIComponent(url);
|
||||
|
||||
if (isUrlAbsolute(url)) {
|
||||
return match;
|
||||
}
|
||||
|
||||
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
|
||||
|
||||
return `href="#root/${targetNoteId}"`;
|
||||
});
|
||||
|
||||
content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
|
||||
if (noteTitle.trim() === text.trim()) {
|
||||
return ""; // remove whole H1 tag
|
||||
}
|
||||
else {
|
||||
return match;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (type === 'relation-map' && noteMeta) {
|
||||
const relationMapLinks = (noteMeta.attributes || [])
|
||||
.filter(attr => attr.type === 'relation' && attr.name === 'relationMapLink');
|
||||
|
||||
// this will replace relation map links
|
||||
for (const link of relationMapLinks) {
|
||||
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
|
||||
content = content.replace(new RegExp(link.value, "g"), getNewNoteId(link.value));
|
||||
}
|
||||
}
|
||||
|
||||
let note = repository.getNote(noteId);
|
||||
|
||||
if (note) {
|
||||
note.setContent(content);
|
||||
}
|
||||
else {
|
||||
({note} = noteService.createNewNote({
|
||||
parentNoteId: parentNoteId,
|
||||
title: noteTitle,
|
||||
content: content,
|
||||
noteId,
|
||||
type,
|
||||
mime,
|
||||
prefix: noteMeta ? noteMeta.prefix : '',
|
||||
isExpanded: noteMeta ? noteMeta.isExpanded : false,
|
||||
notePosition: noteMeta ? noteMeta.notePosition : false,
|
||||
isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
|
||||
}));
|
||||
|
||||
saveAttributes(note, noteMeta);
|
||||
|
||||
if (!firstNote) {
|
||||
firstNote = note;
|
||||
}
|
||||
|
||||
if (type === 'text') {
|
||||
filePath = utils.removeTextFileExtension(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (!noteMeta && (type === 'file' || type === 'image')) {
|
||||
attributes.push({
|
||||
noteId,
|
||||
type: 'label',
|
||||
name: 'originalFileName',
|
||||
value: path.basename(filePath)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** @return {string} path without leading or trailing slash and backslashes converted to forward ones*/
|
||||
function normalizeFilePath(filePath) {
|
||||
filePath = filePath.replace(/\\/g, "/");
|
||||
|
||||
if (filePath.startsWith("/")) {
|
||||
filePath = filePath.substr(1);
|
||||
}
|
||||
|
||||
if (filePath.endsWith("/")) {
|
||||
filePath = filePath.substr(0, filePath.length - 1);
|
||||
}
|
||||
|
||||
return filePath;
|
||||
}
|
||||
|
||||
extract.on('entry', function(header, stream, next) {
|
||||
const chunks = [];
|
||||
|
||||
stream.on("data", function (chunk) {
|
||||
chunks.push(chunk);
|
||||
});
|
||||
|
||||
// header is the tar header
|
||||
// stream is the content body (might be an empty stream)
|
||||
// call next when you are done with this entry
|
||||
|
||||
stream.on('end', function() {
|
||||
const filePath = normalizeFilePath(header.name);
|
||||
|
||||
const content = Buffer.concat(chunks);
|
||||
|
||||
if (filePath === '!!!meta.json') {
|
||||
metaFile = JSON.parse(content.toString("UTF-8"));
|
||||
}
|
||||
else if (header.type === 'directory') {
|
||||
saveDirectory(filePath);
|
||||
}
|
||||
else if (header.type === 'file') {
|
||||
saveNote(filePath, content);
|
||||
}
|
||||
else {
|
||||
log.info("Ignoring tar import entry with type " + header.type);
|
||||
}
|
||||
|
||||
taskContext.increaseProgressCount();
|
||||
|
||||
next(); // ready for next entry
|
||||
});
|
||||
|
||||
stream.resume(); // just auto drain the stream
|
||||
});
|
||||
|
||||
return new Promise(resolve => {
|
||||
extract.on('finish', function() {
|
||||
const createdNoteIds = {};
|
||||
|
||||
for (const path in createdPaths) {
|
||||
const noteId = createdPaths[path];
|
||||
|
||||
createdNoteIds[noteId] = true;
|
||||
}
|
||||
|
||||
for (const noteId in createdNoteIds) { // now the noteIds are unique
|
||||
noteService.scanForLinks(repository.getNote(noteId));
|
||||
|
||||
if (!metaFile) {
|
||||
// if there's no meta file then the notes are created based on the order in that tar file but that
|
||||
// is usually quite random so we sort the notes in the way they would appear in the file manager
|
||||
treeService.sortNotesAlphabetically(noteId, true);
|
||||
}
|
||||
|
||||
taskContext.increaseProgressCount();
|
||||
}
|
||||
|
||||
// we're saving attributes and links only now so that all relation and link target notes
|
||||
// are already in the database (we don't want to have "broken" relations, not even transitionally)
|
||||
for (const attr of attributes) {
|
||||
if (attr.type !== 'relation' || attr.value in createdNoteIds) {
|
||||
new Attribute(attr).save();
|
||||
}
|
||||
else {
|
||||
log.info("Relation not imported since target note doesn't exist: " + JSON.stringify(attr));
|
||||
}
|
||||
}
|
||||
|
||||
resolve(firstNote);
|
||||
});
|
||||
|
||||
const bufferStream = new stream.PassThrough();
|
||||
bufferStream.end(fileBuffer);
|
||||
|
||||
bufferStream.pipe(extract);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
importTar
|
||||
};
|
||||
Loading…
x
Reference in New Issue
Block a user