gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-06-06 18:08:33 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

import/export logic refactored into separate files per format, closes #237

Browse Source
This commit is contained in:
azivner 2018-11-16 12:12:04 +01:00
parent e0f9a7fc6a
commit 5b9a1ef0e9
11 changed files with 645 additions and 583 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/public/javascripts/dialogs/export_subtree.js
View File

@ -6,7 +6,6 @@ import exportService from "../services/export.js";
const $dialog = $("#export-subtree-dialog"); const $dialog = $("#export-subtree-dialog");
const $form = $("#export-subtree-form"); const $form = $("#export-subtree-form");
const $noteTitle = $dialog.find(".note-title"); const $noteTitle = $dialog.find(".note-title");
const $exportFormat = $dialog.find("input[name='export-format']:checked");
async function showDialog() { async function showDialog() {
glob.activeDialog = $dialog; glob.activeDialog = $dialog;
@ -20,7 +19,7 @@ async function showDialog() {
} }
$form.submit(() => { $form.submit(() => {
const exportFormat = $exportFormat.val(); const exportFormat = $dialog.find("input[name='export-format']:checked").val();
const currentNode = treeService.getCurrentNode(); const currentNode = treeService.getCurrentNode();

260
src/routes/api/export.js
View File

@ -1,270 +1,34 @@
"use strict"; "use strict";
const html = require('html'); const nativeTarExportService = require('../../services/export/native_tar');
const tar = require('tar-stream'); const markdownTarExportService = require('../../services/export/markdown_tar');
const sanitize = require("sanitize-filename"); const markdownSingleExportService = require('../../services/export/markdown_single');
const opmlExportService = require('../../services/export/opml');
const repository = require("../../services/repository"); const repository = require("../../services/repository");
const utils = require('../../services/utils');
const TurndownService = require('turndown');
async function exportNote(req, res) { async function exportNote(req, res) {
// entityId maybe either noteId or branchId depending on format // entityId maybe either noteId or branchId depending on format
const entityId = req.params.entityId; const entityId = req.params.entityId;
const format = req.params.format; const format = req.params.format;
if (format === 'tar') { if (format === 'native-tar') {
await exportToTar(await repository.getBranch(entityId), res); await nativeTarExportService.exportToTar(await repository.getBranch(entityId), res);
} }
else if (format === 'opml') { else if (format === 'markdown-tar') {
await exportToOpml(await repository.getBranch(entityId), res); await markdownTarExportService.exportToMarkdown(await repository.getBranch(entityId), res);
}
else if (format === 'markdown') {
await exportToMarkdown(await repository.getBranch(entityId), res);
} }
// export single note without subtree // export single note without subtree
else if (format === 'markdown-single') { else if (format === 'markdown-single') {
await exportSingleMarkdown(await repository.getNote(entityId), res); await markdownSingleExportService.exportSingleMarkdown(await repository.getNote(entityId), res);
}
else if (format === 'opml') {
await opmlExportService.exportToOpml(await repository.getBranch(entityId), res);
} }
else { else {
return [404, "Unrecognized export format " + format]; return [404, "Unrecognized export format " + format];
} }
} }
function escapeXmlAttribute(text) {
return text.replace(/&/g, '&')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;');
}
function prepareText(text) {
const newLines = text.replace(/(<p[^>]*>|<br\s*\/?>)/g, '\n')
.replace(/&nbsp;/g, ' '); // nbsp isn't in XML standard (only HTML)
const stripped = utils.stripTags(newLines);
const escaped = escapeXmlAttribute(stripped);
return escaped.replace(/\n/g, '&#10;');
}
async function exportToOpml(branch, res) {
const note = await branch.getNote();
const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title;
const sanitizedTitle = sanitize(title);
async function exportNoteInner(branchId) {
const branch = await repository.getBranch(branchId);
const note = await branch.getNote();
const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title;
const preparedTitle = prepareText(title);
const preparedContent = prepareText(note.content);
res.write(`<outline title="${preparedTitle}" text="${preparedContent}">\n`);
for (const child of await note.getChildBranches()) {
await exportNoteInner(child.branchId);
}
res.write('</outline>');
}
res.setHeader('Content-Disposition', 'file; filename="' + sanitizedTitle + '.opml"');
res.setHeader('Content-Type', 'text/x-opml');
res.write(`<?xml version="1.0" encoding="UTF-8"?>
<opml version="1.0">
<head>
<title>Trilium export</title>
</head>
<body>`);
await exportNoteInner(branch.branchId);
res.write(`</body>
</opml>`);
res.end();
}
async function exportToTar(branch, res) {
const pack = tar.pack();
const exportedNoteIds = [];
const name = await exportNoteInner(branch, '');
async function exportNoteInner(branch, directory) {
const note = await branch.getNote();
const childFileName = directory + sanitize(note.title);
if (exportedNoteIds.includes(note.noteId)) {
saveMetadataFile(childFileName, {
version: 1,
clone: true,
noteId: note.noteId,
prefix: branch.prefix
});
return;
}
const metadata = {
version: 1,
clone: false,
noteId: note.noteId,
title: note.title,
prefix: branch.prefix,
isExpanded: branch.isExpanded,
type: note.type,
mime: note.mime,
// we don't export dateCreated and dateModified of any entity since that would be a bit misleading
attributes: (await note.getOwnedAttributes()).map(attribute => {
return {
type: attribute.type,
name: attribute.name,
value: attribute.value,
isInheritable: attribute.isInheritable,
position: attribute.position
};
}),
links: (await note.getLinks()).map(link => {
return {
type: link.type,
targetNoteId: link.targetNoteId
}
})
};
if (await note.hasLabel('excludeFromExport')) {
return;
}
saveMetadataFile(childFileName, metadata);
saveDataFile(childFileName, note);
exportedNoteIds.push(note.noteId);
const childBranches = await note.getChildBranches();
if (childBranches.length > 0) {
saveDirectory(childFileName);
}
for (const childBranch of childBranches) {
await exportNoteInner(childBranch, childFileName + "/");
}
return childFileName;
}
function saveDataFile(childFileName, note) {
const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content;
pack.entry({name: childFileName + ".dat", size: content.length}, content);
}
function saveMetadataFile(childFileName, metadata) {
const metadataJson = JSON.stringify(metadata, null, '\t');
pack.entry({name: childFileName + ".meta", size: metadataJson.length}, metadataJson);
}
function saveDirectory(childFileName) {
pack.entry({name: childFileName, type: 'directory'});
}
pack.finalize();
res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"');
res.setHeader('Content-Type', 'application/tar');
pack.pipe(res);
}
async function exportToMarkdown(branch, res) {
const note = await branch.getNote();
if (!await note.hasChildren()) {
await exportSingleMarkdown(note, res);
return;
}
const turndownService = new TurndownService();
const pack = tar.pack();
const name = await exportNoteInner(note, '');
async function exportNoteInner(note, directory) {
const childFileName = directory + sanitize(note.title);
if (await note.hasLabel('excludeFromExport')) {
return;
}
saveDataFile(childFileName, note);
const childNotes = await note.getChildNotes();
if (childNotes.length > 0) {
saveDirectory(childFileName);
}
for (const childNote of childNotes) {
await exportNoteInner(childNote, childFileName + "/");
}
return childFileName;
}
function saveDataFile(childFileName, note) {
if (note.type !== 'text' && note.type !== 'code') {
return;
}
if (note.content.trim().length === 0) {
return;
}
let markdown;
if (note.type === 'code') {
markdown = '```\n' + note.content + "\n```";
}
else if (note.type === 'text') {
markdown = turndownService.turndown(note.content);
}
else {
// other note types are not supported
return;
}
pack.entry({name: childFileName + ".md", size: markdown.length}, markdown);
}
function saveDirectory(childFileName) {
pack.entry({name: childFileName, type: 'directory'});
}
pack.finalize();
res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"');
res.setHeader('Content-Type', 'application/tar');
pack.pipe(res);
}
async function exportSingleMarkdown(note, res) {
const turndownService = new TurndownService();
const markdown = turndownService.turndown(note.content);
const name = sanitize(note.title);
res.setHeader('Content-Disposition', 'file; filename="' + name + '.md"');
res.setHeader('Content-Type', 'text/markdown; charset=UTF-8');
res.send(markdown);
}
module.exports = { module.exports = {
exportNote exportNote
}; };

339
src/routes/api/import.js
View File

@ -1,18 +1,11 @@
"use strict"; "use strict";
const Attribute = require('../../entities/attribute');
const Link = require('../../entities/link');
const repository = require('../../services/repository'); const repository = require('../../services/repository');
const log = require('../../services/log'); const enexImportService = require('../../services/import/enex');
const utils = require('../../services/utils'); const opmlImportService = require('../../services/import/opml');
const enex = require('../../services/import/enex'); const tarImportService = require('../../services/import/tar');
const noteService = require('../../services/notes'); const markdownImportService = require('../../services/import/markdown');
const Branch = require('../../entities/branch');
const tar = require('tar-stream');
const stream = require('stream');
const path = require('path'); const path = require('path');
const parseString = require('xml2js').parseString;
const commonmark = require('commonmark');
async function importToBranch(req) { async function importToBranch(req) {
const parentNoteId = req.params.parentNoteId; const parentNoteId = req.params.parentNoteId;
@ -31,338 +24,22 @@ async function importToBranch(req) {
const extension = path.extname(file.originalname).toLowerCase(); const extension = path.extname(file.originalname).toLowerCase();
if (extension === '.tar') { if (extension === '.tar') {
return await importTar(file, parentNote); return await tarImportService.importTar(file, parentNote);
} }
else if (extension === '.opml') { else if (extension === '.opml') {
return await importOpml(file, parentNote); return await opmlImportService.importOpml(file, parentNote);
} }
else if (extension === '.md') { else if (extension === '.md') {
return await importMarkdown(file, parentNote); return await markdownImportService.importMarkdown(file, parentNote);
} }
else if (extension === '.enex') { else if (extension === '.enex') {
return await enex.importEnex(file, parentNote); return await enexImportService.importEnex(file, parentNote);
} }
else { else {
return [400, `Unrecognized extension ${extension}, must be .tar or .opml`]; return [400, `Unrecognized extension ${extension}, must be .tar or .opml`];
} }
} }
function toHtml(text) {
if (!text) {
return '';
}
return '<p>' + text.replace(/(?:\r\n|\r|\n)/g, '</p><p>') + '</p>';
}
async function importOutline(outline, parentNoteId) {
const {note} = await noteService.createNote(parentNoteId, outline.$.title, toHtml(outline.$.text));
for (const childOutline of (outline.outline || [])) {
await importOutline(childOutline, note.noteId);
}
return note;
}
async function importOpml(file, parentNote) {
const xml = await new Promise(function(resolve, reject)
{
parseString(file.buffer, function (err, result) {
if (err) {
reject(err);
}
else {
resolve(result);
}
});
});
if (xml.opml.$.version !== '1.0' && xml.opml.$.version !== '1.1') {
return [400, 'Unsupported OPML version ' + xml.opml.$.version + ', 1.0 or 1.1 expected instead.'];
}
const outlines = xml.opml.body[0].outline || [];
let returnNote = null;
for (const outline of outlines) {
const note = await importOutline(outline, parentNote.noteId);
// first created note will be activated after import
returnNote = returnNote || note;
}
return returnNote;
}
/**
* Complication of this export is the need to balance two needs:
* -
*/
async function importTar(file, parentNote) {
const files = await parseImportFile(file);
const ctx = {
// maps from original noteId (in tar file) to newly generated noteId
noteIdMap: {},
// new noteIds of notes which were actually created (not just referenced)
createdNoteIds: [],
attributes: [],
links: [],
reader: new commonmark.Parser(),
writer: new commonmark.HtmlRenderer()
};
ctx.getNewNoteId = function(origNoteId) {
// in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution
if (!origNoteId.trim()) {
return "";
}
if (!ctx.noteIdMap[origNoteId]) {
ctx.noteIdMap[origNoteId] = utils.newEntityId();
}
return ctx.noteIdMap[origNoteId];
};
const note = await importNotes(ctx, files, parentNote.noteId);
// we save attributes and links after importing notes because we need to check that target noteIds
// have been really created (relation/links with targets outside of the export are not created)
for (const attr of ctx.attributes) {
if (attr.type === 'relation') {
attr.value = ctx.getNewNoteId(attr.value);
if (!ctx.createdNoteIds.includes(attr.value)) {
// relation targets note outside of the export
continue;
}
}
await new Attribute(attr).save();
}
for (const link of ctx.links) {
link.targetNoteId = ctx.getNewNoteId(link.targetNoteId);
if (!ctx.createdNoteIds.includes(link.targetNoteId)) {
// link targets note outside of the export
continue;
}
await new Link(link).save();
}
return note;
}
function getFileName(name) {
let key;
if (name.endsWith(".dat")) {
key = "data";
name = name.substr(0, name.length - 4);
}
else if (name.endsWith(".md")) {
key = "markdown";
name = name.substr(0, name.length - 3);
}
else if (name.endsWith((".meta"))) {
key = "meta";
name = name.substr(0, name.length - 5);
}
else {
log.error("Unknown file type in import: " + name);
}
return {name, key};
}
async function parseImportFile(file) {
const fileMap = {};
const files = [];
const extract = tar.extract();
extract.on('entry', function(header, stream, next) {
let name, key;
if (header.type === 'file') {
({name, key} = getFileName(header.name));
}
else if (header.type === 'directory') {
// directory entries in tar often end with directory separator
name = (header.name.endsWith("/") || header.name.endsWith("\\")) ? header.name.substr(0, header.name.length - 1) : header.name;
key = 'directory';
}
else {
log.error("Unrecognized tar entry: " + JSON.stringify(header));
return;
}
let file = fileMap[name];
if (!file) {
file = fileMap[name] = {
name: path.basename(name),
children: []
};
let parentFileName = path.dirname(header.name);
if (parentFileName && parentFileName !== '.') {
fileMap[parentFileName].children.push(file);
}
else {
files.push(file);
}
}
const chunks = [];
stream.on("data", function (chunk) {
chunks.push(chunk);
});
// header is the tar header
// stream is the content body (might be an empty stream)
// call next when you are done with this entry
stream.on('end', function() {
file[key] = Buffer.concat(chunks);
if (key === "meta") {
file[key] = JSON.parse(file[key].toString("UTF-8"));
}
next(); // ready for next entry
});
stream.resume(); // just auto drain the stream
});
return new Promise(resolve => {
extract.on('finish', function() {
resolve(files);
});
const bufferStream = new stream.PassThrough();
bufferStream.end(file.buffer);
bufferStream.pipe(extract);
});
}
async function importNotes(ctx, files, parentNoteId) {
let returnNote = null;
for (const file of files) {
let note;
if (!file.meta) {
let content = '';
if (file.data) {
content = file.data.toString("UTF-8");
}
else if (file.markdown) {
const parsed = ctx.reader.parse(file.markdown.toString("UTF-8"));
content = ctx.writer.render(parsed);
}
note = (await noteService.createNote(parentNoteId, file.name, content, {
type: 'text',
mime: 'text/html'
})).note;
}
else {
if (file.meta.version !== 1) {
throw new Error("Can't read meta data version " + file.meta.version);
}
if (file.meta.clone) {
await new Branch({
parentNoteId: parentNoteId,
noteId: ctx.getNewNoteId(file.meta.noteId),
prefix: file.meta.prefix,
isExpanded: !!file.meta.isExpanded
}).save();
return;
}
if (file.meta.type !== 'file' && file.meta.type !== 'image') {
file.data = file.data.toString("UTF-8");
// this will replace all internal links (<a> and <img>) inside the body
// links pointing outside the export will be broken and changed (ctx.getNewNoteId() will still assign new noteId)
for (const link of file.meta.links || []) {
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
file.data = file.data.replace(new RegExp(link.targetNoteId, "g"), ctx.getNewNoteId(link.targetNoteId));
}
}
note = (await noteService.createNote(parentNoteId, file.meta.title, file.data, {
noteId: ctx.getNewNoteId(file.meta.noteId),
type: file.meta.type,
mime: file.meta.mime,
prefix: file.meta.prefix
})).note;
ctx.createdNoteIds.push(note.noteId);
for (const attribute of file.meta.attributes || []) {
ctx.attributes.push({
noteId: note.noteId,
type: attribute.type,
name: attribute.name,
value: attribute.value,
isInheritable: attribute.isInheritable,
position: attribute.position
});
}
for (const link of file.meta.links || []) {
ctx.links.push({
noteId: note.noteId,
type: link.type,
targetNoteId: link.targetNoteId
});
}
}
// first created note will be activated after import
returnNote = returnNote || note;
if (file.children.length > 0) {
await importNotes(ctx, file.children, note.noteId);
}
}
return returnNote;
}
async function importMarkdown(file, parentNote) {
const markdownContent = file.buffer.toString("UTF-8");
const reader = new commonmark.Parser();
const writer = new commonmark.HtmlRenderer();
const parsed = reader.parse(markdownContent);
const htmlContent = writer.render(parsed);
const title = file.originalname.substr(0, file.originalname.length - 3); // strip .md extension
const {note} = await noteService.createNote(parentNote.noteId, title, htmlContent, {
type: 'text',
mime: 'text/html'
});
return note;
}
module.exports = { module.exports = {
importToBranch importToBranch
}; };

19
src/services/export/markdown_single.js Normal file
View File

@ -0,0 +1,19 @@
"use strict";
const sanitize = require("sanitize-filename");
const TurndownService = require('turndown');
async function exportSingleMarkdown(note, res) {
const turndownService = new TurndownService();
const markdown = turndownService.turndown(note.content);
const name = sanitize(note.title);
res.setHeader('Content-Disposition', 'file; filename="' + name + '.md"');
res.setHeader('Content-Type', 'text/markdown; charset=UTF-8');
res.send(markdown);
}
module.exports = {
exportSingleMarkdown
};

82
src/services/export/markdown_tar.js Normal file
View File

@ -0,0 +1,82 @@
"use strict";
const tar = require('tar-stream');
const TurndownService = require('turndown');
const sanitize = require("sanitize-filename");
const markdownSingleExportService = require('../../services/export/markdown_single');
async function exportToMarkdown(branch, res) {
const note = await branch.getNote();
if (!await note.hasChildren()) {
await markdownSingleExportService.exportSingleMarkdown(note, res);
return;
}
const turndownService = new TurndownService();
const pack = tar.pack();
const name = await exportNoteInner(note, '');
async function exportNoteInner(note, directory) {
const childFileName = directory + sanitize(note.title);
if (await note.hasLabel('excludeFromExport')) {
return;
}
saveDataFile(childFileName, note);
const childNotes = await note.getChildNotes();
if (childNotes.length > 0) {
saveDirectory(childFileName);
}
for (const childNote of childNotes) {
await exportNoteInner(childNote, childFileName + "/");
}
return childFileName;
}
function saveDataFile(childFileName, note) {
if (note.type !== 'text' && note.type !== 'code') {
return;
}
if (note.content.trim().length === 0) {
return;
}
let markdown;
if (note.type === 'code') {
markdown = '```\n' + note.content + "\n```";
}
else if (note.type === 'text') {
markdown = turndownService.turndown(note.content);
}
else {
// other note types are not supported
return;
}
pack.entry({name: childFileName + ".md", size: markdown.length}, markdown);
}
function saveDirectory(childFileName) {
pack.entry({name: childFileName, type: 'directory'});
}
pack.finalize();
res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"');
res.setHeader('Content-Type', 'application/tar');
pack.pipe(res);
}
module.exports = {
exportToMarkdown
};

103
src/services/export/native_tar.js Normal file
View File

@ -0,0 +1,103 @@
"use strict";
const html = require('html');
const native_tar = require('tar-stream');
const sanitize = require("sanitize-filename");
async function exportToTar(branch, res) {
const pack = native_tar.pack();
const exportedNoteIds = [];
const name = await exportNoteInner(branch, '');
async function exportNoteInner(branch, directory) {
const note = await branch.getNote();
const childFileName = directory + sanitize(note.title);
if (exportedNoteIds.includes(note.noteId)) {
saveMetadataFile(childFileName, {
version: 1,
clone: true,
noteId: note.noteId,
prefix: branch.prefix
});
return;
}
const metadata = {
version: 1,
clone: false,
noteId: note.noteId,
title: note.title,
prefix: branch.prefix,
isExpanded: branch.isExpanded,
type: note.type,
mime: note.mime,
// we don't export dateCreated and dateModified of any entity since that would be a bit misleading
attributes: (await note.getOwnedAttributes()).map(attribute => {
return {
type: attribute.type,
name: attribute.name,
value: attribute.value,
isInheritable: attribute.isInheritable,
position: attribute.position
};
}),
links: (await note.getLinks()).map(link => {
return {
type: link.type,
targetNoteId: link.targetNoteId
}
})
};
if (await note.hasLabel('excludeFromExport')) {
return;
}
saveMetadataFile(childFileName, metadata);
saveDataFile(childFileName, note);
exportedNoteIds.push(note.noteId);
const childBranches = await note.getChildBranches();
if (childBranches.length > 0) {
saveDirectory(childFileName);
}
for (const childBranch of childBranches) {
await exportNoteInner(childBranch, childFileName + "/");
}
return childFileName;
}
function saveDataFile(childFileName, note) {
const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content;
pack.entry({name: childFileName + ".dat", size: content.length}, content);
}
function saveMetadataFile(childFileName, metadata) {
const metadataJson = JSON.stringify(metadata, null, '\t');
pack.entry({name: childFileName + ".meta", size: metadataJson.length}, metadataJson);
}
function saveDirectory(childFileName) {
pack.entry({name: childFileName, type: 'directory'});
}
pack.finalize();
res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"');
res.setHeader('Content-Type', 'application/tar');
pack.pipe(res);
}
module.exports = {
exportToTar
};

67
src/services/export/opml.js Normal file
View File

@ -0,0 +1,67 @@
"use strict";
const sanitize = require("sanitize-filename");
const repository = require("../../services/repository");
const utils = require('../../services/utils');
async function exportToOpml(branch, res) {
const note = await branch.getNote();
const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title;
const sanitizedTitle = sanitize(title);
async function exportNoteInner(branchId) {
const branch = await repository.getBranch(branchId);
const note = await branch.getNote();
const title = (branch.prefix ? (branch.prefix + ' - ') : '') + note.title;
const preparedTitle = prepareText(title);
const preparedContent = prepareText(note.content);
res.write(`<outline title="${preparedTitle}" text="${preparedContent}">\n`);
for (const child of await note.getChildBranches()) {
await exportNoteInner(child.branchId);
}
res.write('</outline>');
}
res.setHeader('Content-Disposition', 'file; filename="' + sanitizedTitle + '.opml"');
res.setHeader('Content-Type', 'text/x-opml');
res.write(`<?xml version="1.0" encoding="UTF-8"?>
<opml version="1.0">
<head>
<title>Trilium export</title>
</head>
<body>`);
await exportNoteInner(branch.branchId);
res.write(`</body>
</opml>`);
res.end();
}
function prepareText(text) {
const newLines = text.replace(/(<p[^>]*>|<br\s*\/?>)/g, '\n')
.replace(/&nbsp;/g, ' '); // nbsp isn't in XML standard (only HTML)
const stripped = utils.stripTags(newLines);
const escaped = escapeXmlAttribute(stripped);
return escaped.replace(/\n/g, '&#10;');
}
function escapeXmlAttribute(text) {
return text.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;');
}
module.exports = {
exportToOpml
};

30
src/services/import/markdown.js Normal file
View File

@ -0,0 +1,30 @@
"use strict";
// note that this is for import of single markdown file only - for archive/structure of markdown files
// see tar export/import
const noteService = require('../../services/notes');
const commonmark = require('commonmark');
async function importMarkdown(file, parentNote) {
const markdownContent = file.buffer.toString("UTF-8");
const reader = new commonmark.Parser();
const writer = new commonmark.HtmlRenderer();
const parsed = reader.parse(markdownContent);
const htmlContent = writer.render(parsed);
const title = file.originalname.substr(0, file.originalname.length - 3); // strip .md extension
const {note} = await noteService.createNote(parentNote.noteId, title, htmlContent, {
type: 'text',
mime: 'text/html'
});
return note;
}
module.exports = {
importMarkdown
};

56
src/services/import/opml.js Normal file
View File

@ -0,0 +1,56 @@
"use strict";
const noteService = require('../../services/notes');
const parseString = require('xml2js').parseString;
async function importOpml(file, parentNote) {
const xml = await new Promise(function(resolve, reject)
{
parseString(file.buffer, function (err, result) {
if (err) {
reject(err);
}
else {
resolve(result);
}
});
});
if (xml.opml.$.version !== '1.0' && xml.opml.$.version !== '1.1') {
return [400, 'Unsupported OPML version ' + xml.opml.$.version + ', 1.0 or 1.1 expected instead.'];
}
const outlines = xml.opml.body[0].outline || [];
let returnNote = null;
for (const outline of outlines) {
const note = await importOutline(outline, parentNote.noteId);
// first created note will be activated after import
returnNote = returnNote || note;
}
return returnNote;
}
function toHtml(text) {
if (!text) {
return '';
}
return '<p>' + text.replace(/(?:\r\n|\r|\n)/g, '</p><p>') + '</p>';
}
async function importOutline(outline, parentNoteId) {
const {note} = await noteService.createNote(parentNoteId, outline.$.title, toHtml(outline.$.text));
for (const childOutline of (outline.outline || [])) {
await importOutline(childOutline, note.noteId);
}
return note;
}
module.exports = {
importOpml
};

265
src/services/import/tar.js Normal file
View File

@ -0,0 +1,265 @@
"use strict";
const Attribute = require('../../entities/attribute');
const Link = require('../../entities/link');
const log = require('../../services/log');
const utils = require('../../services/utils');
const noteService = require('../../services/notes');
const Branch = require('../../entities/branch');
const tar = require('tar-stream');
const stream = require('stream');
const path = require('path');
const commonmark = require('commonmark');
/**
* Complication of this export is the need to balance two needs:
* -
*/
async function importTar(file, parentNote) {
const files = await parseImportFile(file);
const ctx = {
// maps from original noteId (in tar file) to newly generated noteId
noteIdMap: {},
// new noteIds of notes which were actually created (not just referenced)
createdNoteIds: [],
attributes: [],
links: [],
reader: new commonmark.Parser(),
writer: new commonmark.HtmlRenderer()
};
ctx.getNewNoteId = function(origNoteId) {
// in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution
if (!origNoteId.trim()) {
return "";
}
if (!ctx.noteIdMap[origNoteId]) {
ctx.noteIdMap[origNoteId] = utils.newEntityId();
}
return ctx.noteIdMap[origNoteId];
};
const note = await importNotes(ctx, files, parentNote.noteId);
// we save attributes and links after importing notes because we need to check that target noteIds
// have been really created (relation/links with targets outside of the export are not created)
for (const attr of ctx.attributes) {
if (attr.type === 'relation') {
attr.value = ctx.getNewNoteId(attr.value);
if (!ctx.createdNoteIds.includes(attr.value)) {
// relation targets note outside of the export
continue;
}
}
await new Attribute(attr).save();
}
for (const link of ctx.links) {
link.targetNoteId = ctx.getNewNoteId(link.targetNoteId);
if (!ctx.createdNoteIds.includes(link.targetNoteId)) {
// link targets note outside of the export
continue;
}
await new Link(link).save();
}
return note;
}
function getFileName(name) {
let key;
if (name.endsWith(".dat")) {
key = "data";
name = name.substr(0, name.length - 4);
}
else if (name.endsWith(".md")) {
key = "markdown";
name = name.substr(0, name.length - 3);
}
else if (name.endsWith((".meta"))) {
key = "meta";
name = name.substr(0, name.length - 5);
}
else {
log.error("Unknown file type in import: " + name);
}
return {name, key};
}
async function parseImportFile(file) {
const fileMap = {};
const files = [];
const extract = tar.extract();
extract.on('entry', function(header, stream, next) {
let name, key;
if (header.type === 'file') {
({name, key} = getFileName(header.name));
}
else if (header.type === 'directory') {
// directory entries in tar often end with directory separator
name = (header.name.endsWith("/") || header.name.endsWith("\\")) ? header.name.substr(0, header.name.length - 1) : header.name;
key = 'directory';
}
else {
log.error("Unrecognized tar entry: " + JSON.stringify(header));
return;
}
let file = fileMap[name];
if (!file) {
file = fileMap[name] = {
name: path.basename(name),
children: []
};
let parentFileName = path.dirname(header.name);
if (parentFileName && parentFileName !== '.') {
fileMap[parentFileName].children.push(file);
}
else {
files.push(file);
}
}
const chunks = [];
stream.on("data", function (chunk) {
chunks.push(chunk);
});
// header is the tar header
// stream is the content body (might be an empty stream)
// call next when you are done with this entry
stream.on('end', function() {
file[key] = Buffer.concat(chunks);
if (key === "meta") {
file[key] = JSON.parse(file[key].toString("UTF-8"));
}
next(); // ready for next entry
});
stream.resume(); // just auto drain the stream
});
return new Promise(resolve => {
extract.on('finish', function() {
resolve(files);
});
const bufferStream = new stream.PassThrough();
bufferStream.end(file.buffer);
bufferStream.pipe(extract);
});
}
async function importNotes(ctx, files, parentNoteId) {
let returnNote = null;
for (const file of files) {
let note;
if (!file.meta) {
let content = '';
if (file.data) {
content = file.data.toString("UTF-8");
}
else if (file.markdown) {
const parsed = ctx.reader.parse(file.markdown.toString("UTF-8"));
content = ctx.writer.render(parsed);
}
note = (await noteService.createNote(parentNoteId, file.name, content, {
type: 'text',
mime: 'text/html'
})).note;
}
else {
if (file.meta.version !== 1) {
throw new Error("Can't read meta data version " + file.meta.version);
}
if (file.meta.clone) {
await new Branch({
parentNoteId: parentNoteId,
noteId: ctx.getNewNoteId(file.meta.noteId),
prefix: file.meta.prefix,
isExpanded: !!file.meta.isExpanded
}).save();
return;
}
if (file.meta.type !== 'file' && file.meta.type !== 'image') {
file.data = file.data.toString("UTF-8");
// this will replace all internal links (<a> and <img>) inside the body
// links pointing outside the export will be broken and changed (ctx.getNewNoteId() will still assign new noteId)
for (const link of file.meta.links || []) {
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
file.data = file.data.replace(new RegExp(link.targetNoteId, "g"), ctx.getNewNoteId(link.targetNoteId));
}
}
note = (await noteService.createNote(parentNoteId, file.meta.title, file.data, {
noteId: ctx.getNewNoteId(file.meta.noteId),
type: file.meta.type,
mime: file.meta.mime,
prefix: file.meta.prefix
})).note;
ctx.createdNoteIds.push(note.noteId);
for (const attribute of file.meta.attributes || []) {
ctx.attributes.push({
noteId: note.noteId,
type: attribute.type,
name: attribute.name,
value: attribute.value,
isInheritable: attribute.isInheritable,
position: attribute.position
});
}
for (const link of file.meta.links || []) {
ctx.links.push({
noteId: note.noteId,
type: link.type,
targetNoteId: link.targetNoteId
});
}
}
// first created note will be activated after import
returnNote = returnNote || note;
if (file.children.length > 0) {
await importNotes(ctx, file.children, note.noteId);
}
}
return returnNote;
}
module.exports = {
importTar
};

4
src/views/dialogs/export_subtree.ejs
View File

@ -14,7 +14,7 @@
<br/> <br/>
<div class="form-check"> <div class="form-check">
<input class="form-check-input" type="radio" name="export-format" id="export-format-tar" value="tar" checked> <input class="form-check-input" type="radio" name="export-format" id="export-format-tar" value="native-tar" checked>
<label class="form-check-label" for="export-format-tar">Native TAR - this is Trilium's native format which preserves all notes' data & metadata.</label> <label class="form-check-label" for="export-format-tar">Native TAR - this is Trilium's native format which preserves all notes' data & metadata.</label>
</div> </div>
@ -31,7 +31,7 @@
<div class="form-check disabled"> <div class="form-check disabled">
<input class="form-check-input" type="radio" name="export-format" id="export-format-markdown" <input class="form-check-input" type="radio" name="export-format" id="export-format-markdown"
value="markdown"> value="markdown-tar">
<label class="form-check-label" for="export-format-markdown"> <label class="form-check-label" for="export-format-markdown">
Markdown - TAR archive of Markdown formatted notes Markdown - TAR archive of Markdown formatted notes
</label> </label>