gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

added ImportContext

Browse Source
This commit is contained in:
zadam 2019-02-10 19:36:03 +01:00
parent 5baa251944
commit e4c78f3887
8 changed files with 118 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/public/javascripts/dialogs/import.js
View File

@ -1,4 +1,5 @@
import treeService from '../services/tree.js';
import utils from '../services/utils.js';
import treeUtils from "../services/tree_utils.js";
import server from "../services/server.js";
import infoService from "../services/info.js";
@ -12,7 +13,11 @@ const $importNoteCountWrapper = $("#import-note-count-wrapper");
const $importNoteCount = $("#import-note-count");
const $importButton = $("#import-button");
let importId;
async function showDialog() {
// each opening of the dialog resets the importId so we don't associate it with previous imports anymore
importId = '';
$importNoteCountWrapper.hide();
$importNoteCount.text('0');
$fileUploadInput.val('').change(); // to trigger Import button disabling listener below
@ -40,8 +45,12 @@ function importIntoNote(importNoteId) {
const formData = new FormData();
formData.append('upload', $fileUploadInput[0].files[0]);
// we generate it here (and not on opening) for the case when you try to import multiple times from the same
// dialog (which shouldn't happen, but still ...)
importId = utils.randomString(10);
$.ajax({
url: baseApiUrl + 'notes/' + importNoteId + '/import',
url: baseApiUrl + 'notes/' + importNoteId + '/import/' + importId,
headers: server.getHeaders(),
data: formData,
dataType: 'json',
@ -54,6 +63,11 @@ function importIntoNote(importNoteId) {
}
messagingService.subscribeToMessages(async message => {
if (!message.importId || message.importId !== importId) {
// incoming messages must correspond to this import instance
return;
}
if (message.type === 'import-note-count') {
$importNoteCountWrapper.show();

47
src/routes/api/import.js
View File

@ -5,12 +5,45 @@ const enexImportService = require('../../services/import/enex');
const opmlImportService = require('../../services/import/opml');
const tarImportService = require('../../services/import/tar');
const singleImportService = require('../../services/import/single');
const messagingService = require('../../services/messaging');
const cls = require('../../services/cls');
const path = require('path');
const noteCacheService = require('../../services/note_cache');
class ImportContext {
constructor(importId) {
// importId is to distinguish between different import events - it is possible (though not recommended)
// to have multiple imports going at the same time
this.importId = importId;
this.count = 0;
this.lastSentCountTs = Date.now();
}
async increaseCount() {
this.count++;
if (Date.now() - this.lastSentCountTs >= 1000) {
this.lastSentCountTs = Date.now();
await messagingService.sendMessageToAllClients({
importId: this.importId,
type: 'import-note-count',
count: this.count
});
}
}
async importFinished(noteId) {
await messagingService.sendMessageToAllClients({
importId: this.importId,
type: 'import-finished',
noteId: noteId
});
}
}
async function importToBranch(req) {
const parentNoteId = req.params.parentNoteId;
const {parentNoteId, importId} = req.params;
const file = req.file;
if (!file) {
@ -31,20 +64,22 @@ async function importToBranch(req) {
let note; // typically root of the import - client can show it after finishing the import
const importContext = new ImportContext(importId);
if (extension === '.tar') {
note = await tarImportService.importTar(file.buffer, parentNote);
note = await tarImportService.importTar(importContext, file.buffer, parentNote);
}
else if (extension === '.opml') {
note = await opmlImportService.importOpml(file.buffer, parentNote);
note = await opmlImportService.importOpml(importContext, file.buffer, parentNote);
}
else if (extension === '.md') {
note = await singleImportService.importMarkdown(file, parentNote);
note = await singleImportService.importMarkdown(importContext, file, parentNote);
}
else if (extension === '.html' || extension === '.htm') {
note = await singleImportService.importHtml(file, parentNote);
note = await singleImportService.importHtml(importContext, file, parentNote);
}
else if (extension === '.enex') {
note = await enexImportService.importEnex(file, parentNote);
note = await enexImportService.importEnex(importContext, file, parentNote);
}
else {
return [400, `Unrecognized extension ${extension}, must be .tar or .opml`];

2
src/routes/routes.js
View File

@ -129,7 +129,7 @@ function register(app) {
apiRoute(PUT, '/api/notes/:noteId/clone-after/:afterBranchId', cloningApiRoute.cloneNoteAfter);
route(GET, '/api/notes/:branchId/export/:type/:format', [auth.checkApiAuthOrElectron], exportRoute.exportBranch);
route(POST, '/api/notes/:parentNoteId/import', [auth.checkApiAuthOrElectron, uploadMiddleware], importRoute.importToBranch, apiResultHandler);
route(POST, '/api/notes/:parentNoteId/import/:importId', [auth.checkApiAuthOrElectron, uploadMiddleware], importRoute.importToBranch, apiResultHandler);
route(POST, '/api/notes/:parentNoteId/upload', [auth.checkApiAuthOrElectron, uploadMiddleware],
filesRoute.uploadFile, apiResultHandler);

56
src/services/import/enex.js
View File

@ -19,7 +19,7 @@ function parseDate(text) {
let note = {};
let resource;
async function importEnex(file, parentNote) {
async function importEnex(importContext, file, parentNote) {
const saxStream = sax.createStream(true);
const xmlBuilder = new xml2js.Builder({ headless: true });
const parser = new xml2js.Parser({ explicitArray: true });
@ -218,6 +218,8 @@ async function importEnex(file, parentNote) {
mime: 'text/html'
})).note;
importContext.increaseCount();
const noteContent = await noteEntity.getNoteContent();
for (const resource of resources) {
@ -232,39 +234,40 @@ async function importEnex(file, parentNote) {
}
const createResourceNote = async () => {
const resourceNote = (await noteService.createNote(noteEntity.noteId, resource.title, resource.content, {
attributes: resource.attributes,
type: 'file',
mime: resource.mime
})).note;
const resourceNote = (await noteService.createNote(noteEntity.noteId, resource.title, resource.content, {
attributes: resource.attributes,
type: 'file',
mime: resource.mime
})).note;
const resourceLink = `<a href="#root/${resourceNote.noteId}">${utils.escapeHtml(resource.title)}</a>`;
importContext.increaseCount();
noteContent.content = noteContent.content.replace(mediaRegex, resourceLink);
const resourceLink = `<a href="#root/${resourceNote.noteId}">${utils.escapeHtml(resource.title)}</a>`;
noteContent.content = noteContent.content.replace(mediaRegex, resourceLink);
};
if (["image/jpeg", "image/png", "image/gif"].includes(resource.mime)) {
try {
const originalName = "image." + resource.mime.substr(6);
try {
const originalName = "image." + resource.mime.substr(6);
const { url } = await imageService.saveImage(resource.content, originalName, noteEntity.noteId);
const {url} = await imageService.saveImage(resource.content, originalName, noteEntity.noteId);
const imageLink = `<img src="${url}">`;
const imageLink = `<img src="${url}">`;
noteContent.content = noteContent.content.replace(mediaRegex, imageLink);
noteContent.content = noteContent.content.replace(mediaRegex, imageLink);
if (!noteContent.content.includes(imageLink)) {
// if there wasn't any match for the reference, we'll add the image anyway
// otherwise image would be removed since no note would include it
noteContent.content += imageLink;
if (!noteContent.content.includes(imageLink)) {
// if there wasn't any match for the reference, we'll add the image anyway
// otherwise image would be removed since no note would include it
noteContent.content += imageLink;
}
} catch (e) {
log.error("error when saving image from ENEX file: " + e);
await createResourceNote();
}
} catch (e) {
log.error("error when saving image from ENEX file: " + e);
} else {
await createResourceNote();
}
}
else {
await createResourceNote();
}
}
@ -295,7 +298,12 @@ async function importEnex(file, parentNote) {
return new Promise((resolve, reject) =>
{
// resolve only when we parse the whole document AND saving of all notes have been finished
saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) });
saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => {
importContext.importFinished(rootNote.noteId);
resolve(rootNote);
});
});
const bufferStream = new stream.PassThrough();
bufferStream.end(file.buffer);

14
src/services/import/opml.js
View File

@ -3,7 +3,13 @@
const noteService = require('../../services/notes');
const parseString = require('xml2js').parseString;
async function importOpml(fileBuffer, parentNote) {
/**
* @param {ImportContext} importContext
* @param {Buffer} fileBuffer
* @param {Note} parentNote
* @return {Promise<*[]|*>}
*/
async function importOpml(importContext, fileBuffer, parentNote) {
const xml = await new Promise(function(resolve, reject)
{
parseString(fileBuffer, function (err, result) {
@ -24,7 +30,7 @@ async function importOpml(fileBuffer, parentNote) {
let returnNote = null;
for (const outline of outlines) {
const note = await importOutline(outline, parentNote.noteId);
const note = await importOutline(importContext, outline, parentNote.noteId);
// first created note will be activated after import
returnNote = returnNote || note;
@ -41,9 +47,11 @@ function toHtml(text) {
return '<p>' + text.replace(/(?:\r\n|\r|\n)/g, '</p><p>') + '</p>';
}
async function importOutline(outline, parentNoteId) {
async function importOutline(importContext, outline, parentNoteId) {
const {note} = await noteService.createNote(parentNoteId, outline.$.title, toHtml(outline.$.text));
importContext.increaseCount();
for (const childOutline of (outline.outline || [])) {
await importOutline(childOutline, note.noteId);
}

10
src/services/import/single.js
View File

@ -4,7 +4,7 @@ const noteService = require('../../services/notes');
const commonmark = require('commonmark');
const path = require('path');
async function importMarkdown(file, parentNote) {
async function importMarkdown(importContext, file, parentNote) {
const markdownContent = file.buffer.toString("UTF-8");
const reader = new commonmark.Parser();
@ -20,10 +20,13 @@ async function importMarkdown(file, parentNote) {
mime: 'text/html'
});
importContext.increaseCount();
importContext.importFinished(note.noteId);
return note;
}
async function importHtml(file, parentNote) {
async function importHtml(importContext, file, parentNote) {
const title = getFileNameWithoutExtension(file.originalname);
const content = file.buffer.toString("UTF-8");
@ -32,6 +35,9 @@ async function importHtml(file, parentNote) {
mime: 'text/html'
});
importContext.increaseCount();
importContext.importFinished(note.noteId);
return note;
}

19
src/services/import/tar.js
View File

@ -6,7 +6,6 @@ const utils = require('../../services/utils');
const log = require('../../services/log');
const repository = require('../../services/repository');
const noteService = require('../../services/notes');
const messagingService = require('../../services/messaging');
const Branch = require('../../entities/branch');
const tar = require('tar-stream');
const stream = require('stream');
@ -17,7 +16,13 @@ const mimeTypes = require('mime-types');
let importNoteCount;
let lastSentCountTs = Date.now();
async function importTar(fileBuffer, importRootNote) {
/**
* @param {ImportContext} importContext
* @param {Buffer} fileBuffer
* @param {Note} importRootNote
* @return {Promise<*>}
*/
async function importTar(importContext, fileBuffer, importRootNote) {
importNoteCount = 0;
// maps from original noteId (in tar file) to newly generated noteId
@ -337,13 +342,7 @@ async function importTar(fileBuffer, importRootNote) {
log.info("Ignoring tar import entry with type " + header.type);
}
importNoteCount++;
if (Date.now() - lastSentCountTs >= 1000) {
lastSentCountTs = Date.now();
messagingService.importNoteCount(importNoteCount);
}
importContext.increaseCount();
next(); // ready for next entry
});
@ -379,7 +378,7 @@ async function importTar(fileBuffer, importRootNote) {
}
}
messagingService.importFinished(firstNote);
importContext.importFinished();
resolve(firstNote);
});

12
src/services/messaging.js
View File

@ -78,18 +78,8 @@ async function refreshTree() {
await sendMessageToAllClients({ type: 'refresh-tree' });
}
async function importNoteCount(count) {
await sendMessageToAllClients({ type: 'import-note-count', count: count });
}
async function importFinished(firstNote) {
await sendMessageToAllClients({ type: 'import-finished', noteId: firstNote.noteId });
}
module.exports = {
init,
sendMessageToAllClients,
refreshTree,
importNoteCount,
importFinished
refreshTree
};