diff --git a/src/public/javascripts/dialogs/import.js b/src/public/javascripts/dialogs/import.js
index 27b9629ae..da2dc79f0 100644
--- a/src/public/javascripts/dialogs/import.js
+++ b/src/public/javascripts/dialogs/import.js
@@ -1,4 +1,5 @@
import treeService from '../services/tree.js';
+import utils from '../services/utils.js';
import treeUtils from "../services/tree_utils.js";
import server from "../services/server.js";
import infoService from "../services/info.js";
@@ -12,7 +13,11 @@ const $importNoteCountWrapper = $("#import-note-count-wrapper");
const $importNoteCount = $("#import-note-count");
const $importButton = $("#import-button");
+let importId;
+
async function showDialog() {
+ // each opening of the dialog resets the importId so we don't associate it with previous imports anymore
+ importId = '';
$importNoteCountWrapper.hide();
$importNoteCount.text('0');
$fileUploadInput.val('').change(); // to trigger Import button disabling listener below
@@ -40,8 +45,12 @@ function importIntoNote(importNoteId) {
const formData = new FormData();
formData.append('upload', $fileUploadInput[0].files[0]);
+ // we generate it here (and not on opening) for the case when you try to import multiple times from the same
+ // dialog (which shouldn't happen, but still ...)
+ importId = utils.randomString(10);
+
$.ajax({
- url: baseApiUrl + 'notes/' + importNoteId + '/import',
+ url: baseApiUrl + 'notes/' + importNoteId + '/import/' + importId,
headers: server.getHeaders(),
data: formData,
dataType: 'json',
@@ -54,6 +63,11 @@ function importIntoNote(importNoteId) {
}
messagingService.subscribeToMessages(async message => {
+ if (!message.importId || message.importId !== importId) {
+ // incoming messages must correspond to this import instance
+ return;
+ }
+
if (message.type === 'import-note-count') {
$importNoteCountWrapper.show();
diff --git a/src/routes/api/import.js b/src/routes/api/import.js
index a73e62550..556bb35ec 100644
--- a/src/routes/api/import.js
+++ b/src/routes/api/import.js
@@ -5,12 +5,45 @@ const enexImportService = require('../../services/import/enex');
const opmlImportService = require('../../services/import/opml');
const tarImportService = require('../../services/import/tar');
const singleImportService = require('../../services/import/single');
+const messagingService = require('../../services/messaging');
const cls = require('../../services/cls');
const path = require('path');
const noteCacheService = require('../../services/note_cache');
+class ImportContext {
+ constructor(importId) {
+ // importId is to distinguish between different import events - it is possible (though not recommended)
+ // to have multiple imports going at the same time
+ this.importId = importId;
+ this.count = 0;
+ this.lastSentCountTs = Date.now();
+ }
+
+ async increaseCount() {
+ this.count++;
+
+ if (Date.now() - this.lastSentCountTs >= 1000) {
+ this.lastSentCountTs = Date.now();
+
+ await messagingService.sendMessageToAllClients({
+ importId: this.importId,
+ type: 'import-note-count',
+ count: this.count
+ });
+ }
+ }
+
+ async importFinished(noteId) {
+ await messagingService.sendMessageToAllClients({
+ importId: this.importId,
+ type: 'import-finished',
+ noteId: noteId
+ });
+ }
+}
+
async function importToBranch(req) {
- const parentNoteId = req.params.parentNoteId;
+ const {parentNoteId, importId} = req.params;
const file = req.file;
if (!file) {
@@ -31,20 +64,22 @@ async function importToBranch(req) {
let note; // typically root of the import - client can show it after finishing the import
+ const importContext = new ImportContext(importId);
+
if (extension === '.tar') {
- note = await tarImportService.importTar(file.buffer, parentNote);
+ note = await tarImportService.importTar(importContext, file.buffer, parentNote);
}
else if (extension === '.opml') {
- note = await opmlImportService.importOpml(file.buffer, parentNote);
+ note = await opmlImportService.importOpml(importContext, file.buffer, parentNote);
}
else if (extension === '.md') {
- note = await singleImportService.importMarkdown(file, parentNote);
+ note = await singleImportService.importMarkdown(importContext, file, parentNote);
}
else if (extension === '.html' || extension === '.htm') {
- note = await singleImportService.importHtml(file, parentNote);
+ note = await singleImportService.importHtml(importContext, file, parentNote);
}
else if (extension === '.enex') {
- note = await enexImportService.importEnex(file, parentNote);
+ note = await enexImportService.importEnex(importContext, file, parentNote);
}
else {
return [400, `Unrecognized extension ${extension}, must be .tar or .opml`];
diff --git a/src/routes/routes.js b/src/routes/routes.js
index 13054f331..1dc887eca 100644
--- a/src/routes/routes.js
+++ b/src/routes/routes.js
@@ -129,7 +129,7 @@ function register(app) {
apiRoute(PUT, '/api/notes/:noteId/clone-after/:afterBranchId', cloningApiRoute.cloneNoteAfter);
route(GET, '/api/notes/:branchId/export/:type/:format', [auth.checkApiAuthOrElectron], exportRoute.exportBranch);
- route(POST, '/api/notes/:parentNoteId/import', [auth.checkApiAuthOrElectron, uploadMiddleware], importRoute.importToBranch, apiResultHandler);
+ route(POST, '/api/notes/:parentNoteId/import/:importId', [auth.checkApiAuthOrElectron, uploadMiddleware], importRoute.importToBranch, apiResultHandler);
route(POST, '/api/notes/:parentNoteId/upload', [auth.checkApiAuthOrElectron, uploadMiddleware],
filesRoute.uploadFile, apiResultHandler);
diff --git a/src/services/import/enex.js b/src/services/import/enex.js
index 5793a1d05..0caa89b7f 100644
--- a/src/services/import/enex.js
+++ b/src/services/import/enex.js
@@ -19,7 +19,7 @@ function parseDate(text) {
let note = {};
let resource;
-async function importEnex(file, parentNote) {
+async function importEnex(importContext, file, parentNote) {
const saxStream = sax.createStream(true);
const xmlBuilder = new xml2js.Builder({ headless: true });
const parser = new xml2js.Parser({ explicitArray: true });
@@ -218,6 +218,8 @@ async function importEnex(file, parentNote) {
mime: 'text/html'
})).note;
+ importContext.increaseCount();
+
const noteContent = await noteEntity.getNoteContent();
for (const resource of resources) {
@@ -232,39 +234,40 @@ async function importEnex(file, parentNote) {
}
const createResourceNote = async () => {
- const resourceNote = (await noteService.createNote(noteEntity.noteId, resource.title, resource.content, {
- attributes: resource.attributes,
- type: 'file',
- mime: resource.mime
- })).note;
+ const resourceNote = (await noteService.createNote(noteEntity.noteId, resource.title, resource.content, {
+ attributes: resource.attributes,
+ type: 'file',
+ mime: resource.mime
+ })).note;
- const resourceLink = `${utils.escapeHtml(resource.title)}`;
+ importContext.increaseCount();
- noteContent.content = noteContent.content.replace(mediaRegex, resourceLink);
+ const resourceLink = `${utils.escapeHtml(resource.title)}`;
+
+ noteContent.content = noteContent.content.replace(mediaRegex, resourceLink);
};
if (["image/jpeg", "image/png", "image/gif"].includes(resource.mime)) {
- try {
- const originalName = "image." + resource.mime.substr(6);
+ try {
+ const originalName = "image." + resource.mime.substr(6);
- const { url } = await imageService.saveImage(resource.content, originalName, noteEntity.noteId);
+ const {url} = await imageService.saveImage(resource.content, originalName, noteEntity.noteId);
- const imageLink = ``;
+ const imageLink = `
`;
- noteContent.content = noteContent.content.replace(mediaRegex, imageLink);
+ noteContent.content = noteContent.content.replace(mediaRegex, imageLink);
- if (!noteContent.content.includes(imageLink)) {
- // if there wasn't any match for the reference, we'll add the image anyway
- // otherwise image would be removed since no note would include it
- noteContent.content += imageLink;
+ if (!noteContent.content.includes(imageLink)) {
+ // if there wasn't any match for the reference, we'll add the image anyway
+ // otherwise image would be removed since no note would include it
+ noteContent.content += imageLink;
+ }
+ } catch (e) {
+ log.error("error when saving image from ENEX file: " + e);
+ await createResourceNote();
}
- } catch (e) {
- log.error("error when saving image from ENEX file: " + e);
+ } else {
await createResourceNote();
- }
- }
- else {
- await createResourceNote();
}
}
@@ -295,7 +298,12 @@ async function importEnex(file, parentNote) {
return new Promise((resolve, reject) =>
{
// resolve only when we parse the whole document AND saving of all notes have been finished
- saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) });
+ saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => {
+ importContext.importFinished(rootNote.noteId);
+
+ resolve(rootNote);
+ });
+ });
const bufferStream = new stream.PassThrough();
bufferStream.end(file.buffer);
diff --git a/src/services/import/opml.js b/src/services/import/opml.js
index 8c984069d..7630d3d83 100644
--- a/src/services/import/opml.js
+++ b/src/services/import/opml.js
@@ -3,7 +3,13 @@
const noteService = require('../../services/notes');
const parseString = require('xml2js').parseString;
-async function importOpml(fileBuffer, parentNote) {
+/**
+ * @param {ImportContext} importContext
+ * @param {Buffer} fileBuffer
+ * @param {Note} parentNote
+ * @return {Promise<*[]|*>}
+ */
+async function importOpml(importContext, fileBuffer, parentNote) {
const xml = await new Promise(function(resolve, reject)
{
parseString(fileBuffer, function (err, result) {
@@ -24,7 +30,7 @@ async function importOpml(fileBuffer, parentNote) {
let returnNote = null;
for (const outline of outlines) {
- const note = await importOutline(outline, parentNote.noteId);
+ const note = await importOutline(importContext, outline, parentNote.noteId);
// first created note will be activated after import
returnNote = returnNote || note;
@@ -41,9 +47,11 @@ function toHtml(text) {
return '
' + text.replace(/(?:\r\n|\r|\n)/g, '
') + '
'; } -async function importOutline(outline, parentNoteId) { +async function importOutline(importContext, outline, parentNoteId) { const {note} = await noteService.createNote(parentNoteId, outline.$.title, toHtml(outline.$.text)); + importContext.increaseCount(); + for (const childOutline of (outline.outline || [])) { await importOutline(childOutline, note.noteId); } diff --git a/src/services/import/single.js b/src/services/import/single.js index 11eee60ca..8e378d1f8 100644 --- a/src/services/import/single.js +++ b/src/services/import/single.js @@ -4,7 +4,7 @@ const noteService = require('../../services/notes'); const commonmark = require('commonmark'); const path = require('path'); -async function importMarkdown(file, parentNote) { +async function importMarkdown(importContext, file, parentNote) { const markdownContent = file.buffer.toString("UTF-8"); const reader = new commonmark.Parser(); @@ -20,10 +20,13 @@ async function importMarkdown(file, parentNote) { mime: 'text/html' }); + importContext.increaseCount(); + importContext.importFinished(note.noteId); + return note; } -async function importHtml(file, parentNote) { +async function importHtml(importContext, file, parentNote) { const title = getFileNameWithoutExtension(file.originalname); const content = file.buffer.toString("UTF-8"); @@ -32,6 +35,9 @@ async function importHtml(file, parentNote) { mime: 'text/html' }); + importContext.increaseCount(); + importContext.importFinished(note.noteId); + return note; } diff --git a/src/services/import/tar.js b/src/services/import/tar.js index add0d6243..cf0b44a08 100644 --- a/src/services/import/tar.js +++ b/src/services/import/tar.js @@ -6,7 +6,6 @@ const utils = require('../../services/utils'); const log = require('../../services/log'); const repository = require('../../services/repository'); const noteService = require('../../services/notes'); -const messagingService = require('../../services/messaging'); const Branch = require('../../entities/branch'); const tar = require('tar-stream'); const stream = require('stream'); @@ -17,7 +16,13 @@ const mimeTypes = require('mime-types'); let importNoteCount; let lastSentCountTs = Date.now(); -async function importTar(fileBuffer, importRootNote) { +/** + * @param {ImportContext} importContext + * @param {Buffer} fileBuffer + * @param {Note} importRootNote + * @return {Promise<*>} + */ +async function importTar(importContext, fileBuffer, importRootNote) { importNoteCount = 0; // maps from original noteId (in tar file) to newly generated noteId @@ -337,13 +342,7 @@ async function importTar(fileBuffer, importRootNote) { log.info("Ignoring tar import entry with type " + header.type); } - importNoteCount++; - - if (Date.now() - lastSentCountTs >= 1000) { - lastSentCountTs = Date.now(); - - messagingService.importNoteCount(importNoteCount); - } + importContext.increaseCount(); next(); // ready for next entry }); @@ -379,7 +378,7 @@ async function importTar(fileBuffer, importRootNote) { } } - messagingService.importFinished(firstNote); + importContext.importFinished(); resolve(firstNote); }); diff --git a/src/services/messaging.js b/src/services/messaging.js index 9c7f5c598..f19c0b054 100644 --- a/src/services/messaging.js +++ b/src/services/messaging.js @@ -78,18 +78,8 @@ async function refreshTree() { await sendMessageToAllClients({ type: 'refresh-tree' }); } -async function importNoteCount(count) { - await sendMessageToAllClients({ type: 'import-note-count', count: count }); -} - -async function importFinished(firstNote) { - await sendMessageToAllClients({ type: 'import-finished', noteId: firstNote.noteId }); -} - module.exports = { init, sendMessageToAllClients, - refreshTree, - importNoteCount, - importFinished + refreshTree }; \ No newline at end of file