diff --git a/src/public/javascripts/dialogs/import.js b/src/public/javascripts/dialogs/import.js index 27b9629ae..da2dc79f0 100644 --- a/src/public/javascripts/dialogs/import.js +++ b/src/public/javascripts/dialogs/import.js @@ -1,4 +1,5 @@ import treeService from '../services/tree.js'; +import utils from '../services/utils.js'; import treeUtils from "../services/tree_utils.js"; import server from "../services/server.js"; import infoService from "../services/info.js"; @@ -12,7 +13,11 @@ const $importNoteCountWrapper = $("#import-note-count-wrapper"); const $importNoteCount = $("#import-note-count"); const $importButton = $("#import-button"); +let importId; + async function showDialog() { + // each opening of the dialog resets the importId so we don't associate it with previous imports anymore + importId = ''; $importNoteCountWrapper.hide(); $importNoteCount.text('0'); $fileUploadInput.val('').change(); // to trigger Import button disabling listener below @@ -40,8 +45,12 @@ function importIntoNote(importNoteId) { const formData = new FormData(); formData.append('upload', $fileUploadInput[0].files[0]); + // we generate it here (and not on opening) for the case when you try to import multiple times from the same + // dialog (which shouldn't happen, but still ...) + importId = utils.randomString(10); + $.ajax({ - url: baseApiUrl + 'notes/' + importNoteId + '/import', + url: baseApiUrl + 'notes/' + importNoteId + '/import/' + importId, headers: server.getHeaders(), data: formData, dataType: 'json', @@ -54,6 +63,11 @@ function importIntoNote(importNoteId) { } messagingService.subscribeToMessages(async message => { + if (!message.importId || message.importId !== importId) { + // incoming messages must correspond to this import instance + return; + } + if (message.type === 'import-note-count') { $importNoteCountWrapper.show(); diff --git a/src/routes/api/import.js b/src/routes/api/import.js index a73e62550..556bb35ec 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -5,12 +5,45 @@ const enexImportService = require('../../services/import/enex'); const opmlImportService = require('../../services/import/opml'); const tarImportService = require('../../services/import/tar'); const singleImportService = require('../../services/import/single'); +const messagingService = require('../../services/messaging'); const cls = require('../../services/cls'); const path = require('path'); const noteCacheService = require('../../services/note_cache'); +class ImportContext { + constructor(importId) { + // importId is to distinguish between different import events - it is possible (though not recommended) + // to have multiple imports going at the same time + this.importId = importId; + this.count = 0; + this.lastSentCountTs = Date.now(); + } + + async increaseCount() { + this.count++; + + if (Date.now() - this.lastSentCountTs >= 1000) { + this.lastSentCountTs = Date.now(); + + await messagingService.sendMessageToAllClients({ + importId: this.importId, + type: 'import-note-count', + count: this.count + }); + } + } + + async importFinished(noteId) { + await messagingService.sendMessageToAllClients({ + importId: this.importId, + type: 'import-finished', + noteId: noteId + }); + } +} + async function importToBranch(req) { - const parentNoteId = req.params.parentNoteId; + const {parentNoteId, importId} = req.params; const file = req.file; if (!file) { @@ -31,20 +64,22 @@ async function importToBranch(req) { let note; // typically root of the import - client can show it after finishing the import + const importContext = new ImportContext(importId); + if (extension === '.tar') { - note = await tarImportService.importTar(file.buffer, parentNote); + note = await tarImportService.importTar(importContext, file.buffer, parentNote); } else if (extension === '.opml') { - note = await opmlImportService.importOpml(file.buffer, parentNote); + note = await opmlImportService.importOpml(importContext, file.buffer, parentNote); } else if (extension === '.md') { - note = await singleImportService.importMarkdown(file, parentNote); + note = await singleImportService.importMarkdown(importContext, file, parentNote); } else if (extension === '.html' || extension === '.htm') { - note = await singleImportService.importHtml(file, parentNote); + note = await singleImportService.importHtml(importContext, file, parentNote); } else if (extension === '.enex') { - note = await enexImportService.importEnex(file, parentNote); + note = await enexImportService.importEnex(importContext, file, parentNote); } else { return [400, `Unrecognized extension ${extension}, must be .tar or .opml`]; diff --git a/src/routes/routes.js b/src/routes/routes.js index 13054f331..1dc887eca 100644 --- a/src/routes/routes.js +++ b/src/routes/routes.js @@ -129,7 +129,7 @@ function register(app) { apiRoute(PUT, '/api/notes/:noteId/clone-after/:afterBranchId', cloningApiRoute.cloneNoteAfter); route(GET, '/api/notes/:branchId/export/:type/:format', [auth.checkApiAuthOrElectron], exportRoute.exportBranch); - route(POST, '/api/notes/:parentNoteId/import', [auth.checkApiAuthOrElectron, uploadMiddleware], importRoute.importToBranch, apiResultHandler); + route(POST, '/api/notes/:parentNoteId/import/:importId', [auth.checkApiAuthOrElectron, uploadMiddleware], importRoute.importToBranch, apiResultHandler); route(POST, '/api/notes/:parentNoteId/upload', [auth.checkApiAuthOrElectron, uploadMiddleware], filesRoute.uploadFile, apiResultHandler); diff --git a/src/services/import/enex.js b/src/services/import/enex.js index 5793a1d05..0caa89b7f 100644 --- a/src/services/import/enex.js +++ b/src/services/import/enex.js @@ -19,7 +19,7 @@ function parseDate(text) { let note = {}; let resource; -async function importEnex(file, parentNote) { +async function importEnex(importContext, file, parentNote) { const saxStream = sax.createStream(true); const xmlBuilder = new xml2js.Builder({ headless: true }); const parser = new xml2js.Parser({ explicitArray: true }); @@ -218,6 +218,8 @@ async function importEnex(file, parentNote) { mime: 'text/html' })).note; + importContext.increaseCount(); + const noteContent = await noteEntity.getNoteContent(); for (const resource of resources) { @@ -232,39 +234,40 @@ async function importEnex(file, parentNote) { } const createResourceNote = async () => { - const resourceNote = (await noteService.createNote(noteEntity.noteId, resource.title, resource.content, { - attributes: resource.attributes, - type: 'file', - mime: resource.mime - })).note; + const resourceNote = (await noteService.createNote(noteEntity.noteId, resource.title, resource.content, { + attributes: resource.attributes, + type: 'file', + mime: resource.mime + })).note; - const resourceLink = `${utils.escapeHtml(resource.title)}`; + importContext.increaseCount(); - noteContent.content = noteContent.content.replace(mediaRegex, resourceLink); + const resourceLink = `${utils.escapeHtml(resource.title)}`; + + noteContent.content = noteContent.content.replace(mediaRegex, resourceLink); }; if (["image/jpeg", "image/png", "image/gif"].includes(resource.mime)) { - try { - const originalName = "image." + resource.mime.substr(6); + try { + const originalName = "image." + resource.mime.substr(6); - const { url } = await imageService.saveImage(resource.content, originalName, noteEntity.noteId); + const {url} = await imageService.saveImage(resource.content, originalName, noteEntity.noteId); - const imageLink = ``; + const imageLink = ``; - noteContent.content = noteContent.content.replace(mediaRegex, imageLink); + noteContent.content = noteContent.content.replace(mediaRegex, imageLink); - if (!noteContent.content.includes(imageLink)) { - // if there wasn't any match for the reference, we'll add the image anyway - // otherwise image would be removed since no note would include it - noteContent.content += imageLink; + if (!noteContent.content.includes(imageLink)) { + // if there wasn't any match for the reference, we'll add the image anyway + // otherwise image would be removed since no note would include it + noteContent.content += imageLink; + } + } catch (e) { + log.error("error when saving image from ENEX file: " + e); + await createResourceNote(); } - } catch (e) { - log.error("error when saving image from ENEX file: " + e); + } else { await createResourceNote(); - } - } - else { - await createResourceNote(); } } @@ -295,7 +298,12 @@ async function importEnex(file, parentNote) { return new Promise((resolve, reject) => { // resolve only when we parse the whole document AND saving of all notes have been finished - saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) }); + saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => { + importContext.importFinished(rootNote.noteId); + + resolve(rootNote); + }); + }); const bufferStream = new stream.PassThrough(); bufferStream.end(file.buffer); diff --git a/src/services/import/opml.js b/src/services/import/opml.js index 8c984069d..7630d3d83 100644 --- a/src/services/import/opml.js +++ b/src/services/import/opml.js @@ -3,7 +3,13 @@ const noteService = require('../../services/notes'); const parseString = require('xml2js').parseString; -async function importOpml(fileBuffer, parentNote) { +/** + * @param {ImportContext} importContext + * @param {Buffer} fileBuffer + * @param {Note} parentNote + * @return {Promise<*[]|*>} + */ +async function importOpml(importContext, fileBuffer, parentNote) { const xml = await new Promise(function(resolve, reject) { parseString(fileBuffer, function (err, result) { @@ -24,7 +30,7 @@ async function importOpml(fileBuffer, parentNote) { let returnNote = null; for (const outline of outlines) { - const note = await importOutline(outline, parentNote.noteId); + const note = await importOutline(importContext, outline, parentNote.noteId); // first created note will be activated after import returnNote = returnNote || note; @@ -41,9 +47,11 @@ function toHtml(text) { return '

' + text.replace(/(?:\r\n|\r|\n)/g, '

') + '

'; } -async function importOutline(outline, parentNoteId) { +async function importOutline(importContext, outline, parentNoteId) { const {note} = await noteService.createNote(parentNoteId, outline.$.title, toHtml(outline.$.text)); + importContext.increaseCount(); + for (const childOutline of (outline.outline || [])) { await importOutline(childOutline, note.noteId); } diff --git a/src/services/import/single.js b/src/services/import/single.js index 11eee60ca..8e378d1f8 100644 --- a/src/services/import/single.js +++ b/src/services/import/single.js @@ -4,7 +4,7 @@ const noteService = require('../../services/notes'); const commonmark = require('commonmark'); const path = require('path'); -async function importMarkdown(file, parentNote) { +async function importMarkdown(importContext, file, parentNote) { const markdownContent = file.buffer.toString("UTF-8"); const reader = new commonmark.Parser(); @@ -20,10 +20,13 @@ async function importMarkdown(file, parentNote) { mime: 'text/html' }); + importContext.increaseCount(); + importContext.importFinished(note.noteId); + return note; } -async function importHtml(file, parentNote) { +async function importHtml(importContext, file, parentNote) { const title = getFileNameWithoutExtension(file.originalname); const content = file.buffer.toString("UTF-8"); @@ -32,6 +35,9 @@ async function importHtml(file, parentNote) { mime: 'text/html' }); + importContext.increaseCount(); + importContext.importFinished(note.noteId); + return note; } diff --git a/src/services/import/tar.js b/src/services/import/tar.js index add0d6243..cf0b44a08 100644 --- a/src/services/import/tar.js +++ b/src/services/import/tar.js @@ -6,7 +6,6 @@ const utils = require('../../services/utils'); const log = require('../../services/log'); const repository = require('../../services/repository'); const noteService = require('../../services/notes'); -const messagingService = require('../../services/messaging'); const Branch = require('../../entities/branch'); const tar = require('tar-stream'); const stream = require('stream'); @@ -17,7 +16,13 @@ const mimeTypes = require('mime-types'); let importNoteCount; let lastSentCountTs = Date.now(); -async function importTar(fileBuffer, importRootNote) { +/** + * @param {ImportContext} importContext + * @param {Buffer} fileBuffer + * @param {Note} importRootNote + * @return {Promise<*>} + */ +async function importTar(importContext, fileBuffer, importRootNote) { importNoteCount = 0; // maps from original noteId (in tar file) to newly generated noteId @@ -337,13 +342,7 @@ async function importTar(fileBuffer, importRootNote) { log.info("Ignoring tar import entry with type " + header.type); } - importNoteCount++; - - if (Date.now() - lastSentCountTs >= 1000) { - lastSentCountTs = Date.now(); - - messagingService.importNoteCount(importNoteCount); - } + importContext.increaseCount(); next(); // ready for next entry }); @@ -379,7 +378,7 @@ async function importTar(fileBuffer, importRootNote) { } } - messagingService.importFinished(firstNote); + importContext.importFinished(); resolve(firstNote); }); diff --git a/src/services/messaging.js b/src/services/messaging.js index 9c7f5c598..f19c0b054 100644 --- a/src/services/messaging.js +++ b/src/services/messaging.js @@ -78,18 +78,8 @@ async function refreshTree() { await sendMessageToAllClients({ type: 'refresh-tree' }); } -async function importNoteCount(count) { - await sendMessageToAllClients({ type: 'import-note-count', count: count }); -} - -async function importFinished(firstNote) { - await sendMessageToAllClients({ type: 'import-finished', noteId: firstNote.noteId }); -} - module.exports = { init, sendMessageToAllClients, - refreshTree, - importNoteCount, - importFinished + refreshTree }; \ No newline at end of file