diff --git a/src/public/javascripts/services/import.js b/src/public/javascripts/services/import.js index 451b92405..cc493d496 100644 --- a/src/public/javascripts/services/import.js +++ b/src/public/javascripts/services/import.js @@ -9,7 +9,7 @@ export async function uploadFiles(parentNoteId, files, options) { return; } - const importId = utils.randomString(10); + const taskId = utils.randomString(10); let noteId; let counter = 0; @@ -18,7 +18,7 @@ export async function uploadFiles(parentNoteId, files, options) { const formData = new FormData(); formData.append('upload', file); - formData.append('importId', importId); + formData.append('taskId', taskId); formData.append('last', counter === files.length ? "true" : "false"); for (const key in options) { @@ -41,23 +41,23 @@ export async function uploadFiles(parentNoteId, files, options) { ws.subscribeToMessages(async message => { const toast = { id: "import", - title: "Import", + title: "Import status", icon: "plus" }; - if (message.type === 'import-error') { + if (message.type === 'task-error' && message.taskType === 'import') { infoService.closePersistent(toast.id); infoService.showError(message.message); return; } - if (message.type === 'import-progress-count') { + if (message.type === 'task-progress-count' && message.taskType === 'import') { toast.message = "Import in progress: " + message.progressCount; infoService.showPersistent(toast); } - if (message.type === 'import-succeeded') { + if (message.type === 'task-succeeded' && message.taskType === 'import') { toast.message = "Import finished successfully."; toast.closeAfter = 5000; diff --git a/src/routes/api/import.js b/src/routes/api/import.js index 049aedfc1..8ca0771e3 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -9,11 +9,11 @@ const cls = require('../../services/cls'); const path = require('path'); const noteCacheService = require('../../services/note_cache'); const log = require('../../services/log'); -const ImportContext = require('../../services/import_context'); +const TaskContext = require('../../services/task_context.js'); async function importToBranch(req) { const {parentNoteId} = req.params; - const {importId, last} = req.body; + const {taskId, last} = req.body; const options = { safeImport: req.body.safeImport !== 'false', @@ -43,22 +43,22 @@ async function importToBranch(req) { let note; // typically root of the import - client can show it after finishing the import - const importContext = ImportContext.getInstance(importId, options); + const taskContext = TaskContext.getInstance(taskId, options); try { if (extension === '.tar' && options.explodeArchives) { - note = await tarImportService.importTar(importContext, file.buffer, parentNote); + note = await tarImportService.importTar(taskContext, file.buffer, parentNote); } else if (extension === '.opml' && options.explodeArchives) { - note = await opmlImportService.importOpml(importContext, file.buffer, parentNote); + note = await opmlImportService.importOpml(taskContext, file.buffer, parentNote); } else if (extension === '.enex' && options.explodeArchives) { - note = await enexImportService.importEnex(importContext, file, parentNote); + note = await enexImportService.importEnex(taskContext, file, parentNote); } else { - note = await singleImportService.importSingleFile(importContext, file, parentNote); + note = await singleImportService.importSingleFile(taskContext, file, parentNote); } } catch (e) { const message = "Import failed with following error: '" + e.message + "'. More details might be in the logs."; - importContext.reportError(message); + taskContext.reportError(message); log.error(message + e.stack); @@ -67,7 +67,7 @@ async function importToBranch(req) { if (last === "true") { // small timeout to avoid race condition (message is received before the transaction is committed) - setTimeout(() => importContext.importSucceeded(parentNoteId, note.noteId), 1000); + setTimeout(() => taskContext.taskSucceeded(parentNoteId, note.noteId), 1000); } // import has deactivated note events so note cache is not updated diff --git a/src/services/import/enex.js b/src/services/import/enex.js index 854be6f6b..045187ead 100644 --- a/src/services/import/enex.js +++ b/src/services/import/enex.js @@ -20,7 +20,7 @@ function parseDate(text) { let note = {}; let resource; -async function importEnex(importContext, file, parentNote) { +async function importEnex(taskContext, file, parentNote) { const saxStream = sax.createStream(true); const xmlBuilder = new xml2js.Builder({ headless: true }); const parser = new xml2js.Parser({ explicitArray: true }); @@ -221,7 +221,7 @@ async function importEnex(importContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), })).note; - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); let noteContent = await noteEntity.getContent(); @@ -244,7 +244,7 @@ async function importEnex(importContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), })).note; - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); const resourceLink = `${utils.escapeHtml(resource.title)}`; @@ -255,7 +255,7 @@ async function importEnex(importContext, file, parentNote) { try { const originalName = "image." + resource.mime.substr(6); - const {url} = await imageService.saveImage(resource.content, originalName, noteEntity.noteId, importContext.shrinkImages); + const {url} = await imageService.saveImage(resource.content, originalName, noteEntity.noteId, taskContext.data.shrinkImages); const imageLink = ``; diff --git a/src/services/import/mime.js b/src/services/import/mime.js index 8ea3f5361..179bffebd 100644 --- a/src/services/import/mime.js +++ b/src/services/import/mime.js @@ -79,13 +79,13 @@ function getMime(fileName) { return mimeTypes.lookup(fileName); } -function getType(importContext, mime) { +function getType(options, mime) { mime = mime ? mime.toLowerCase() : ''; - if (importContext.textImportedAsText && (mime === 'text/html' || ['text/markdown', 'text/x-markdown'].includes(mime))) { + if (options.textImportedAsText && (mime === 'text/html' || ['text/markdown', 'text/x-markdown'].includes(mime))) { return 'text'; } - else if (importContext.codeImportedAsCode && mime in CODE_MIME_TYPES) { + else if (options.codeImportedAsCode && mime in CODE_MIME_TYPES) { return 'code'; } else if (mime.startsWith("image/")) { diff --git a/src/services/import/opml.js b/src/services/import/opml.js index 45f66891a..3b9e4f852 100644 --- a/src/services/import/opml.js +++ b/src/services/import/opml.js @@ -5,12 +5,12 @@ const parseString = require('xml2js').parseString; const protectedSessionService = require('../protected_session'); /** - * @param {ImportContext} importContext + * @param {TaskContext} taskContext * @param {Buffer} fileBuffer * @param {Note} parentNote * @return {Promise<*[]|*>} */ -async function importOpml(importContext, fileBuffer, parentNote) { +async function importOpml(taskContext, fileBuffer, parentNote) { const xml = await new Promise(function(resolve, reject) { parseString(fileBuffer, function (err, result) { @@ -48,7 +48,7 @@ async function importOpml(importContext, fileBuffer, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), }); - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); for (const childOutline of (outline.outline || [])) { await importOutline(childOutline, note.noteId); diff --git a/src/services/import/single.js b/src/services/import/single.js index 0ef9c0750..f5a9167ca 100644 --- a/src/services/import/single.js +++ b/src/services/import/single.js @@ -7,39 +7,39 @@ const commonmark = require('commonmark'); const path = require('path'); const mimeService = require('./mime'); -async function importSingleFile(importContext, file, parentNote) { +async function importSingleFile(taskContext, file, parentNote) { const mime = mimeService.getMime(file.originalname) || file.mimetype; - if (importContext.textImportedAsText) { + if (taskContext.data.textImportedAsText) { if (mime === 'text/html') { - return await importHtml(importContext, file, parentNote); + return await importHtml(taskContext, file, parentNote); } else if (['text/markdown', 'text/x-markdown'].includes(mime)) { - return await importMarkdown(importContext, file, parentNote); + return await importMarkdown(taskContext, file, parentNote); } else if (mime === 'text/plain') { - return await importPlainText(importContext, file, parentNote); + return await importPlainText(taskContext, file, parentNote); } } - if (importContext.codeImportedAsCode && mimeService.getType(importContext, mime) === 'code') { - return await importCodeNote(importContext, file, parentNote); + if (taskContext.data.codeImportedAsCode && mimeService.getType(taskContext.data, mime) === 'code') { + return await importCodeNote(taskContext, file, parentNote); } if (["image/jpeg", "image/gif", "image/png", "image/webp"].includes(mime)) { - return await importImage(file, parentNote, importContext); + return await importImage(file, parentNote, taskContext); } - return await importFile(importContext, file, parentNote); + return await importFile(taskContext, file, parentNote); } -async function importImage(file, parentNote, importContext) { - const {note} = await imageService.saveImage(file.buffer, file.originalname, parentNote.noteId, importContext.shrinkImages); +async function importImage(file, parentNote, taskContext) { + const {note} = await imageService.saveImage(file.buffer, file.originalname, parentNote.noteId, taskContext.data.shrinkImages); - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); return note; } -async function importFile(importContext, file, parentNote) { +async function importFile(taskContext, file, parentNote) { const originalName = file.originalname; const size = file.size; @@ -54,12 +54,12 @@ async function importFile(importContext, file, parentNote) { ] }); - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); return note; } -async function importCodeNote(importContext, file, parentNote) { +async function importCodeNote(taskContext, file, parentNote) { const title = getFileNameWithoutExtension(file.originalname); const content = file.buffer.toString("UTF-8"); const detectedMime = mimeService.getMime(file.originalname) || file.mimetype; @@ -71,12 +71,12 @@ async function importCodeNote(importContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable() }); - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); return note; } -async function importPlainText(importContext, file, parentNote) { +async function importPlainText(taskContext, file, parentNote) { const title = getFileNameWithoutExtension(file.originalname); const plainTextContent = file.buffer.toString("UTF-8"); const htmlContent = convertTextToHtml(plainTextContent); @@ -87,7 +87,7 @@ async function importPlainText(importContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), }); - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); return note; } @@ -110,7 +110,7 @@ function convertTextToHtml(text) { return text; } -async function importMarkdown(importContext, file, parentNote) { +async function importMarkdown(taskContext, file, parentNote) { const markdownContent = file.buffer.toString("UTF-8"); const reader = new commonmark.Parser(); @@ -127,12 +127,12 @@ async function importMarkdown(importContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), }); - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); return note; } -async function importHtml(importContext, file, parentNote) { +async function importHtml(taskContext, file, parentNote) { const title = getFileNameWithoutExtension(file.originalname); const content = file.buffer.toString("UTF-8"); @@ -142,7 +142,7 @@ async function importHtml(importContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), }); - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); return note; } diff --git a/src/services/import/tar.js b/src/services/import/tar.js index 5d6399167..4e2b52b21 100644 --- a/src/services/import/tar.js +++ b/src/services/import/tar.js @@ -11,18 +11,18 @@ const tar = require('tar-stream'); const stream = require('stream'); const path = require('path'); const commonmark = require('commonmark'); -const ImportContext = require('../import_context'); +const TaskContext = require('../task_context.js'); const protectedSessionService = require('../protected_session'); const mimeService = require("./mime"); const treeService = require("../tree"); /** - * @param {ImportContext} importContext + * @param {TaskContext} taskContext * @param {Buffer} fileBuffer * @param {Note} importRootNote * @return {Promise<*>} */ -async function importTar(importContext, fileBuffer, importRootNote) { +async function importTar(taskContext, fileBuffer, importRootNote) { // maps from original noteId (in tar file) to newly generated noteId const noteIdMap = {}; const attributes = []; @@ -127,9 +127,9 @@ async function importTar(importContext, fileBuffer, importRootNote) { return noteId; } - function detectFileTypeAndMime(importContext, filePath) { + function detectFileTypeAndMime(taskContext, filePath) { const mime = mimeService.getMime(filePath) || "application/octet-stream"; - const type = mimeService.getType(importContext, mime); + const type = mimeService.getType(taskContext.data, mime); return { mime, type }; } @@ -156,7 +156,7 @@ async function importTar(importContext, fileBuffer, importRootNote) { attr.value = getNewNoteId(attr.value); } - if (importContext.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) { + if (taskContext.data.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) { attr.name = 'disabled-' + attr.name; } @@ -248,7 +248,7 @@ async function importTar(importContext, fileBuffer, importRootNote) { return; } - const {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(importContext, filePath); + const {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath); if (type !== 'file' && type !== 'image') { content = content.toString("UTF-8"); @@ -402,7 +402,7 @@ async function importTar(importContext, fileBuffer, importRootNote) { log.info("Ignoring tar import entry with type " + header.type); } - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); next(); // ready for next entry }); @@ -429,7 +429,7 @@ async function importTar(importContext, fileBuffer, importRootNote) { await treeService.sortNotesAlphabetically(noteId, true); } - importContext.increaseProgressCount(); + taskContext.increaseProgressCount(); } // we're saving attributes and links only now so that all relation and link target notes diff --git a/src/services/import_context.js b/src/services/import_context.js deleted file mode 100644 index 77980fd68..000000000 --- a/src/services/import_context.js +++ /dev/null @@ -1,65 +0,0 @@ -"use strict"; - -const ws = require('./ws.js'); - -// importId => ImportContext -const importContexts = {}; - -class ImportContext { - constructor(importId, options) { - // importId is to distinguish between different import events - it is possible (though not recommended) - // to have multiple imports going at the same time - this.importId = importId; - - this.safeImport = options.safeImport; - this.shrinkImages = options.shrinkImages; - this.codeImportedAsCode = options.codeImportedAsCode; - this.textImportedAsText = options.textImportedAsText; - - // // count is mean to represent count of exported notes where practical, otherwise it's just some measure of progress - this.progressCount = 0; - this.lastSentCountTs = Date.now(); - } - - /** @return {ImportContext} */ - static getInstance(importId, options) { - if (!importContexts[importId]) { - importContexts[importId] = new ImportContext(importId, options); - } - - return importContexts[importId]; - } - - async increaseProgressCount() { - this.progressCount++; - - if (Date.now() - this.lastSentCountTs >= 300) { - this.lastSentCountTs = Date.now(); - - await ws.sendMessageToAllClients({ - importId: this.importId, - type: 'import-progress-count', - progressCount: this.progressCount - }); - } - } - - // must remaing non-static - async reportError(message) { - await ws.sendMessageToAllClients({ - type: 'import-error', - message: message - }); - } - - // must remaing non-static - async importSucceeded(parentNoteId, importedNoteId) { - await ws.sendMessageToAllClients({ - type: 'import-succeeded', - parentNoteId: parentNoteId, - importedNoteId: importedNoteId - }); - } -} - -module.exports = ImportContext; \ No newline at end of file diff --git a/src/services/sql_init.js b/src/services/sql_init.js index f3189cc83..f0844c528 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -9,7 +9,7 @@ const cls = require('./cls'); const utils = require('./utils'); const optionService = require('./options'); const Option = require('../entities/option'); -const ImportContext = require('../services/import_context'); +const TaskContext = require('./task_context.js'); async function createConnection() { return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise}); @@ -112,10 +112,10 @@ async function createInitialDatabase(username, password, theme) { notePosition: 0 }).save(); - const dummyImportContext = new ImportContext("1", false); + const dummyTaskContext = new TaskContext("1", 'import', false); const tarImportService = require("./import/tar"); - await tarImportService.importTar(dummyImportContext, demoFile, rootNote); + await tarImportService.importTar(dummyTaskContext, demoFile, rootNote); const startNoteId = await sql.getValue("SELECT noteId FROM branches WHERE parentNoteId = 'root' AND isDeleted = 0 ORDER BY notePosition"); diff --git a/src/services/task_context.js b/src/services/task_context.js new file mode 100644 index 000000000..ed092cbfa --- /dev/null +++ b/src/services/task_context.js @@ -0,0 +1,65 @@ +"use strict"; + +const ws = require('./ws.js'); + +// taskId => TaskContext +const taskContexts = {}; + +class TaskContext { + constructor(taskId, taskType, data) { + this.taskId = taskId; + this.taskType = taskType; + this.data = data; + + // progressCount is meant to represent just some progress - to indicate the task is not stuck + this.progressCount = 0; + this.lastSentCountTs = Date.now(); + } + + /** @return {TaskContext} */ + static getInstance(taskId, data) { + if (!taskContexts[taskId]) { + taskContexts[taskId] = new TaskContext(taskId, 'import', data); + } + + return taskContexts[taskId]; + } + + async increaseProgressCount() { + this.progressCount++; + + if (Date.now() - this.lastSentCountTs >= 300) { + this.lastSentCountTs = Date.now(); + + await ws.sendMessageToAllClients({ + type: 'task-progress-count', + taskId: this.taskId, + taskType: this.taskType, + progressCount: this.progressCount + }); + } + } + + // must remaing non-static + async reportError(message) { + await ws.sendMessageToAllClients({ + type: 'task-error', + taskId: this.taskId, + taskType: this.taskType, + message: message + }); + } + + // must remaing non-static + async taskSucceeded(parentNoteId, importedNoteId) { + await ws.sendMessageToAllClients({ + type: 'task-succeeded', + taskId: this.taskId, + taskType: this.taskType, + parentNoteId: parentNoteId, + importedNoteId: importedNoteId + }); + } +} + +module.exports = TaskContext; \ No newline at end of file diff --git a/src/views/mobile.ejs b/src/views/mobile.ejs index 217093173..83decb5dc 100644 --- a/src/views/mobile.ejs +++ b/src/views/mobile.ejs @@ -4,7 +4,7 @@ Trilium Notes - +