From 764d251b0a279692a6e6c537ec854180ccc4a574 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sun, 17 Mar 2024 21:29:57 +0200 Subject: [PATCH] server-ts: Port services/import/enex --- package-lock.json | 38 ++++++++ package.json | 2 + src/routes/api/import.js | 10 +- src/services/import/common.ts | 5 + src/services/import/{enex.js => enex.ts} | 113 +++++++++++++++-------- src/services/note-interface.ts | 25 +++++ src/services/ws.ts | 5 +- 7 files changed, 155 insertions(+), 43 deletions(-) create mode 100644 src/services/import/common.ts rename src/services/import/{enex.js => enex.ts} (81%) create mode 100644 src/services/note-interface.ts diff --git a/package-lock.json b/package-lock.json index c5d7bd009..4123aca01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -99,6 +99,8 @@ "@types/mime-types": "^2.1.4", "@types/node": "^20.11.19", "@types/sanitize-html": "^2.11.0", + "@types/sax": "^1.2.7", + "@types/stream-throttle": "^0.1.4", "@types/turndown": "^5.0.4", "@types/ws": "^8.5.10", "@types/xml2js": "^0.4.14", @@ -1587,6 +1589,15 @@ "entities": "^4.4.0" } }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -1608,6 +1619,15 @@ "@types/node": "*" } }, + "node_modules/@types/stream-throttle": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@types/stream-throttle/-/stream-throttle-0.1.4.tgz", + "integrity": "sha512-VxXIHGjVuK8tYsVm60rIQMmF/0xguCeen5OmK5S4Y6K64A+z+y4/GI6anRnVzaUZaJB9Ah9IfbDcO0o1gZCc/w==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/tough-cookie": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", @@ -14511,6 +14531,15 @@ } } }, + "@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -14532,6 +14561,15 @@ "@types/node": "*" } }, + "@types/stream-throttle": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@types/stream-throttle/-/stream-throttle-0.1.4.tgz", + "integrity": "sha512-VxXIHGjVuK8tYsVm60rIQMmF/0xguCeen5OmK5S4Y6K64A+z+y4/GI6anRnVzaUZaJB9Ah9IfbDcO0o1gZCc/w==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/tough-cookie": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", diff --git a/package.json b/package.json index a555fbad3..91e82dbcd 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,8 @@ "@types/mime-types": "^2.1.4", "@types/node": "^20.11.19", "@types/sanitize-html": "^2.11.0", + "@types/sax": "^1.2.7", + "@types/stream-throttle": "^0.1.4", "@types/turndown": "^5.0.4", "@types/ws": "^8.5.10", "@types/xml2js": "^0.4.14", diff --git a/src/routes/api/import.js b/src/routes/api/import.js index ca59f5159..978fe9d7c 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -1,6 +1,6 @@ "use strict"; -const enexImportService = require('../../services/import/enex.js'); +const enexImportService = require('../../services/import/enex'); const opmlImportService = require('../../services/import/opml'); const zipImportService = require('../../services/import/zip'); const singleImportService = require('../../services/import/single'); @@ -13,8 +13,8 @@ const TaskContext = require('../../services/task_context'); const ValidationError = require('../../errors/validation_error'); async function importNotesToBranch(req) { - const {parentNoteId} = req.params; - const {taskId, last} = req.body; + const { parentNoteId } = req.params; + const { taskId, last } = req.body; const options = { safeImport: req.body.safeImport !== 'false', @@ -81,8 +81,8 @@ async function importNotesToBranch(req) { } async function importAttachmentsToNote(req) { - const {parentNoteId} = req.params; - const {taskId, last} = req.body; + const { parentNoteId } = req.params; + const { taskId, last } = req.body; const options = { shrinkImages: req.body.shrinkImages !== 'false', diff --git a/src/services/import/common.ts b/src/services/import/common.ts new file mode 100644 index 000000000..75c61fb35 --- /dev/null +++ b/src/services/import/common.ts @@ -0,0 +1,5 @@ +export interface File { + originalname: string; + mimetype: string; + buffer: string | Buffer; +} \ No newline at end of file diff --git a/src/services/import/enex.js b/src/services/import/enex.ts similarity index 81% rename from src/services/import/enex.js rename to src/services/import/enex.ts index 74ae5b243..8277ae1bd 100644 --- a/src/services/import/enex.js +++ b/src/services/import/enex.ts @@ -1,20 +1,23 @@ -const sax = require("sax"); -const stream = require('stream'); -const {Throttle} = require('stream-throttle'); -const log = require('../log'); -const utils = require('../utils'); -const sql = require('../sql'); -const noteService = require('../notes'); -const imageService = require('../image'); -const protectedSessionService = require('../protected_session'); -const htmlSanitizer = require('../html_sanitizer'); -const {sanitizeAttributeName} = require('../sanitize_attribute_name'); +import sax = require("sax"); +import stream = require('stream'); +import { Throttle } from 'stream-throttle'; +import log = require('../log'); +import utils = require('../utils'); +import sql = require('../sql'); +import noteService = require('../notes'); +import imageService = require('../image'); +import protectedSessionService = require('../protected_session'); +import htmlSanitizer = require('../html_sanitizer'); +import sanitizeAttributeName = require('../sanitize_attribute_name'); +import TaskContext = require("../task_context"); +import BNote = require("../../becca/entities/bnote"); +import { File } from "./common"; /** * date format is e.g. 20181121T193703Z or 2013-04-14T16:19:00.000Z (Mac evernote, see #3496) * @returns trilium date format, e.g. 2013-04-14 16:19:00.000Z */ -function parseDate(text) { +function parseDate(text: string) { // convert ISO format to the "20181121T193703Z" format text = text.replace(/[-:]/g, ""); @@ -25,10 +28,34 @@ function parseDate(text) { return text; } -let note = {}; -let resource; +interface Attribute { + type: string; + name: string; + value: string; +} -function importEnex(taskContext, file, parentNote) { +interface Resource { + title: string; + content?: Buffer | string; + mime?: string; + attributes: Attribute[]; +} + +interface Note { + title: string; + attributes: Attribute[]; + utcDateCreated: string; + utcDateModified: string; + noteId: string; + blobId: string; + content: string; + resources: Resource[] +} + +let note: Partial = {}; +let resource: Resource; + +function importEnex(taskContext: TaskContext, file: File, parentNote: BNote) { const saxStream = sax.createStream(true); const rootNoteTitle = file.originalname.toLowerCase().endsWith(".enex") @@ -45,7 +72,7 @@ function importEnex(taskContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), }).note; - function extractContent(content) { + function extractContent(content: string) { const openingNoteIndex = content.indexOf(''); if (openingNoteIndex !== -1) { @@ -90,7 +117,7 @@ function importEnex(taskContext, file, parentNote) { } - const path = []; + const path: string[] = []; function getCurrentTag() { if (path.length >= 1) { @@ -108,8 +135,8 @@ function importEnex(taskContext, file, parentNote) { // unhandled errors will throw, since this is a proper node event emitter. log.error(`error when parsing ENEX file: ${e}`); // clear the error - this._parser.error = null; - this._parser.resume(); + (saxStream._parser as any).error = null; + saxStream._parser.resume(); }); saxStream.on("text", text => { @@ -123,13 +150,15 @@ function importEnex(taskContext, file, parentNote) { labelName = 'pageUrl'; } - labelName = sanitizeAttributeName(labelName); + labelName = sanitizeAttributeName.sanitizeAttributeName(labelName || ""); - note.attributes.push({ - type: 'label', - name: labelName, - value: text - }); + if (note.attributes) { + note.attributes.push({ + type: 'label', + name: labelName, + value: text + }); + } } else if (previousTag === 'resource-attributes') { if (currentTag === 'file-name') { @@ -169,10 +198,10 @@ function importEnex(taskContext, file, parentNote) { note.utcDateCreated = parseDate(text); } else if (currentTag === 'updated') { note.utcDateModified = parseDate(text); - } else if (currentTag === 'tag') { + } else if (currentTag === 'tag' && note.attributes) { note.attributes.push({ type: 'label', - name: sanitizeAttributeName(text), + name: sanitizeAttributeName.sanitizeAttributeName(text), value: '' }) } @@ -201,11 +230,13 @@ function importEnex(taskContext, file, parentNote) { attributes: [] }; - note.resources.push(resource); + if (note.resources) { + note.resources.push(resource); + } } }); - function updateDates(note, utcDateCreated, utcDateModified) { + function updateDates(note: BNote, utcDateCreated?: string, utcDateModified?: string) { // it's difficult to force custom dateCreated and dateModified to Note entity, so we do it post-creation with SQL sql.execute(` UPDATE notes @@ -227,6 +258,10 @@ function importEnex(taskContext, file, parentNote) { // make a copy because stream continues with the next call and note gets overwritten let {title, content, attributes, resources, utcDateCreated, utcDateModified} = note; + if (!title || !content) { + throw new Error("Missing title or content for note."); + } + content = extractContent(content); const noteEntity = noteService.createNewNote({ @@ -239,7 +274,7 @@ function importEnex(taskContext, file, parentNote) { isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), }).note; - for (const attr of attributes) { + for (const attr of attributes || []) { noteEntity.addAttribute(attr.type, attr.name, attr.value); } @@ -249,12 +284,14 @@ function importEnex(taskContext, file, parentNote) { taskContext.increaseProgressCount(); - for (const resource of resources) { + for (const resource of resources || []) { if (!resource.content) { continue; } - resource.content = utils.fromBase64(resource.content); + if (typeof resource.content === "string") { + resource.content = utils.fromBase64(resource.content); + } const hash = utils.md5(resource.content); @@ -273,6 +310,10 @@ function importEnex(taskContext, file, parentNote) { resource.mime = resource.mime || "application/octet-stream"; const createFileNote = () => { + if (typeof resource.content !== "string") { + throw new Error("Missing or wrong content type for resource."); + } + const resourceNote = noteService.createNewNote({ parentNoteId: noteEntity.noteId, title: resource.title, @@ -292,7 +333,7 @@ function importEnex(taskContext, file, parentNote) { const resourceLink = `${utils.escapeHtml(resource.title)}`; - content = content.replace(mediaRegex, resourceLink); + content = (content || "").replace(mediaRegex, resourceLink); }; if (resource.mime && resource.mime.startsWith('image/')) { @@ -301,7 +342,7 @@ function importEnex(taskContext, file, parentNote) { ? resource.title : `image.${resource.mime.substr(6)}`; // default if real name is not present - const attachment = imageService.saveImageToAttachment(noteEntity.noteId, resource.content, originalName, taskContext.data.shrinkImages); + const attachment = imageService.saveImageToAttachment(noteEntity.noteId, resource.content, originalName, !!taskContext.data?.shrinkImages); const encodedTitle = encodeURIComponent(attachment.title); const url = `api/attachments/${attachment.attachmentId}/image/${encodedTitle}`; @@ -314,7 +355,7 @@ function importEnex(taskContext, file, parentNote) { // otherwise the image would be removed since no note would include it content += imageLink; } - } catch (e) { + } catch (e: any) { log.error(`error when saving image from ENEX file: ${e.message}`); createFileNote(); } @@ -368,4 +409,4 @@ function importEnex(taskContext, file, parentNote) { }); } -module.exports = { importEnex }; +export = { importEnex }; diff --git a/src/services/note-interface.ts b/src/services/note-interface.ts new file mode 100644 index 000000000..f35caa906 --- /dev/null +++ b/src/services/note-interface.ts @@ -0,0 +1,25 @@ +import { NoteType } from "../becca/entities/rows"; + +export interface NoteParams { + /** optionally can force specific noteId */ + noteId?: string; + parentNoteId: string; + templateNoteId?: string; + title: string; + content: string; + type: NoteType; + /** default value is derived from default mimes for type */ + mime?: string; + /** default is false */ + isProtected?: boolean; + /** default is false */ + isExpanded?: boolean; + /** default is empty string */ + prefix?: string; + /** default is the last existing notePosition in a parent + 10 */ + notePosition?: number; + dateCreated?: string; + utcDateCreated?: string; + ignoreForbiddenParents?: boolean; + target?: "into"; +} \ No newline at end of file diff --git a/src/services/ws.ts b/src/services/ws.ts index fe5ad4dfd..ca6e0ff98 100644 --- a/src/services/ws.ts +++ b/src/services/ws.ts @@ -30,10 +30,11 @@ let lastSyncedPush: number | null = null; interface Message { type: string; - data?: TaskData | null | { + data?: { lastSyncedPush?: number | null, entityChanges?: any[], - }, + shrinkImages?: boolean + } | null, lastSyncedPush?: number | null, progressCount?: number;