diff --git a/src/becca/entities/rows.ts b/src/becca/entities/rows.ts index a10c471e4..4428b6dde 100644 --- a/src/becca/entities/rows.ts +++ b/src/becca/entities/rows.ts @@ -62,7 +62,7 @@ export interface BlobRow { utcDateModified: string; } -export type AttributeType = "label" | "relation"; +export type AttributeType = "label" | "relation" | "label-definition" | "relation-definition"; export interface AttributeRow { attributeId?: string; diff --git a/src/services/import/mime.ts b/src/services/import/mime.ts index eebb54ffe..dd4c96244 100644 --- a/src/services/import/mime.ts +++ b/src/services/import/mime.ts @@ -2,6 +2,7 @@ import mimeTypes = require('mime-types'); import path = require('path'); +import { TaskData } from '../task_context_interface'; const CODE_MIME_TYPES: Record = { 'text/plain': true, @@ -79,12 +80,7 @@ function getMime(fileName: string) { return mimeTypes.lookup(fileName); } -interface GetTypeOpts { - textImportedAsText?: boolean; - codeImportedAsCode?: boolean; -} - -function getType(options: GetTypeOpts, mime: string) { +function getType(options: TaskData, mime: string) { mime = mime ? mime.toLowerCase() : ''; if (options.textImportedAsText && (mime === 'text/html' || ['text/markdown', 'text/x-markdown'].includes(mime))) { diff --git a/src/services/import/zip.js b/src/services/import/zip.ts similarity index 72% rename from src/services/import/zip.js rename to src/services/import/zip.ts index 9d35c539a..fbf7c539b 100644 --- a/src/services/import/zip.js +++ b/src/services/import/zip.ts @@ -1,43 +1,45 @@ "use strict"; -const BAttribute = require('../../becca/entities/battribute'); -const utils = require('../../services/utils'); -const log = require('../../services/log'); -const noteService = require('../../services/notes'); -const attributeService = require('../../services/attributes'); -const BBranch = require('../../becca/entities/bbranch'); -const path = require('path'); -const protectedSessionService = require('../protected_session'); -const mimeService = require('./mime'); -const treeService = require('../tree'); -const yauzl = require("yauzl"); -const htmlSanitizer = require('../html_sanitizer'); -const becca = require('../../becca/becca'); -const BAttachment = require('../../becca/entities/battachment'); -const markdownService = require('./markdown'); +import BAttribute = require('../../becca/entities/battribute'); +import utils = require('../../services/utils'); +import log = require('../../services/log'); +import noteService = require('../../services/notes'); +import attributeService = require('../../services/attributes'); +import BBranch = require('../../becca/entities/bbranch'); +import path = require('path'); +import protectedSessionService = require('../protected_session'); +import mimeService = require('./mime'); +import treeService = require('../tree'); +import yauzl = require("yauzl"); +import htmlSanitizer = require('../html_sanitizer'); +import becca = require('../../becca/becca'); +import BAttachment = require('../../becca/entities/battachment'); +import markdownService = require('./markdown'); +import TaskContext = require('../task_context'); +import BNote = require('../../becca/entities/bnote'); +import NoteMeta = require('../meta/note_meta'); +import AttributeMeta = require('../meta/attribute_meta'); +import { Stream } from 'stream'; +import { NoteType } from '../../becca/entities/rows'; -/** - * @param {TaskContext} taskContext - * @param {Buffer} fileBuffer - * @param {BNote} importRootNote - * @returns {Promise} - */ -async function importZip(taskContext, fileBuffer, importRootNote) { - /** @type {Object.} maps from original noteId (in ZIP file) to newly generated noteId */ - const noteIdMap = {}; - /** @type {Object.} maps from original attachmentId (in ZIP file) to newly generated attachmentId */ - const attachmentIdMap = {}; - const attributes = []; +interface MetaFile { + files: NoteMeta[] +} + +async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRootNote: BNote): Promise { + /** maps from original noteId (in ZIP file) to newly generated noteId */ + const noteIdMap: Record = {}; + /** type maps from original attachmentId (in ZIP file) to newly generated attachmentId */ + const attachmentIdMap: Record = {}; + const attributes: AttributeMeta[] = []; // path => noteId, used only when meta file is not available - /** @type {Object.} path => noteId | attachmentId */ - const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId }; - let metaFile = null; - /** @type {BNote} */ - let firstNote = null; - /** @type {Set.} */ - const createdNoteIds = new Set(); + /** path => noteId | attachmentId */ + const createdPaths: Record = { '/': importRootNote.noteId, '\\': importRootNote.noteId }; + let metaFile!: MetaFile; + let firstNote!: BNote; + const createdNoteIds = new Set(); - function getNewNoteId(origNoteId) { + function getNewNoteId(origNoteId: string) { if (!origNoteId.trim()) { // this probably shouldn't happen, but still good to have this precaution return "empty_note_id"; @@ -55,7 +57,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return noteIdMap[origNoteId]; } - function getNewAttachmentId(origAttachmentId) { + function getNewAttachmentId(origAttachmentId: string) { if (!origAttachmentId.trim()) { // this probably shouldn't happen, but still good to have this precaution return "empty_attachment_id"; @@ -68,12 +70,8 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return attachmentIdMap[origAttachmentId]; } - /** - * @param {NoteMeta} parentNoteMeta - * @param {string} dataFileName - */ - function getAttachmentMeta(parentNoteMeta, dataFileName) { - for (const noteMeta of parentNoteMeta.children) { + function getAttachmentMeta(parentNoteMeta: NoteMeta, dataFileName: string) { + for (const noteMeta of parentNoteMeta.children || []) { for (const attachmentMeta of noteMeta.attachments || []) { if (attachmentMeta.dataFileName === dataFileName) { return { @@ -88,22 +86,20 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return {}; } - /** @returns {{noteMeta: NoteMeta|undefined, parentNoteMeta: NoteMeta|undefined, attachmentMeta: AttachmentMeta|undefined}} */ - function getMeta(filePath) { + function getMeta(filePath: string) { if (!metaFile) { return {}; } const pathSegments = filePath.split(/[\/\\]/g); - /** @type {NoteMeta} */ - let cursor = { + let cursor: NoteMeta | undefined = { isImportRoot: true, - children: metaFile.files + children: metaFile.files, + dataFileName: "" }; - /** @type {NoteMeta} */ - let parent; + let parent!: NoteMeta; for (const segment of pathSegments) { if (!cursor?.children?.length) { @@ -111,7 +107,9 @@ async function importZip(taskContext, fileBuffer, importRootNote) { } parent = cursor; - cursor = parent.children.find(file => file.dataFileName === segment || file.dirFileName === segment); + if (parent.children) { + cursor = parent.children.find(file => file.dataFileName === segment || file.dirFileName === segment); + } if (!cursor) { return getAttachmentMeta(parent, segment); @@ -120,19 +118,15 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return { parentNoteMeta: parent, - noteMeta: cursor + noteMeta: cursor, + attachmentMeta: null }; } - /** - * @param {string} filePath - * @param {NoteMeta} parentNoteMeta - * @return {string} - */ - function getParentNoteId(filePath, parentNoteMeta) { + function getParentNoteId(filePath: string, parentNoteMeta?: NoteMeta) { let parentNoteId; - if (parentNoteMeta) { + if (parentNoteMeta?.noteId) { parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId); } else { @@ -151,13 +145,8 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return parentNoteId; } - /** - * @param {NoteMeta} noteMeta - * @param {string} filePath - * @return {string} - */ - function getNoteId(noteMeta, filePath) { - if (noteMeta) { + function getNoteId(noteMeta: NoteMeta | undefined, filePath: string): string { + if (noteMeta?.noteId) { return getNewNoteId(noteMeta.noteId); } @@ -176,23 +165,19 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return noteId; } - function detectFileTypeAndMime(taskContext, filePath) { + function detectFileTypeAndMime(taskContext: TaskContext, filePath: string) { const mime = mimeService.getMime(filePath) || "application/octet-stream"; - const type = mimeService.getType(taskContext.data, mime); + const type = mimeService.getType(taskContext.data || {}, mime); return { mime, type }; } - /** - * @param {BNote} note - * @param {NoteMeta} noteMeta - */ - function saveAttributes(note, noteMeta) { + function saveAttributes(note: BNote, noteMeta: NoteMeta | undefined) { if (!noteMeta) { return; } - for (const attr of noteMeta.attributes) { + for (const attr of noteMeta.attributes || []) { attr.noteId = note.noteId; if (attr.type === 'label-definition') { @@ -218,11 +203,11 @@ async function importZip(taskContext, fileBuffer, importRootNote) { attr.value = getNewNoteId(attr.value); } - if (taskContext.data.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) { + if (taskContext.data?.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) { attr.name = `disabled:${attr.name}`; } - if (taskContext.data.safeImport) { + if (taskContext.data?.safeImport) { attr.name = htmlSanitizer.sanitize(attr.name); attr.value = htmlSanitizer.sanitize(attr.value); } @@ -231,7 +216,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { } } - function saveDirectory(filePath) { + function saveDirectory(filePath: string) { const { parentNoteMeta, noteMeta } = getMeta(filePath); const noteId = getNoteId(noteMeta, filePath); @@ -240,12 +225,16 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return; } - const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta); + const noteTitle = utils.getNoteTitle(filePath, !!taskContext.data?.replaceUnderscoresWithSpaces, noteMeta); const parentNoteId = getParentNoteId(filePath, parentNoteMeta); + if (!parentNoteId) { + throw new Error("Missing parent note ID."); + } + const {note} = noteService.createNewNote({ parentNoteId: parentNoteId, - title: noteTitle, + title: noteTitle || "", content: '', noteId: noteId, type: resolveNoteType(noteMeta?.type), @@ -265,8 +254,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return noteId; } - /** @returns {{attachmentId: string}|{noteId: string}} */ - function getEntityIdFromRelativeUrl(url, filePath) { + function getEntityIdFromRelativeUrl(url: string, filePath: string) { while (url.startsWith("./")) { url = url.substr(2); } @@ -287,7 +275,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { const { noteMeta, attachmentMeta } = getMeta(absUrl); - if (attachmentMeta) { + if (attachmentMeta && attachmentMeta.attachmentId && noteMeta.noteId) { return { attachmentId: getNewAttachmentId(attachmentMeta.attachmentId), noteId: getNewNoteId(noteMeta.noteId) @@ -299,15 +287,8 @@ async function importZip(taskContext, fileBuffer, importRootNote) { } } - /** - * @param {string} content - * @param {string} noteTitle - * @param {string} filePath - * @param {NoteMeta} noteMeta - * @return {string} - */ - function processTextNoteContent(content, noteTitle, filePath, noteMeta) { - function isUrlAbsolute(url) { + function processTextNoteContent(content: string, noteTitle: string, filePath: string, noteMeta?: NoteMeta) { + function isUrlAbsolute(url: string) { return /^(?:[a-z]+:)?\/\//i.test(url); } @@ -321,7 +302,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { } }); - if (taskContext.data.safeImport) { + if (taskContext.data?.safeImport) { content = htmlSanitizer.sanitize(content); } @@ -336,7 +317,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { try { url = decodeURIComponent(url).trim(); - } catch (e) { + } catch (e: any) { log.error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`); return `src="${url}"`; } @@ -359,7 +340,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { content = content.replace(/href="([^"]*)"/g, (match, url) => { try { url = decodeURIComponent(url).trim(); - } catch (e) { + } catch (e: any) { log.error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`); return `href="${url}"`; } @@ -395,7 +376,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return content; } - function removeTriliumTags(content) { + function removeTriliumTags(content: string) { const tagsToRemove = [ '

([^<]*)<\/h1>', '([^<]*)<\/title>' @@ -407,26 +388,18 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return content; } - /** - * @param {NoteMeta} noteMeta - * @param {string} type - * @param {string} mime - * @param {string|Buffer} content - * @param {string} noteTitle - * @param {string} filePath - * @return {string} - */ - function processNoteContent(noteMeta, type, mime, content, noteTitle, filePath) { - if (noteMeta?.format === 'markdown' - || (!noteMeta && taskContext.data.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime))) { + function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Buffer, noteTitle: string, filePath: string) { + if ((noteMeta?.format === 'markdown' + || (!noteMeta && taskContext.data?.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime))) + && typeof content === "string") { content = markdownService.renderToHtml(content, noteTitle); } - if (type === 'text') { + if (type === 'text' && typeof content === "string") { content = processTextNoteContent(content, noteTitle, filePath, noteMeta); } - if (type === 'relationMap' && noteMeta) { + if (type === 'relationMap' && noteMeta && typeof content === "string") { const relationMapLinks = (noteMeta.attributes || []) .filter(attr => attr.type === 'relation' && attr.name === 'relationMapLink'); @@ -440,11 +413,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return content; } - /** - * @param {string} filePath - * @param {Buffer} content - */ - function saveNote(filePath, content) { + function saveNote(filePath: string, content: string | Buffer) { const { parentNoteMeta, noteMeta, attachmentMeta } = getMeta(filePath); if (noteMeta?.noImport) { @@ -453,7 +422,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { const noteId = getNoteId(noteMeta, filePath); - if (attachmentMeta) { + if (attachmentMeta && attachmentMeta.attachmentId) { const attachment = new BAttachment({ attachmentId: getNewAttachmentId(attachmentMeta.attachmentId), ownerId: noteId, @@ -487,16 +456,20 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return; } - let { type, mime } = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath); - type = resolveNoteType(type); + let { mime } = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath); + if (!mime) { + throw new Error("Unable to resolve mime type."); + } + + let type = resolveNoteType(noteMeta?.type); if (type !== 'file' && type !== 'image') { content = content.toString("utf-8"); } - const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta); + const noteTitle = utils.getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta); - content = processNoteContent(noteMeta, type, mime, content, noteTitle, filePath); + content = processNoteContent(noteMeta, type, mime, content, noteTitle || "", filePath); let note = becca.getNote(noteId); @@ -508,7 +481,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { if (note.type === undefined) { note.type = type; note.mime = mime; - note.title = noteTitle; + note.title = noteTitle || ""; note.isProtected = isProtected; note.save(); } @@ -519,16 +492,20 @@ async function importZip(taskContext, fileBuffer, importRootNote) { new BBranch({ noteId, parentNoteId, - isExpanded: noteMeta.isExpanded, - prefix: noteMeta.prefix, - notePosition: noteMeta.notePosition + isExpanded: noteMeta?.isExpanded, + prefix: noteMeta?.prefix, + notePosition: noteMeta?.notePosition }).save(); } } else { + if (typeof content !== "string") { + throw new Error("Incorrect content type."); + } + ({note} = noteService.createNewNote({ parentNoteId: parentNoteId, - title: noteTitle, + title: noteTitle || "", content: content, noteId, type, @@ -560,7 +537,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { // we're running two passes to make sure that the meta file is loaded before the rest of the files is processed. - await readZipFile(fileBuffer, async (zipfile, entry) => { + await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => { const filePath = normalizeFilePath(entry.fileName); if (filePath === '!!!meta.json') { @@ -572,7 +549,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { zipfile.readEntry(); }); - await readZipFile(fileBuffer, async (zipfile, entry) => { + await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => { const filePath = normalizeFilePath(entry.fileName); if (/\/$/.test(entry.fileName)) { @@ -590,6 +567,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { for (const noteId of createdNoteIds) { const note = becca.getNote(noteId); + if (!note) continue; await noteService.asyncPostProcessContent(note, note.getContent()); if (!metaFile) { @@ -615,8 +593,8 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return firstNote; } -/** @returns {string} path without leading or trailing slash and backslashes converted to forward ones */ -function normalizeFilePath(filePath) { +/** @returns path without leading or trailing slash and backslashes converted to forward ones */ +function normalizeFilePath(filePath: string): string { filePath = filePath.replace(/\\/g, "/"); if (filePath.startsWith("/")) { @@ -630,29 +608,30 @@ function normalizeFilePath(filePath) { return filePath; } -/** @returns {Promise<Buffer>} */ -function streamToBuffer(stream) { - const chunks = []; +function streamToBuffer(stream: Stream): Promise<Buffer> { + const chunks: Uint8Array[] = []; stream.on('data', chunk => chunks.push(chunk)); return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks)))); } -/** @returns {Promise<Buffer>} */ -function readContent(zipfile, entry) { +function readContent(zipfile: yauzl.ZipFile, entry: yauzl.Entry): Promise<Buffer> { return new Promise((res, rej) => { zipfile.openReadStream(entry, function(err, readStream) { if (err) rej(err); + if (!readStream) throw new Error("Unable to read content."); streamToBuffer(readStream).then(res); }); }); } -function readZipFile(buffer, processEntryCallback) { +function readZipFile(buffer: Buffer, processEntryCallback: (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => void) { return new Promise((res, rej) => { yauzl.fromBuffer(buffer, {lazyEntries: true, validateEntrySizes: false}, function(err, zipfile) { if (err) throw err; + if (!zipfile) throw new Error("Unable to read zip file."); + zipfile.readEntry(); zipfile.on("entry", entry => processEntryCallback(zipfile, entry)); zipfile.on("end", res); @@ -660,20 +639,19 @@ function readZipFile(buffer, processEntryCallback) { }); } -function resolveNoteType(type) { +function resolveNoteType(type: string | undefined): NoteType { // BC for ZIPs created in Triliun 0.57 and older if (type === 'relation-map') { - type = 'relationMap'; + return 'relationMap'; } else if (type === 'note-map') { - type = 'noteMap'; + return 'noteMap'; } else if (type === 'web-view') { - type = 'webView'; + return 'webView'; } - return type || "text"; + return "text"; } - -module.exports = { +export = { importZip }; diff --git a/src/services/meta/attribute_meta.ts b/src/services/meta/attribute_meta.ts index 319295944..de79df913 100644 --- a/src/services/meta/attribute_meta.ts +++ b/src/services/meta/attribute_meta.ts @@ -1,9 +1,12 @@ +import { AttributeType } from "../../becca/entities/rows"; + interface AttributeMeta { - type: string; + noteId?: string; + type: AttributeType; name: string; value: string; - isInheritable: boolean; - position: number; + isInheritable?: boolean; + position?: number; } export = AttributeMeta; diff --git a/src/services/meta/note_meta.ts b/src/services/meta/note_meta.ts index a82d84e58..b3012f29a 100644 --- a/src/services/meta/note_meta.ts +++ b/src/services/meta/note_meta.ts @@ -17,6 +17,7 @@ interface NoteMeta { dirFileName?: string; /** this file should not be imported (e.g., HTML navigation) */ noImport?: boolean; + isImportRoot?: boolean; attributes?: AttributeMeta[]; attachments?: AttachmentMeta[]; children?: NoteMeta[]; diff --git a/src/services/notes.ts b/src/services/notes.ts index 606bd0b15..26c96ceb8 100644 --- a/src/services/notes.ts +++ b/src/services/notes.ts @@ -167,7 +167,7 @@ interface NoteParams { /** default is false */ isExpanded?: boolean; /** default is empty string */ - prefix?: string; + prefix?: string | null; /** default is the last existing notePosition in a parent + 10 */ notePosition?: number; dateCreated?: string; @@ -657,7 +657,7 @@ function saveAttachments(note: BNote, content: string) { return content; } -function saveLinks(note: BNote, content: string) { +function saveLinks(note: BNote, content: string | Buffer) { if ((note.type !== 'text' && note.type !== 'relationMap') || (note.isProtected && !protectedSessionService.isProtectedSessionAvailable())) { return { @@ -669,7 +669,7 @@ function saveLinks(note: BNote, content: string) { const foundLinks: FoundLink[] = []; let forceFrontendReload = false; - if (note.type === 'text') { + if (note.type === 'text' && typeof content === "string") { content = downloadImages(note.noteId, content); content = saveAttachments(note, content); @@ -679,7 +679,7 @@ function saveLinks(note: BNote, content: string) { ({forceFrontendReload, content} = checkImageAttachments(note, content)); } - else if (note.type === 'relationMap') { + else if (note.type === 'relationMap' && typeof content === "string") { findRelationMapLinks(content, foundLinks); } else { @@ -874,7 +874,7 @@ function getUndeletedParentBranchIds(noteId: string, deleteId: string) { AND parentNote.isDeleted = 0`, [noteId, deleteId]); } -function scanForLinks(note: BNote, content: string) { +function scanForLinks(note: BNote, content: string | Buffer) { if (!note || !['text', 'relationMap'].includes(note.type)) { return; } @@ -896,7 +896,7 @@ function scanForLinks(note: BNote, content: string) { /** * Things which have to be executed after updating content, but asynchronously (separate transaction) */ -async function asyncPostProcessContent(note: BNote, content: string) { +async function asyncPostProcessContent(note: BNote, content: string | Buffer) { if (cls.isMigrationRunning()) { // this is rarely needed for migrations, but can cause trouble by e.g. triggering downloads return; diff --git a/src/services/utils.ts b/src/services/utils.ts index 25f2e6be8..6b2974df0 100644 --- a/src/services/utils.ts +++ b/src/services/utils.ts @@ -226,8 +226,8 @@ function removeTextFileExtension(filePath: string) { } } -function getNoteTitle(filePath: string, replaceUnderscoresWithSpaces: boolean, noteMeta?: { title: string }) { - if (noteMeta) { +function getNoteTitle(filePath: string, replaceUnderscoresWithSpaces: boolean, noteMeta?: { title?: string }) { + if (noteMeta?.title) { return noteMeta.title; } else { const basename = path.basename(removeTextFileExtension(filePath));