const sql = require('./sql'); const sqlInit = require('./sql_init'); const optionService = require('./options'); const dateUtils = require('./date_utils'); const entityChangesService = require('./entity_changes.js'); const eventService = require('./events'); const repository = require('./repository'); const cls = require('../services/cls'); const Note = require('../entities/note'); const NoteRevision = require('../entities/note_revision'); const Branch = require('../entities/branch'); const Attribute = require('../entities/attribute'); const hoistedNoteService = require('../services/hoisted_note'); const protectedSessionService = require('../services/protected_session'); const log = require('../services/log'); const utils = require('../services/utils'); const noteRevisionService = require('../services/note_revisions'); const attributeService = require('../services/attributes'); const request = require('./request'); const path = require('path'); const url = require('url'); function getNewNotePosition(parentNoteId) { const maxNotePos = sql.getValue(` SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0`, [parentNoteId]); return maxNotePos === null ? 0 : maxNotePos + 10; } function triggerChildNoteCreated(childNote, parentNote) { eventService.emit(eventService.CHILD_NOTE_CREATED, { childNote, parentNote }); } function triggerNoteTitleChanged(note) { eventService.emit(eventService.NOTE_TITLE_CHANGED, note); } function deriveMime(type, mime) { if (!type) { throw new Error(`Note type is a required param`); } if (mime) { return mime; } if (type === 'text') { mime = 'text/html'; } else if (type === 'code') { mime = 'text/plain'; } else if (['relation-map', 'search'].includes(type)) { mime = 'application/json'; } else if (['render', 'book'].includes(type)) { mime = ''; } return mime; } function copyChildAttributes(parentNote, childNote) { for (const attr of parentNote.getOwnedAttributes()) { if (attr.name.startsWith("child:")) { new Attribute({ noteId: childNote.noteId, type: attr.type, name: attr.name.substr(6), value: attr.value, position: attr.position, isInheritable: attr.isInheritable }).save(); childNote.invalidateAttributeCache(); } } } /** * Following object properties are mandatory: * - {string} parentNoteId * - {string} title * - {*} content * - {string} type - text, code, file, image, search, book, relation-map, render * * Following are optional (have defaults) * - {string} mime - value is derived from default mimes for type * - {boolean} isProtected - default is false * - {boolean} isExpanded - default is false * - {string} prefix - default is empty string * - {integer} notePosition - default is last existing notePosition in a parent + 10 * * @param params * @return {{note: Note, branch: Branch}} */ function createNewNote(params) { const parentNote = repository.getNote(params.parentNoteId); if (!parentNote) { throw new Error(`Parent note "${params.parentNoteId}" not found.`); } if (!params.title || params.title.trim().length === 0) { throw new Error(`Note title must not be empty`); } return sql.transactional(() => { const note = new Note({ noteId: params.noteId, // optionally can force specific noteId title: params.title, isProtected: !!params.isProtected, type: params.type, mime: deriveMime(params.type, params.mime) }).save(); note.setContent(params.content); const branch = new Branch({ noteId: note.noteId, parentNoteId: params.parentNoteId, notePosition: params.notePosition !== undefined ? params.notePosition : getNewNotePosition(params.parentNoteId), prefix: params.prefix, isExpanded: !!params.isExpanded }).save(); scanForLinks(note); copyChildAttributes(parentNote, note); triggerNoteTitleChanged(note); triggerChildNoteCreated(note, parentNote); return { note, branch }; }); } function createNewNoteWithTarget(target, targetBranchId, params) { if (!params.type) { const parentNote = repository.getNote(params.parentNoteId); // code note type can be inherited, otherwise text is default params.type = parentNote.type === 'code' ? 'code' : 'text'; params.mime = parentNote.type === 'code' ? parentNote.mime : 'text/html'; } if (target === 'into') { return createNewNote(params); } else if (target === 'after') { const afterNote = sql.getRow('SELECT notePosition FROM branches WHERE branchId = ?', [targetBranchId]); // not updating utcDateModified to avoig having to sync whole rows sql.execute('UPDATE branches SET notePosition = notePosition + 10 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0', [params.parentNoteId, afterNote.notePosition]); params.notePosition = afterNote.notePosition + 10; const retObject = createNewNote(params); entityChangesService.addNoteReorderingEntityChange(params.parentNoteId); return retObject; } else { throw new Error(`Unknown target ${target}`); } } function protectNoteRecursively(note, protect, includingSubTree, taskContext) { protectNote(note, protect); taskContext.increaseProgressCount(); if (includingSubTree) { for (const child of note.getChildNotes()) { protectNoteRecursively(child, protect, includingSubTree, taskContext); } } } function protectNote(note, protect) { if (protect !== note.isProtected) { const content = note.getContent(); note.isProtected = protect; // this will force de/encryption note.setContent(content); note.save(); } noteRevisionService.protectNoteRevisions(note); } function findImageLinks(content, foundLinks) { const re = /src="[^"]*api\/images\/([a-zA-Z0-9]+)\//g; let match; while (match = re.exec(content)) { foundLinks.push({ name: 'imageLink', value: match[1] }); } // removing absolute references to server to keep it working between instances // we also omit / at the beginning to keep the paths relative return content.replace(/src="[^"]*\/api\/images\//g, 'src="api/images/'); } function findInternalLinks(content, foundLinks) { const re = /href="[^"]*#root[a-zA-Z0-9\/]*\/([a-zA-Z0-9]+)\/?"/g; let match; while (match = re.exec(content)) { foundLinks.push({ name: 'internalLink', value: match[1] }); } // removing absolute references to server to keep it working between instances return content.replace(/href="[^"]*#root/g, 'href="#root'); } function findIncludeNoteLinks(content, foundLinks) { const re = /
]*>/g; let match; while (match = re.exec(content)) { foundLinks.push({ name: 'includeNoteLink', value: match[1] }); } return content; } function findRelationMapLinks(content, foundLinks) { const obj = JSON.parse(content); for (const note of obj.notes) { foundLinks.push({ name: 'relationMapLink', value: note.noteId }); } } const imageUrlToNoteIdMapping = {}; async function downloadImage(noteId, imageUrl) { try { const imageBuffer = await request.getImage(imageUrl); const parsedUrl = url.parse(imageUrl); const title = path.basename(parsedUrl.pathname); const imageService = require('../services/image'); const {note} = imageService.saveImage(noteId, imageBuffer, title, true); note.addLabel('imageUrl', imageUrl); imageUrlToNoteIdMapping[imageUrl] = note.noteId; log.info(`Download of ${imageUrl} succeeded and was saved as image note ${note.noteId}`); } catch (e) { log.error(`Download of ${imageUrl} for note ${noteId} failed with error: ${e.message} ${e.stack}`); } } /** url => download promise */ const downloadImagePromises = {}; function replaceUrl(content, url, imageNote) { const quotedUrl = utils.quoteRegex(url); return content.replace(new RegExp(`\\s+src=[\"']${quotedUrl}[\"']`, "ig"), ` src="api/images/${imageNote.noteId}/${imageNote.title}"`); } function downloadImages(noteId, content) { const imageRe = /]*?\ssrc=['"]([^'">]+)['"]/ig; let imageMatch; while (imageMatch = imageRe.exec(content)) { const url = imageMatch[1]; const inlineImageMatch = /^data:image\/[a-z]+;base64,/.exec(url); if (inlineImageMatch) { const imageBase64 = url.substr(inlineImageMatch[0].length); const imageBuffer = Buffer.from(imageBase64, 'base64'); const imageService = require('../services/image'); const {note} = imageService.saveImage(noteId, imageBuffer, "inline image", true); content = content.substr(0, imageMatch.index) + ` note.type === 'image'); if (existingImage) { imageUrlToNoteIdMapping[url] = existingImage.noteId; content = replaceUrl(content, url, existingImage); continue; } if (url in downloadImagePromises) { // download is already in progress continue; } // this is done asynchronously, it would be too slow to wait for the download // given that save can be triggered very often downloadImagePromises[url] = downloadImage(noteId, url); } } Promise.all(Object.values(downloadImagePromises)).then(() => { setTimeout(() => { // the normal expected flow of the offline image saving is that users will paste the image(s) // which will get asynchronously downloaded, during that time they keep editing the note // once the download is finished, the image note representing downloaded image will be used // to replace the IMG link. // However there's another flow where user pastes the image and leaves the note before the images // are downloaded and the IMG references are not updated. For this occassion we have this code // which upon the download of all the images will update the note if the links have not been fixed before sql.transactional(() => { const imageNotes = repository.getNotes(Object.values(imageUrlToNoteIdMapping)); const origNote = repository.getNote(noteId); const origContent = origNote.getContent(); let updatedContent = origContent; for (const url in imageUrlToNoteIdMapping) { const imageNote = imageNotes.find(note => note.noteId === imageUrlToNoteIdMapping[url]); if (imageNote && !imageNote.isDeleted) { updatedContent = replaceUrl(updatedContent, url, imageNote); } } // update only if the links have not been already fixed. if (updatedContent !== origContent) { origNote.setContent(updatedContent); scanForLinks(origNote); console.log(`Fixed the image links for note ${noteId} to the offline saved.`); } }); }, 5000); }); return content; } function saveLinks(note, content) { if (note.type !== 'text' && note.type !== 'relation-map') { return content; } if (note.isProtected && !protectedSessionService.isProtectedSessionAvailable()) { return content; } const foundLinks = []; if (note.type === 'text') { content = downloadImages(note.noteId, content); content = findImageLinks(content, foundLinks); content = findInternalLinks(content, foundLinks); content = findIncludeNoteLinks(content, foundLinks); } else if (note.type === 'relation-map') { findRelationMapLinks(content, foundLinks); } else { throw new Error("Unrecognized type " + note.type); } const existingLinks = note.getLinks(); for (const foundLink of foundLinks) { const targetNote = repository.getNote(foundLink.value); if (!targetNote || targetNote.isDeleted) { continue; } const existingLink = existingLinks.find(existingLink => existingLink.value === foundLink.value && existingLink.name === foundLink.name); if (!existingLink) { const newLink = new Attribute({ noteId: note.noteId, type: 'relation', name: foundLink.name, value: foundLink.value, }).save(); existingLinks.push(newLink); } else if (existingLink.isDeleted) { existingLink.isDeleted = false; existingLink.save(); } // else the link exists so we don't need to do anything } // marking links as deleted if they are not present on the page anymore const unusedLinks = existingLinks.filter(existingLink => !foundLinks.some(foundLink => existingLink.value === foundLink.value && existingLink.name === foundLink.name)); for (const unusedLink of unusedLinks) { unusedLink.isDeleted = true; unusedLink.save(); } return content; } function saveNoteRevision(note) { // files and images are versioned separately if (note.type === 'file' || note.type === 'image' || note.hasLabel('disableVersioning')) { return; } const now = new Date(); const noteRevisionSnapshotTimeInterval = parseInt(optionService.getOption('noteRevisionSnapshotTimeInterval')); const revisionCutoff = dateUtils.utcDateStr(new Date(now.getTime() - noteRevisionSnapshotTimeInterval * 1000)); const existingNoteRevisionId = sql.getValue( "SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND utcDateCreated >= ?", [note.noteId, revisionCutoff]); const msSinceDateCreated = now.getTime() - dateUtils.parseDateTime(note.utcDateCreated).getTime(); if (!existingNoteRevisionId && msSinceDateCreated >= noteRevisionSnapshotTimeInterval * 1000) { const noteRevision = new NoteRevision({ noteId: note.noteId, // title and text should be decrypted now title: note.title, type: note.type, mime: note.mime, isProtected: false, // will be fixed in the protectNoteRevisions() call utcDateLastEdited: note.utcDateModified, utcDateCreated: dateUtils.utcNowDateTime(), utcDateModified: dateUtils.utcNowDateTime(), dateLastEdited: note.dateModified, dateCreated: dateUtils.localNowDateTime() }).save(); noteRevision.setContent(note.getContent()); } } function updateNote(noteId, noteUpdates) { const note = repository.getNote(noteId); if (!note.isContentAvailable) { throw new Error(`Note ${noteId} is not available for change!`); } saveNoteRevision(note); // if protected status changed, then we need to encrypt/decrypt the content anyway if (['file', 'image'].includes(note.type) && note.isProtected !== noteUpdates.isProtected) { noteUpdates.content = note.getContent(); } const noteTitleChanged = note.title !== noteUpdates.title; note.title = noteUpdates.title; note.isProtected = noteUpdates.isProtected; note.save(); if (noteUpdates.content !== undefined && noteUpdates.content !== null) { noteUpdates.content = saveLinks(note, noteUpdates.content); note.setContent(noteUpdates.content); } if (noteTitleChanged) { triggerNoteTitleChanged(note); } noteRevisionService.protectNoteRevisions(note); return { dateModified: note.dateModified, utcDateModified: note.utcDateModified }; } /** * @param {Branch} branch * @param {string} deleteId * @param {TaskContext} taskContext * * @return {boolean} - true if note has been deleted, false otherwise */ function deleteBranch(branch, deleteId, taskContext) { taskContext.increaseProgressCount(); if (!branch || branch.isDeleted) { return false; } if (branch.branchId === 'root' || branch.noteId === 'root' || branch.noteId === hoistedNoteService.getHoistedNoteId()) { throw new Error("Can't delete root branch/note"); } branch.isDeleted = true; branch.deleteId = deleteId; branch.save(); const note = branch.getNote(); const notDeletedBranches = note.getBranches(); if (notDeletedBranches.length === 0) { for (const childBranch of note.getChildBranches()) { deleteBranch(childBranch, deleteId, taskContext); } // first delete children and then parent - this will show up better in recent changes note.isDeleted = true; note.deleteId = deleteId; note.save(); log.info("Deleting note " + note.noteId); for (const attribute of note.getOwnedAttributes()) { attribute.isDeleted = true; attribute.deleteId = deleteId; attribute.save(); } for (const relation of note.getTargetRelations()) { relation.isDeleted = true; relation.deleteId = deleteId; relation.save(); } return true; } else { return false; } } /** * @param {Note} note * @param {string} deleteId * @param {TaskContext} taskContext */ function undeleteNote(note, deleteId, taskContext) { const undeletedParentBranches = getUndeletedParentBranches(note.noteId, deleteId); if (undeletedParentBranches.length === 0) { // cannot undelete if there's no undeleted parent return; } for (const parentBranch of undeletedParentBranches) { undeleteBranch(parentBranch, deleteId, taskContext); } } /** * @param {Branch} branch * @param {string} deleteId * @param {TaskContext} taskContext */ function undeleteBranch(branch, deleteId, taskContext) { if (!branch.isDeleted) { return; } const note = branch.getNote(); if (note.isDeleted && note.deleteId !== deleteId) { return; } branch.isDeleted = false; branch.save(); taskContext.increaseProgressCount(); if (note.isDeleted && note.deleteId === deleteId) { note.isDeleted = false; note.save(); const attrs = repository.getEntities(` SELECT * FROM attributes WHERE isDeleted = 1 AND deleteId = ? AND (noteId = ? OR (type = 'relation' AND value = ?))`, [deleteId, note.noteId, note.noteId]); for (const attr of attrs) { attr.isDeleted = false; attr.save(); } const childBranches = repository.getEntities(` SELECT branches.* FROM branches WHERE branches.isDeleted = 1 AND branches.deleteId = ? AND branches.parentNoteId = ?`, [deleteId, note.noteId]); for (const childBranch of childBranches) { undeleteBranch(childBranch, deleteId, taskContext); } } } /** * @return return deleted branches of an undeleted parent note */ function getUndeletedParentBranches(noteId, deleteId) { return repository.getEntities(` SELECT branches.* FROM branches JOIN notes AS parentNote ON parentNote.noteId = branches.parentNoteId WHERE branches.noteId = ? AND branches.isDeleted = 1 AND branches.deleteId = ? AND parentNote.isDeleted = 0`, [noteId, deleteId]); } function scanForLinks(note) { if (!note || !['text', 'relation-map'].includes(note.type)) { return; } try { const content = note.getContent(); const newContent = saveLinks(note, content); if (content !== newContent) { note.setContent(newContent); } } catch (e) { log.error(`Could not scan for links note ${note.noteId}: ${e.message} ${e.stack}`); } } function eraseDeletedNotes() { const eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds'); const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000); const noteIdsToErase = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND isErased = 0 AND notes.utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]); if (noteIdsToErase.length === 0) { return; } // it's better to not use repository for this because: // - it would complain about saving protected notes out of protected session // - we don't want these changes to be synced (since they are done on all instances anyway) // - we don't want change the hash since this erasing happens on each instance separately // and changing the hash would fire up the sync errors temporarily sql.executeMany(` UPDATE notes SET title = '[deleted]', isProtected = 0, isErased = 1 WHERE noteId IN (???)`, noteIdsToErase); sql.executeMany(` UPDATE note_contents SET content = NULL WHERE noteId IN (???)`, noteIdsToErase); // deleting first contents since the WHERE relies on isErased = 0 sql.executeMany(` UPDATE note_revision_contents SET content = NULL WHERE noteRevisionId IN (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN (???))`, noteIdsToErase); sql.executeMany(` UPDATE note_revisions SET isErased = 1, title = NULL WHERE isErased = 0 AND noteId IN (???)`, noteIdsToErase); sql.executeMany(` UPDATE attributes SET name = 'deleted', value = '' WHERE noteId IN (???)`, noteIdsToErase); log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`); } function duplicateNote(noteId, parentNoteId) { const origNote = repository.getNote(noteId); if (origNote.isProtected && !protectedSessionService.isProtectedSessionAvailable()) { throw new Error(`Cannot duplicate note=${origNote.noteId} because it is protected and protected session is not available`); } // might be null if orig note is not in the target parentNoteId const origBranch = (origNote.getBranches()).find(branch => branch.parentNoteId === parentNoteId); const newNote = new Note(origNote); newNote.noteId = undefined; // force creation of new note newNote.title += " (dup)"; newNote.save(); newNote.setContent(origNote.getContent()); const newBranch = new Branch({ noteId: newNote.noteId, parentNoteId: parentNoteId, // here increasing just by 1 to make sure it's directly after original notePosition: origBranch ? origBranch.notePosition + 1 : null }).save(); for (const attribute of origNote.getOwnedAttributes()) { const attr = new Attribute(attribute); attr.attributeId = undefined; // force creation of new attribute attr.noteId = newNote.noteId; attr.save(); } return { note: newNote, branch: newBranch }; } sqlInit.dbReady.then(() => { // first cleanup kickoff 5 minutes after startup setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000); setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000); }); module.exports = { createNewNote, createNewNoteWithTarget, updateNote, deleteBranch, undeleteNote, protectNoteRecursively, scanForLinks, duplicateNote, getUndeletedParentBranches, triggerNoteTitleChanged };