Merge branch 'master' into ckeditor

migrations/0048__add_note_tree_id_to_recent_notes.sql

@@ -0,0 +1,10 @@
+DROP TABLE recent_notes;
+
+CREATE TABLE `recent_notes` (
+  'note_tree_id'TEXT NOT NULL PRIMARY KEY,
+  `note_path` TEXT NOT NULL,
+  `date_accessed` INTEGER NOT NULL ,
+  is_deleted INT
+);
+
+DELETE FROM sync WHERE entity_name = 'recent_notes';

package-lock.json

@@ -4264,6 +4264,14 @@
       "resolved": "https://registry.npmjs.org/hsts/-/hsts-2.1.0.tgz",
       "integrity": "sha512-zXhh/DqgrTXJ7erTN6Fh5k/xjMhDGXCqdYN3wvxUvGUQvnxcFfUd8E+6vLg/nk3ss1TYMb+DhRl25fYABioTvA=="
     },
+    "html": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/html/-/html-1.0.0.tgz",
+      "integrity": "sha1-pUT6nqVJK/s6LMqCEKEL57WvH2E=",
+      "requires": {
+        "concat-stream": "1.6.0"
+      }
+    },
     "html-comment-regex": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.1.tgz",

package.json

@@ -25,9 +25,11 @@
     "express-session": "^1.15.6",
     "fs-extra": "^4.0.2",
     "helmet": "^3.9.0",
+    "html": "^1.0.0",
     "ini": "^1.3.4",
     "request": "^2.83.0",
     "request-promise": "^4.2.2",
+    "rimraf": "^2.6.2",
     "scrypt": "^6.0.3",
     "serve-favicon": "~2.4.5",
     "session-file-store": "^1.1.2",

public/javascripts/dialogs/recent_changes.js

@@ -33,9 +33,18 @@ const recentChanges = (function() {
                     .attr('note-path', change.note_id)
                     .attr('note-history-id', change.note_history_id);
 
+                let noteLink;
+
+                if (change.current_is_deleted) {
+                    noteLink = change.current_note_title;
+                }
+                else {
+                    noteLink = link.createNoteLink(change.note_id, change.note_title);
+                }
+
                 changesListEl.append($('<li>')
                     .append(formattedTime + ' - ')
-                    .append(link.createNoteLink(change.note_id))
+                    .append(noteLink)
                     .append(' (').append(revLink).append(')'));
             }
 

public/javascripts/dialogs/recent_notes.js

@@ -8,30 +8,26 @@ const recentNotes = (function() {
     const addCurrentAsChildEl = $("#recent-notes-add-current-as-child");
     const addRecentAsChildEl = $("#recent-notes-add-recent-as-child");
     const noteDetailEl = $('#note-detail');
+    // list of recent note paths
     let list = [];
 
-    server.get('recent-notes').then(result => {
-        list = result.map(r => r.note_tree_id);
-    });
+    async function reload() {
+        const result = await server.get('recent-notes');
 
-    function addRecentNote(notePath) {
+        list = result.map(r => r.note_path);
+    }
+
+    function addRecentNote(noteTreeId, notePath) {
         setTimeout(async () => {
             // we include the note into recent list only if the user stayed on the note at least 5 seconds
             if (notePath && notePath === noteTree.getCurrentNotePath()) {
-                const result = await server.put('recent-notes/' + encodeURIComponent(notePath));
+                const result = await server.put('recent-notes/' + noteTreeId + '/' + encodeURIComponent(notePath));
 
                 list = result.map(r => r.note_path);
             }
         }, 1500);
     }
 
-    // FIXME: this should be probably just refresh upon deletion, not explicit delete
-    async function removeRecentNote(notePathIdToRemove) {
-        const result = await server.remove('recent-notes/' + encodeURIComponent(notePathIdToRemove));
-
-        list = result.map(r => r.note_path);
-    }
-
     function showDialog() {
         glob.activeDialog = dialogEl;
 
@@ -133,6 +129,8 @@ const recentNotes = (function() {
         e.preventDefault();
     });
 
+    reload();
+
     $(document).bind('keydown', 'alt+q', showDialog);
 
     selectBoxEl.dblclick(e => {
@@ -147,6 +145,6 @@ const recentNotes = (function() {
     return {
         showDialog,
         addRecentNote,
-        removeRecentNote
+        reload
     };
 })();

public/javascripts/messaging.js

@@ -33,15 +33,23 @@ const messaging = (function() {
                 noteEditor.reload();
             }
 
+            if (data.recent_notes) {
+                console.log("Reloading recent notes because of background changes");
+
+                recentNotes.reload();
+            }
+
             const changesToPushCountEl = $("#changesToPushCount");
             changesToPushCountEl.html(message.changesToPushCount);
         }
     }
 
     function connectWebSocket() {
+        const protocol = document.location.protocol === 'https:' ? 'wss' : 'ws';
+
         // use wss for secure messaging
-        ws = new WebSocket("ws://" + location.host);
-        ws.onopen = function (event) {};
+        ws = new WebSocket(protocol + "://" + location.host);
+        ws.onopen = event => console.log("Connected to server with WebSocket");
         ws.onmessage = messageHandler;
         ws.onclose = function(){
             // Try to reconnect in 5 seconds

public/javascripts/note_tree.js

@@ -184,7 +184,32 @@ const noteTree = (function() {
     }
 
     async function activateNode(notePath) {
+        const runPath = getRunPath(notePath);
+        const noteId = treeUtils.getNoteIdFromNotePath(notePath);
+
+        let parentNoteId = 'root';
+
+        for (const childNoteId of runPath) {
+            const node = getNodesByNoteId(childNoteId).find(node => node.data.note_pid === parentNoteId);
+
+            if (childNoteId === noteId) {
+                await node.setActive();
+            }
+            else {
+                await node.setExpanded();
+            }
+
+            parentNoteId = childNoteId;
+        }
+    }
+
+    /**
+     * Accepts notePath and tries to resolve it. Part of the path might not be valid because of note moving (which causes
+     * path change) or other corruption, in that case this will try to get some other valid path to the correct note.
+     */
+    function getRunPath(notePath) {
         const path = notePath.split("/").reverse();
+        path.push('root');
 
         const effectivePath = [];
         let childNoteId = null;
@@ -224,27 +249,16 @@ const noteTree = (function() {
                 }
             }
 
+            if (parentNoteId === 'root') {
+                break;
+            }
+            else {
                 effectivePath.push(parentNoteId);
                 childNoteId = parentNoteId;
             }
-
-        const noteId = treeUtils.getNoteIdFromNotePath(notePath);
-
-        const runPath = effectivePath.reverse();
-        let parentNoteId = 'root';
-
-        for (const childNoteId of runPath) {
-            const node = getNodesByNoteId(childNoteId).find(node => node.data.note_pid === parentNoteId);
-
-            if (childNoteId === noteId) {
-                await node.setActive();
-            }
-            else {
-                await node.setExpanded();
         }
 
-            parentNoteId = childNoteId;
-        }
+        return effectivePath.reverse();
     }
 
     function showParentList(noteId, node) {
@@ -319,10 +333,11 @@ const noteTree = (function() {
 
     function setCurrentNotePathToHash(node) {
         const currentNotePath = treeUtils.getNotePath(node);
+        const currentNoteTreeId = node.data.note_tree_id;
 
         document.location.hash = currentNotePath;
 
-        recentNotes.addRecentNote(currentNotePath);
+        recentNotes.addRecentNote(currentNoteTreeId, currentNotePath);
     }
 
     function initFancyTree(noteTree) {
@@ -343,13 +358,13 @@ const noteTree = (function() {
                 const beforeNode = node.getPrevSibling();
 
                 if (beforeNode !== null) {
-                    treeChanges.moveBeforeNode(node, beforeNode, false);
+                    treeChanges.moveBeforeNode(node, beforeNode);
                 }
             },
             "shift+down": node => {
                 let afterNode = node.getNextSibling();
                 if (afterNode !== null) {
-                    treeChanges.moveAfterNode(node, afterNode, false);
+                    treeChanges.moveAfterNode(node, afterNode);
                 }
             },
             "shift+left": node => {
@@ -625,6 +640,6 @@ const noteTree = (function() {
         createNewTopLevelNote,
         createNote,
         setPrefix,
-
+        getNotePathTitle
     };
 })();

public/javascripts/tree_changes.js

@@ -1,29 +1,21 @@
 "use strict";
 
 const treeChanges = (function() {
-    async function moveBeforeNode(node, beforeNode, changeInPath = true) {
+    async function moveBeforeNode(node, beforeNode) {
         await server.put('notes/' + node.data.note_tree_id + '/move-before/' + beforeNode.data.note_tree_id);
 
         node.moveTo(beforeNode, 'before');
 
-        if (changeInPath) {
-            recentNotes.removeRecentNote(noteTree.getCurrentNotePath());
-
         noteTree.setCurrentNotePathToHash(node);
     }
-    }
 
-    async function moveAfterNode(node, afterNode, changeInPath = true) {
+    async function moveAfterNode(node, afterNode) {
         await server.put('notes/' + node.data.note_tree_id + '/move-after/' + afterNode.data.note_tree_id);
 
         node.moveTo(afterNode, 'after');
 
-        if (changeInPath) {
-            recentNotes.removeRecentNote(noteTree.getCurrentNotePath());
-
         noteTree.setCurrentNotePathToHash(node);
     }
-    }
 
     // beware that first arg is noteId and second is noteTreeId!
     async function cloneNoteAfter(noteId, afterNoteTreeId) {
@@ -47,8 +39,6 @@ const treeChanges = (function() {
         toNode.folder = true;
         toNode.renderTitle();
 
-        recentNotes.removeRecentNote(noteTree.getCurrentNotePath());
-
         noteTree.setCurrentNotePathToHash(node);
     }
 
@@ -75,8 +65,6 @@ const treeChanges = (function() {
             node.getParent().renderTitle();
         }
 
-        recentNotes.removeRecentNote(noteTree.getCurrentNotePath());
-
         let next = node.getNextSibling();
         if (!next) {
             next = node.getParent();

routes/api/export.js

@@ -0,0 +1,55 @@
+"use strict";
+
+const express = require('express');
+const router = express.Router();
+const rimraf = require('rimraf');
+const fs = require('fs');
+const sql = require('../../services/sql');
+const data_dir = require('../../services/data_dir');
+const html = require('html');
+
+router.get('/:noteId/to/:directory', async (req, res, next) => {
+    const noteId = req.params.noteId;
+    const directory = req.params.directory.replace(/[^0-9a-zA-Z_-]/gi, '');
+
+    if (!fs.existsSync(data_dir.EXPORT_DIR)) {
+        fs.mkdirSync(data_dir.EXPORT_DIR);
+    }
+
+    const completeExportDir = data_dir.EXPORT_DIR + '/' + directory;
+
+    if (fs.existsSync(completeExportDir)) {
+        rimraf.sync(completeExportDir);
+    }
+
+    fs.mkdirSync(completeExportDir);
+
+    const noteTreeId = await sql.getSingleValue('SELECT note_tree_id FROM notes_tree WHERE note_id = ?', [noteId]);
+
+    await exportNote(noteTreeId, completeExportDir);
+
+    res.send({});
+});
+
+async function exportNote(noteTreeId, dir) {
+    const noteTree = await sql.getSingleResult("SELECT * FROM notes_tree WHERE note_tree_id = ?", [noteTreeId]);
+    const note = await sql.getSingleResult("SELECT * FROM notes WHERE note_id = ?", [noteTree.note_id]);
+
+    const pos = (noteTree.note_pos + '').padStart(4, '0');
+
+    fs.writeFileSync(dir + '/' + pos + '-' + note.note_title + '.html', html.prettyPrint(note.note_text, {indent_size: 2}));
+
+    const children = await sql.getResults("SELECT * FROM notes_tree WHERE note_pid = ? AND is_deleted = 0", [note.note_id]);
+
+    if (children.length > 0) {
+        const childrenDir = dir + '/' + pos + '-' + note.note_title;
+
+        fs.mkdirSync(childrenDir);
+
+        for (const child of children) {
+            await exportNote(child.note_tree_id, childrenDir);
+        }
+    }
+}
+
+module.exports = router;

routes/api/import.js

@@ -0,0 +1,102 @@
+"use strict";
+
+const express = require('express');
+const router = express.Router();
+const rimraf = require('rimraf');
+const fs = require('fs');
+const sql = require('../../services/sql');
+const data_dir = require('../../services/data_dir');
+const utils = require('../../services/utils');
+const sync_table = require('../../services/sync_table');
+
+router.get('/:directory/to/:parentNoteId', async (req, res, next) => {
+    const directory = req.params.directory.replace(/[^0-9a-zA-Z_-]/gi, '');
+    const parentNoteId = req.params.parentNoteId;
+
+    const dir = data_dir.EXPORT_DIR + '/' + directory;
+
+    await sql.doInTransaction(async () => await importNotes(dir, parentNoteId));
+
+    res.send({});
+});
+
+async function importNotes(dir, parentNoteId) {
+    const parent = await sql.getSingleResult("SELECT * FROM notes WHERE note_id = ?", [parentNoteId]);
+
+    if (!parent) {
+        return;
+    }
+
+    const fileList = fs.readdirSync(dir);
+
+    for (const file of fileList) {
+        const path = dir + '/' + file;
+
+        if (fs.lstatSync(path).isDirectory()) {
+            continue;
+        }
+
+        if (!file.endsWith('.html')) {
+            continue;
+        }
+
+        const fileNameWithoutExt = file.substr(0, file.length - 5);
+
+        let noteTitle;
+        let notePos;
+
+        const match = fileNameWithoutExt.match(/^([0-9]{4})-(.*)$/);
+        if (match) {
+            notePos = parseInt(match[1]);
+            noteTitle = match[2];
+        }
+        else {
+            let maxPos = await sql.getSingleValue("SELECT MAX(note_pos) FROM notes_tree WHERE note_pid = ? AND is_deleted = 0", [parentNoteId]);
+            if (maxPos) {
+                notePos = maxPos + 1;
+            }
+            else {
+                notePos = 0;
+            }
+
+            noteTitle = fileNameWithoutExt;
+        }
+
+        const noteText = fs.readFileSync(path, "utf8");
+
+        const noteId = utils.newNoteId();
+        const noteTreeId = utils.newNoteHistoryId();
+
+        await sql.insert('notes_tree', {
+            note_tree_id: noteTreeId,
+            note_id: noteId,
+            note_pid: parentNoteId,
+            note_pos: notePos,
+            is_expanded: 0,
+            is_deleted: 0,
+            date_modified: utils.nowTimestamp()
+        });
+
+        await sync_table.addNoteTreeSync(noteTreeId);
+
+        await sql.insert('notes', {
+            note_id: noteId,
+            note_title: noteTitle,
+            note_text: noteText,
+            is_deleted: 0,
+            is_protected: 0,
+            date_created: utils.nowTimestamp(),
+            date_modified: utils.nowTimestamp()
+        });
+
+        await sync_table.addNoteSync(noteId);
+
+        const noteDir = dir + '/' + fileNameWithoutExt;
+
+        if (fs.existsSync(noteDir) && fs.lstatSync(noteDir).isDirectory()) {
+            await importNotes(noteDir, noteId);
+        }
+    }
+}
+
+module.exports = router;

routes/api/recent_changes.js

@@ -6,7 +6,17 @@ const sql = require('../../services/sql');
 const auth = require('../../services/auth');
 
 router.get('/', auth.checkApiAuth, async (req, res, next) => {
-    const recentChanges = await sql.getResults("SELECT * FROM notes_history order by date_modified_to desc limit 1000");
+    const recentChanges = await sql.getResults(
+        `SELECT 
+            notes.is_deleted AS current_is_deleted,
+            notes.note_title AS current_note_title,
+            notes_history.*
+        FROM 
+            notes_history
+            JOIN notes USING(note_id)
+        ORDER BY 
+            date_modified_to DESC 
+        LIMIT 1000`);
 
     res.send(recentChanges);
 });

routes/api/recent_notes.js

@@ -12,17 +12,19 @@ router.get('', auth.checkApiAuth, async (req, res, next) => {
     res.send(await getRecentNotes());
 });
 
-router.put('/:notePath', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteTreeId/:notePath', auth.checkApiAuth, async (req, res, next) => {
+    const noteTreeId = req.params.noteTreeId;
     const notePath = req.params.notePath;
 
     await sql.doInTransaction(async () => {
         await sql.replace('recent_notes', {
+            note_tree_id: noteTreeId,
             note_path: notePath,
             date_accessed: utils.nowTimestamp(),
             is_deleted: 0
         });
 
-        await sync_table.addRecentNoteSync(notePath);
+        await sync_table.addRecentNoteSync(noteTreeId);
 
         await options.setOption('start_note_tree_id', notePath);
     });
@@ -30,18 +32,6 @@ router.put('/:notePath', auth.checkApiAuth, async (req, res, next) => {
     res.send(await getRecentNotes());
 });
 
-router.delete('/:notePath', auth.checkApiAuth, async (req, res, next) => {
-    const notePath = req.params.notePath;
-
-    await sql.doInTransaction(async () => {
-        await sql.execute('UPDATE recent_notes SET is_deleted = 1 WHERE note_path = ?', [notePath]);
-
-        await sync_table.addRecentNoteSync(notePath);
-    });
-
-    res.send(await getRecentNotes());
-});
-
 async function getRecentNotes() {
     await deleteOld();
 

routes/api/sync.js

@@ -66,10 +66,10 @@ router.get('/notes_reordering/:noteTreeParentId', auth.checkApiAuth, async (req,
     });
 });
 
-router.get('/recent_notes/:notePath', auth.checkApiAuth, async (req, res, next) => {
-    const notePath = req.params.notePath;
+router.get('/recent_notes/:noteTreeId', auth.checkApiAuth, async (req, res, next) => {
+    const noteTreeId = req.params.noteTreeId;
 
-    res.send(await sql.getSingleResult("SELECT * FROM recent_notes WHERE note_path = ?", [notePath]));
+    res.send(await sql.getSingleResult("SELECT * FROM recent_notes WHERE note_tree_id = ?", [noteTreeId]));
 });
 
 router.put('/notes', auth.checkApiAuth, async (req, res, next) => {

routes/routes.js

@@ -17,6 +17,8 @@ const loginApiRoute = require('./api/login');
 const eventLogRoute = require('./api/event_log');
 const recentNotesRoute = require('./api/recent_notes');
 const appInfoRoute = require('./api/app_info');
+const exportRoute = require('./api/export');
+const importRoute = require('./api/import');
 
 function register(app) {
     app.use('/', indexRoute);
@@ -37,6 +39,8 @@ function register(app) {
     app.use('/api/event-log', eventLogRoute);
     app.use('/api/recent-notes', recentNotesRoute);
     app.use('/api/app-info', appInfoRoute);
+    app.use('/api/export', exportRoute);
+    app.use('/api/import', importRoute);
 }
 
 module.exports = {

services/content_hash.js

@@ -22,7 +22,7 @@ async function getContentHash() {
     hash = updateHash(hash, await sql.getResults("SELECT note_history_id, note_id, note_title, note_text, " +
         "date_modified_from, date_modified_to FROM notes_history ORDER BY note_history_id"));
 
-    hash = updateHash(hash, await sql.getResults("SELECT note_path, date_accessed, is_deleted FROM recent_notes " +
+    hash = updateHash(hash, await sql.getResults("SELECT note_tree_id, note_path, date_accessed, is_deleted FROM recent_notes " +
         "ORDER BY note_path"));
 
     const questionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(',');

services/data_dir.js

@@ -12,10 +12,12 @@ if (!fs.existsSync(TRILIUM_DATA_DIR)) {
 const DOCUMENT_PATH = TRILIUM_DATA_DIR + "/document.db";
 const BACKUP_DIR = TRILIUM_DATA_DIR + "/backup";
 const LOG_DIR = TRILIUM_DATA_DIR + "/log";
+const EXPORT_DIR = TRILIUM_DATA_DIR + "/export";
 
 module.exports = {
     TRILIUM_DATA_DIR,
     DOCUMENT_PATH,
     BACKUP_DIR,
-    LOG_DIR
+    LOG_DIR,
+    EXPORT_DIR
 };

services/migration.js

@@ -4,7 +4,7 @@ const options = require('./options');
 const fs = require('fs-extra');
 const log = require('./log');
 
-const APP_DB_VERSION = 47;
+const APP_DB_VERSION = 48;
 const MIGRATIONS_DIR = "migrations";
 
 async function migrate() {

services/sync.js

@@ -224,7 +224,7 @@ async function readAndPushEntity(sync, syncContext) {
         entity = await sql.getSingleResult('SELECT * FROM options WHERE opt_name = ?', [sync.entity_id]);
     }
     else if (sync.entity_name === 'recent_notes') {
-        entity = await sql.getSingleResult('SELECT * FROM recent_notes WHERE note_path = ?', [sync.entity_id]);
+        entity = await sql.getSingleResult('SELECT * FROM recent_notes WHERE note_tree_id = ?', [sync.entity_id]);
     }
     else {
         throw new Error("Unrecognized entity type " + sync.entity_name);

services/sync_update.js

@@ -92,13 +92,13 @@ async function updateOptions(entity, sourceId) {
 }
 
 async function updateRecentNotes(entity, sourceId) {
-    const orig = await sql.getSingleResultOrNull("SELECT * FROM recent_notes WHERE note_path = ?", [entity.note_path]);
+    const orig = await sql.getSingleResultOrNull("SELECT * FROM recent_notes WHERE note_tree_id = ?", [entity.note_tree_id]);
 
     if (orig === null || orig.date_accessed < entity.date_accessed) {
         await sql.doInTransaction(async () => {
             await sql.replace('recent_notes', entity);
 
-            await sync_table.addRecentNoteSync(entity.note_path, sourceId);
+            await sync_table.addRecentNoteSync(entity.note_tree_id, sourceId);
         });
     }
 }