yet another refactoring of working with note's payload/content

This commit is contained in:
zadam 2019-03-26 22:24:04 +01:00
parent 4bdcf32475
commit 29c60581a6
31 changed files with 126 additions and 239 deletions

View File

@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS "note_contents_mig" (
`noteId` TEXT NOT NULL,
`content` TEXT NULL DEFAULT NULL,
`hash` TEXT DEFAULT "" NOT NULL,
`utcDateModified` TEXT NOT NULL,
PRIMARY KEY(`noteId`)
);
INSERT INTO note_contents_mig (noteId, content, hash, utcDateModified)
SELECT noteId, content, hash, utcDateModified FROM note_contents;
DROP TABLE note_contents;
ALTER TABLE note_contents_mig RENAME TO note_contents;

View File

@ -26,7 +26,13 @@ class Entity {
this.hash = this.generateHash();
this.isChanged = origHash !== this.hash;
if (this.forcedChange) {
this.isChanged = true;
delete this.forcedChange;
}
else {
this.isChanged = origHash !== this.hash;
}
}
generateIdIfNecessary() {

View File

@ -1,5 +1,4 @@
const Note = require('../entities/note');
const NoteContent = require('../entities/note_content');
const NoteRevision = require('../entities/note_revision');
const Link = require('../entities/link');
const Branch = require('../entities/branch');
@ -13,7 +12,6 @@ const ENTITY_NAME_TO_ENTITY = {
"attributes": Attribute,
"branches": Branch,
"notes": Note,
"note_contents": NoteContent,
"note_revisions": NoteRevision,
"recent_notes": RecentNote,
"options": Option,
@ -50,9 +48,6 @@ function createEntityFromRow(row) {
else if (row.branchId) {
entity = new Branch(row);
}
else if (row.noteContentId) {
entity = new NoteContent(row);
}
else if (row.noteId) {
entity = new Note(row);
}

View File

@ -2,12 +2,13 @@
const Entity = require('./entity');
const Attribute = require('./attribute');
const NoteContent = require('./note_content');
const protectedSessionService = require('../services/protected_session');
const repository = require('../services/repository');
const sql = require('../services/sql');
const utils = require('../services/utils');
const dateUtils = require('../services/date_utils');
const noteFulltextService = require('../services/note_fulltext');
const syncTableService = require('../services/sync_table');
const LABEL = 'label';
const LABEL_DEFINITION = 'label-definition';
@ -56,37 +57,33 @@ class Note extends Entity {
protectedSessionService.decryptNote(this);
}
else {
// saving ciphertexts in case we do want to update protected note outside of protected session
// (which is allowed)
this.titleCipherText = this.title;
this.title = "[protected]";
}
}
}
/** @returns {Promise<NoteContent>} */
async getNoteContent() {
if (!this.noteContent) {
this.noteContent = await repository.getEntity(`SELECT * FROM note_contents WHERE noteId = ?`, [this.noteId]);
/** @returns {Promise<*>} */
async getContent() {
if (this.content === undefined) {
this.content = await sql.getValue(`SELECT content FROM note_contents WHERE noteId = ?`, [this.noteId]);
if (!this.noteContent) {
throw new Error("Note content not found for noteId=" + this.noteId);
if (this.isProtected) {
if (this.isContentAvailable) {
protectedSessionService.decryptNoteContent(this);
}
else {
this.content = "";
}
}
if (this.isStringNote()) {
this.noteContent.content = this.noteContent.content === null
? "" : this.noteContent.content.toString("UTF-8");
this.content = this.content === null
? ""
: this.content.toString("UTF-8");
}
}
return this.noteContent;
}
/** @returns {Promise<*>} */
async getContent() {
const noteContent = await this.getNoteContent();
return noteContent.content;
return this.content;
}
/** @returns {Promise<*>} */
@ -98,14 +95,31 @@ class Note extends Entity {
/** @returns {Promise} */
async setContent(content) {
if (!this.noteContent) {
// make sure it is loaded
await this.getNoteContent();
this.content = content;
const pojo = {
noteId: this.noteId,
content: content,
utcDateModified: dateUtils.utcNowDateTime(),
hash: utils.hash(this.noteId + "|" + content)
};
if (this.isProtected) {
if (this.isContentAvailable) {
protectedSessionService.encryptNoteContent(pojo);
}
else {
throw new Error(`Cannot update content of noteId=${this.noteId} since we're out of protected session.`);
}
}
this.noteContent.content = content;
await sql.upsert("note_contents", "noteId", pojo);
await this.noteContent.save();
await syncTableService.addNoteContentSync(this.noteId);
this.forcedChange = true;
await this.save();
}
/** @returns {Promise} */
@ -687,14 +701,13 @@ class Note extends Entity {
}
else {
// updating protected note outside of protected session means we will keep original ciphertexts
pojo.title = pojo.titleCipherText;
delete pojo.title;
}
}
delete pojo.isContentAvailable;
delete pojo.__attributeCache;
delete pojo.titleCipherText;
delete pojo.noteContent;
delete pojo.content;
}
async afterSaving() {

View File

@ -1,101 +0,0 @@
"use strict";
const Entity = require('./entity');
const protectedSessionService = require('../services/protected_session');
const repository = require('../services/repository');
const dateUtils = require('../services/date_utils');
const noteFulltextService = require('../services/note_fulltext');
/**
* This represents a Note which is a central object in the Trilium Notes project.
*
* @property {string} noteContentId - primary key
* @property {string} noteId - reference to owning note
* @property {boolean} isProtected - true if note content is protected
* @property {blob} content - note content - e.g. HTML text for text notes, file payload for files
* @property {string} utcDateCreated
* @property {string} utcDateModified
*
* @extends Entity
*/
class NoteContent extends Entity {
static get entityName() {
return "note_contents";
}
static get primaryKeyName() {
return "noteContentId";
}
static get hashedProperties() {
return ["noteContentId", "noteId", "isProtected", "content"];
}
/**
* @param row - object containing database row from "note_contents" table
*/
constructor(row) {
super(row);
this.isProtected = !!this.isProtected;
/* true if content (meaning any kind of potentially encrypted content) is either not encrypted
* or encrypted, but with available protected session (so effectively decrypted) */
this.isContentAvailable = true;
// check if there's noteContentId, otherwise this is a new entity which wasn't encrypted yet
if (this.isProtected && this.noteContentId) {
this.isContentAvailable = protectedSessionService.isProtectedSessionAvailable();
if (this.isContentAvailable) {
protectedSessionService.decryptNoteContent(this);
}
else {
// saving ciphertexts in case we do want to update protected note outside of protected session
// (which is allowed)
this.contentCipherText = this.content;
this.content = "";
}
}
}
/**
* @returns {Promise<Note>}
*/
async getNote() {
return await repository.getNote(this.noteId);
}
beforeSaving() {
if (!this.utcDateCreated) {
this.utcDateCreated = dateUtils.utcNowDateTime();
}
super.beforeSaving();
if (this.isChanged) {
this.utcDateModified = dateUtils.utcNowDateTime();
}
}
// cannot be static!
updatePojo(pojo) {
if (pojo.isProtected) {
if (this.isContentAvailable) {
protectedSessionService.encryptNoteContent(pojo);
}
else {
// updating protected note outside of protected session means we will keep original ciphertext
pojo.content = pojo.contentCipherText;
}
}
delete pojo.isContentAvailable;
delete pojo.contentCipherText;
}
async afterSaving() {
noteFulltextService.triggerNoteFulltextUpdate(this.noteId);
}
}
module.exports = NoteContent;

View File

@ -8,7 +8,7 @@ function showDialog() {
$dialog.modal();
const noteText = noteDetailService.getActiveNote().noteContent.content;
const noteText = noteDetailService.getActiveNote().content;
$noteSource.text(formatHtml(noteText));
}

View File

@ -8,7 +8,7 @@ class NoteFull extends NoteShort {
super(treeCache, row);
/** @param {string} */
this.noteContent = row.noteContent;
this.content = row.content;
/** @param {string} */
this.utcDateCreated = row.utcDateCreated;

View File

@ -117,10 +117,7 @@ async function saveNote() {
}
note.title = $noteTitle.val();
if (note.noteContent != null) { // might be null for file/image
note.noteContent.content = getActiveNoteContent(note);
}
note.content = getActiveNoteContent(note);
// it's important to set the flag back to false immediatelly after retrieving title and content
// otherwise we might overwrite another change (especially async code)

View File

@ -49,7 +49,7 @@ async function show() {
// this needs to happen after the element is shown, otherwise the editor won't be refreshed
// CodeMirror breaks pretty badly on null so even though it shouldn't happen (guarded by consistency check)
// we provide fallback
codeEditor.setValue(activeNote.noteContent.content || "");
codeEditor.setValue(activeNote.content || "");
const info = CodeMirror.findModeByMIME(activeNote.mime);

View File

@ -27,9 +27,9 @@ async function show() {
$fileSize.text((attributeMap.fileSize || "?") + " bytes");
$fileType.text(activeNote.mime);
if (activeNote.noteContent && activeNote.noteContent.content) {
if (activeNote.content) {
$previewRow.show();
$previewContent.text(activeNote.noteContent.content);
$previewContent.text(activeNote.content);
}
else {
$previewRow.hide();

View File

@ -92,9 +92,9 @@ function loadMapData() {
}
};
if (activeNote.noteContent.content) {
if (activeNote.content) {
try {
mapData = JSON.parse(activeNote.noteContent.content);
mapData = JSON.parse(activeNote.content);
} catch (e) {
console.log("Could not parse content: ", e);
}

View File

@ -10,7 +10,7 @@ function show() {
$component.show();
try {
const json = JSON.parse(noteDetailService.getActiveNote().noteContent.content);
const json = JSON.parse(noteDetailService.getActiveNote().content);
$searchString.val(json.searchString);
}

View File

@ -31,7 +31,7 @@ async function show() {
$component.show();
textEditor.setData(noteDetailService.getActiveNote().noteContent.content);
textEditor.setData(noteDetailService.getActiveNote().content);
}
function getContent() {

View File

@ -113,11 +113,11 @@ async function renderTooltip(note, attributes) {
if (note.type === 'text') {
// surround with <div> for a case when note's content is pure text (e.g. "[protected]") which
// then fails the jquery non-empty text test
content += '<div>' + note.noteContent.content + '</div>';
content += '<div>' + note.content + '</div>';
}
else if (note.type === 'code') {
content += $("<pre>")
.text(note.noteContent.content)
.text(note.content)
.prop('outerHTML');
}
else if (note.type === 'image') {

View File

@ -51,7 +51,7 @@ async function downloadNoteFile(noteId, res) {
res.setHeader('Content-Disposition', utils.getContentDisposition(fileName));
res.setHeader('Content-Type', note.mime);
res.send((await note.getNoteContent()).content);
res.send(await note.getContent());
}
async function downloadFile(req, res) {

View File

@ -13,10 +13,10 @@ async function getNote(req) {
}
if (note.isStringNote()) {
const noteContent = await note.getNoteContent();
await note.getContent();
if (note.type === 'file') {
noteContent.content = noteContent.content.substr(0, 10000);
note.content = note.content.substr(0, 10000);
}
}

View File

@ -14,11 +14,11 @@ async function searchNotes(req) {
}
async function saveSearchToNote(req) {
const noteContent = {
const content = {
searchString: req.params.searchString
};
const {note} = await noteService.createNote('root', req.params.searchString, noteContent, {
const {note} = await noteService.createNote('root', req.params.searchString, content, {
json: true,
type: 'search',
mime: "application/json"

View File

@ -4,8 +4,8 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 129;
const SYNC_VERSION = 7;
const APP_DB_VERSION = 130;
const SYNC_VERSION = 8;
module.exports = {
appVersion: packageJson.version,

View File

@ -8,17 +8,16 @@ const messagingService = require('./messaging');
const ApiToken = require('../entities/api_token');
const Branch = require('../entities/branch');
const Note = require('../entities/note');
const NoteContent = require('../entities/note_content');
const Attribute = require('../entities/attribute');
const NoteRevision = require('../entities/note_revision');
const RecentNote = require('../entities/recent_note');
const Option = require('../entities/option');
const Link = require('../entities/link');
async function getHash(entityConstructor, whereBranch) {
async function getHash(tableName, primaryKeyName, whereBranch) {
// subselect is necessary to have correct ordering in GROUP_CONCAT
const query = `SELECT GROUP_CONCAT(hash) FROM (SELECT hash FROM ${entityConstructor.entityName} `
+ (whereBranch ? `WHERE ${whereBranch} ` : '') + `ORDER BY ${entityConstructor.primaryKeyName})`;
const query = `SELECT GROUP_CONCAT(hash) FROM (SELECT hash FROM ${tableName} `
+ (whereBranch ? `WHERE ${whereBranch} ` : '') + `ORDER BY ${primaryKeyName})`;
let contentToHash = await sql.getValue(query);
@ -33,15 +32,15 @@ async function getHashes() {
const startTime = new Date();
const hashes = {
notes: await getHash(Note),
note_contents: await getHash(NoteContent),
branches: await getHash(Branch),
note_revisions: await getHash(NoteRevision),
recent_notes: await getHash(RecentNote),
options: await getHash(Option, "isSynced = 1"),
attributes: await getHash(Attribute),
api_tokens: await getHash(ApiToken),
links: await getHash(Link)
notes: await getHash(Note.entityName, Note.primaryKeyName),
note_contents: await getHash("note_contents", "noteId"),
branches: await getHash(Branch.entityName, Branch.primaryKeyName),
note_revisions: await getHash(NoteRevision.entityName, NoteRevision.primaryKeyName),
recent_notes: await getHash(RecentNote.entityName, RecentNote.primaryKeyName),
options: await getHash(Option.entityName, Option.primaryKeyName, "isSynced = 1"),
attributes: await getHash(Attribute.entityName, Attribute.primaryKeyName),
api_tokens: await getHash(ApiToken.entityName, ApiToken.primaryKeyName),
links: await getHash(Link.entityName, Link.primaryKeyName)
};
const elapseTimeMs = Date.now() - startTime.getTime();

View File

@ -18,32 +18,32 @@ async function exportSingleNote(exportContext, branch, format, res) {
let payload, extension, mime;
const noteContent = await note.getNoteContent();
let content = await note.getContent();
if (note.type === 'text') {
if (format === 'html') {
if (!noteContent.content.toLowerCase().includes("<html")) {
noteContent.content = '<html><head><meta charset="utf-8"></head><body>' + noteContent.content + '</body></html>';
if (!content.toLowerCase().includes("<html")) {
content = '<html><head><meta charset="utf-8"></head><body>' + content + '</body></html>';
}
payload = html.prettyPrint(noteContent.content, {indent_size: 2});
payload = html.prettyPrint(content, {indent_size: 2});
extension = 'html';
mime = 'text/html';
}
else if (format === 'markdown') {
const turndownService = new TurndownService();
payload = turndownService.turndown(noteContent.content);
payload = turndownService.turndown(content);
extension = 'md';
mime = 'text/x-markdown'
}
}
else if (note.type === 'code') {
payload = noteContent.content;
payload = content;
extension = mimeTypes.extension(note.mime) || 'code';
mime = note.mime;
}
else if (note.type === 'relation-map' || note.type === 'search') {
payload = noteContent.content;
payload = content;
extension = 'json';
mime = 'application/json';
}

View File

@ -223,7 +223,7 @@ async function importEnex(importContext, file, parentNote) {
importContext.increaseProgressCount();
const noteContent = await noteEntity.getNoteContent();
let noteContent = await noteEntity.getContent();
for (const resource of resources) {
const hash = utils.md5(resource.content);
@ -248,7 +248,7 @@ async function importEnex(importContext, file, parentNote) {
const resourceLink = `<a href="#root/${resourceNote.noteId}">${utils.escapeHtml(resource.title)}</a>`;
noteContent.content = noteContent.content.replace(mediaRegex, resourceLink);
noteContent = noteContent.replace(mediaRegex, resourceLink);
};
if (["image/jpeg", "image/png", "image/gif"].includes(resource.mime)) {
@ -259,12 +259,12 @@ async function importEnex(importContext, file, parentNote) {
const imageLink = `<img src="${url}">`;
noteContent.content = noteContent.content.replace(mediaRegex, imageLink);
noteContent = noteContent.replace(mediaRegex, imageLink);
if (!noteContent.content.includes(imageLink)) {
if (!noteContent.includes(imageLink)) {
// if there wasn't any match for the reference, we'll add the image anyway
// otherwise image would be removed since no note would include it
noteContent.content += imageLink;
noteContent += imageLink;
}
} catch (e) {
log.error("error when saving image from ENEX file: " + e);
@ -276,7 +276,7 @@ async function importEnex(importContext, file, parentNote) {
}
// save updated content with links to files/images
await noteContent.save();
await noteEntity.setContent(noteContent);
}
saxStream.on("closetag", async tag => {

View File

@ -259,10 +259,7 @@ async function importTar(importContext, fileBuffer, importRootNote) {
let note = await repository.getNote(noteId);
if (note) {
const noteContent = await note.getNoteContent();
noteContent.content = content;
await noteContent.save();
await note.setContent(content);
}
else {
const noteTitle = getNoteTitle(filePath, noteMeta);

View File

@ -14,14 +14,14 @@ async function updateNoteFulltext(note) {
let contentHash = null;
if (['text', 'code'].includes(note.type)) {
const noteContent = await note.getNoteContent();
content = noteContent.content;
content = await note.getContent();
if (note.type === 'text' && note.mime === 'text/html') {
content = html2plaintext(content);
}
contentHash = noteContent.hash;
// FIXME
//contentHash = noteContent.hash;
}
// optimistically try to update first ...

View File

@ -8,7 +8,6 @@ const eventService = require('./events');
const repository = require('./repository');
const cls = require('../services/cls');
const Note = require('../entities/note');
const NoteContent = require('../entities/note_content');
const Link = require('../entities/link');
const NoteRevision = require('../entities/note_revision');
const Branch = require('../entities/branch');
@ -93,10 +92,7 @@ async function createNewNote(parentNoteId, noteData) {
noteData.content = noteData.content || "";
}
note.noteContent = await new NoteContent({
noteId: note.noteId,
content: noteData.content
}).save();
await note.setContent(noteData.content);
const branch = await new Branch({
noteId: note.noteId,
@ -338,16 +334,11 @@ async function updateNote(noteId, noteUpdates) {
note.isProtected = noteUpdates.isProtected;
await note.save();
const noteContent = await note.getNoteContent();
if (!['file', 'image'].includes(note.type)) {
noteUpdates.noteContent.content = await saveLinks(note, noteUpdates.noteContent.content);
noteContent.content = noteUpdates.noteContent.content;
noteUpdates.content = await saveLinks(note, noteUpdates.content);
}
noteContent.isProtected = noteUpdates.isProtected;
await noteContent.save();
await note.setContent(noteUpdates.content);
if (noteTitleChanged) {
await triggerNoteTitleChanged(note);

View File

@ -56,18 +56,14 @@ function decryptNote(note) {
}
}
function decryptNoteContent(noteContent) {
if (!noteContent.isProtected) {
return;
}
function decryptNoteContent(note) {
try {
if (noteContent.content != null) {
noteContent.content = dataEncryptionService.decrypt(getDataKey(), noteContent.content.toString());
if (note.content != null) {
note.content = dataEncryptionService.decrypt(getDataKey(), note.content.toString());
}
}
catch (e) {
e.message = `Cannot decrypt note content for noteContentId=${noteContent.noteContentId}: ` + e.message;
e.message = `Cannot decrypt content for noteId=${note.noteId}: ` + e.message;
throw e;
}
}
@ -98,8 +94,8 @@ function encryptNote(note) {
note.title = dataEncryptionService.encrypt(getDataKey(), note.title);
}
function encryptNoteContent(noteContent) {
noteContent.content = dataEncryptionService.encrypt(getDataKey(), noteContent.content);
function encryptNoteContent(note) {
note.content = dataEncryptionService.encrypt(getDataKey(), note.content);
}
function encryptNoteRevision(revision) {

View File

@ -42,19 +42,6 @@ async function getNote(noteId) {
return await getEntity("SELECT * FROM notes WHERE noteId = ?", [noteId]);
}
/** @returns {Promise<Note|null>} */
async function getNoteWithContent(noteId) {
const note = await getEntity("SELECT * FROM notes WHERE noteId = ?", [noteId]);
await note.getNoteContent();
return note;
}
/** @returns {Promise<NoteContent|null>} */
async function getNoteContent(noteContentId) {
return await getEntity("SELECT * FROM note_contents WHERE noteContentId = ?", [noteContentId]);
}
/** @returns {Promise<Branch|null>} */
async function getBranch(branchId) {
return await getEntity("SELECT * FROM branches WHERE branchId = ?", [branchId]);
@ -138,8 +125,6 @@ module.exports = {
getEntities,
getEntity,
getNote,
getNoteWithContent,
getNoteContent,
getBranch,
getAttribute,
getOption,

View File

@ -58,10 +58,10 @@ async function executeBundle(bundle, apiParams = {}) {
*/
async function executeScript(script, params, startNoteId, currentNoteId, originEntityName, originEntityId) {
const startNote = await repository.getNote(startNoteId);
const currentNote = await repository.getNoteWithContent(currentNoteId);
const currentNote = await repository.getNote(currentNoteId);
const originEntity = await repository.getEntityFromName(originEntityName, originEntityId);
currentNote.noteContent.content = `return await (${script}\r\n)(${getParams(params)})`;
currentNote.content = `return await (${script}\r\n)(${getParams(params)})`;
currentNote.type = 'code';
currentNote.mime = 'application/javascript;env=backend';

View File

@ -78,7 +78,6 @@ async function createInitialDatabase(username, password) {
await sql.executeScript(schema);
const Note = require("../entities/note");
const NoteContent = require("../entities/note_content");
const Branch = require("../entities/branch");
const rootNote = await new Note({
@ -88,10 +87,7 @@ async function createInitialDatabase(username, password) {
mime: 'text/html'
}).save();
const rootNoteContent = await new NoteContent({
noteId: rootNote.noteId,
content: ''
}).save();
await rootNote.setContent('');
await new Branch({
branchId: 'root',

View File

@ -239,7 +239,7 @@ async function syncRequest(syncContext, method, requestPath, body) {
const primaryKeys = {
"notes": "noteId",
"note_contents": "noteContentId",
"note_contents": "noteId",
"branches": "branchId",
"note_revisions": "noteRevisionId",
"recent_notes": "branchId",

View File

@ -8,8 +8,8 @@ async function addNoteSync(noteId, sourceId) {
await addEntitySync("notes", noteId, sourceId)
}
async function addNoteContentSync(noteContentId, sourceId) {
await addEntitySync("note_contents", noteContentId, sourceId)
async function addNoteContentSync(noteId, sourceId) {
await addEntitySync("note_contents", noteId, sourceId)
}
async function addBranchSync(branchId, sourceId) {

View File

@ -77,12 +77,12 @@ async function updateNoteContent(entity, sourceId) {
await sql.transactional(async () => {
await sql.replace("note_contents", entity);
await syncTableService.addNoteContentSync(entity.noteContentId, sourceId);
await syncTableService.addNoteContentSync(entity.noteId, sourceId);
noteFulltextService.triggerNoteFulltextUpdate(entity.noteId);
});
log.info("Update/sync note content " + entity.noteContentId);
log.info("Update/sync note content for noteId=" + entity.noteId);
}
}