gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

initial implementation of note attachments

Browse Source
This commit is contained in:
zadam 2023-01-22 23:36:05 +01:00
parent 6a6ae359b6
commit 339d8a7378
20 changed files with 2447 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/tpl/anonymize-database.sql
View File

@ -4,6 +4,7 @@ UPDATE notes SET title = 'title' WHERE title NOT IN ('root', '_hidden', '_share'
UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL;
UPDATE note_revisions SET title = 'title';
UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
UPDATE note_attachment_contents SET content = 'text' WHERE content IS NOT NULL;
UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'workspaceInbox', 'workspaceSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'shareRoot', 'shareDescription', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteContentChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'inherit', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');
UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN ('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'workspaceInbox', 'workspaceSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'shareRoot', 'shareDescription', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteContentChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'inherit', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');

19
db/migrations/0213__note_attachments.sql Normal file
View File

@ -0,0 +1,19 @@
CREATE TABLE IF NOT EXISTS "note_attachments"
(
noteAttachmentId TEXT not null primary key,
noteId TEXT not null,
name TEXT not null,
mime TEXT not null,
isProtected INT not null DEFAULT 0,
utcDateModified TEXT not null,
isDeleted INT not null,
`deleteId` TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "note_attachment_contents" (`noteAttachmentId` TEXT NOT NULL PRIMARY KEY,
`content` TEXT DEFAULT NULL,
`utcDateModified` TEXT NOT NULL);
CREATE INDEX IDX_note_attachments_name
on note_attachments (name);
CREATE INDEX IDX_note_attachments_noteId
on note_attachments (noteId);

2
db/schema.sql
View File

@ -61,7 +61,7 @@ CREATE TABLE IF NOT EXISTS "note_revisions" (`noteRevisionId` TEXT NOT NULL PRIM
`dateLastEdited` TEXT NOT NULL,
`dateCreated` TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS "note_revision_contents" (`noteRevisionId` TEXT NOT NULL PRIMARY KEY,
`content` TEXT,
`content` TEXT DEFAULT NULL,
`utcDateModified` TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS "options"
(

2385
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
package.json
View File

@ -70,6 +70,7 @@
"node-abi": "3.31.0",
"normalize-strings": "1.1.1",
"open": "8.4.0",
"pdfjs-dist": "2.8.335",
"rand-token": "1.0.1",
"react": "17.0.2",
"react-dom": "17.0.2",
@ -84,6 +85,7 @@
"session-file-store": "1.5.0",
"stream-throttle": "0.1.3",
"striptags": "3.2.0",
"tesseract.js": "^4.0.2",
"tmp": "0.2.1",
"turndown": "7.1.1",
"unescape": "1.0.1",

11
src/becca/becca.js
View File

@ -2,6 +2,7 @@
const sql = require("../services/sql");
const NoteSet = require("../services/search/note_set");
const BNoteRevision = require("./entities/bnote_revision.js");
/**
* Becca is a backend cache of all notes, branches and attributes. There's a similar frontend cache Froca.
@ -121,6 +122,14 @@ class Becca {
return row ? new BNoteRevision(row) : null;
}
/** @returns {BNoteAttachment|null} */
getNoteAttachment(noteAttachmentId) {
const row = sql.getRow("SELECT * FROM note_attachments WHERE noteAttachmentId = ?", [noteAttachmentId]);
const BNoteAttachment = require("./entities/bnote_attachment"); // avoiding circular dependency problems
return row ? new BNoteAttachment(row) : null;
}
/** @returns {BOption|null} */
getOption(name) {
return this.options[name];
@ -143,6 +152,8 @@ class Becca {
if (entityName === 'note_revisions') {
return this.getNoteRevision(entityId);
} else if (entityName === 'note_attachments') {
return this.getNoteAttachment(entityId);
}
const camelCaseEntityName = entityName.toLowerCase().replace(/(_[a-z])/g,

4
src/becca/entities/bbranch.js
View File

@ -198,6 +198,10 @@ class BBranch extends AbstractBeccaEntity {
relation.markAsDeleted(deleteId);
}
for (const noteAttachment of note.getNoteAttachments()) {
noteAttachment.markAsDeleted(deleteId);
}
note.markAsDeleted(deleteId);
return true;

8
src/becca/entities/bnote.js
View File

@ -8,11 +8,12 @@ const dateUtils = require('../../services/date_utils');
const entityChangesService = require('../../services/entity_changes');
const AbstractBeccaEntity = require("./abstract_becca_entity");
const BNoteRevision = require("./bnote_revision");
const BNoteAttachment = require("./bnote_attachment");
const TaskContext = require("../../services/task_context");
const dayjs = require("dayjs");
const utc = require('dayjs/plugin/utc');
const eventService = require("../../services/events");
dayjs.extend(utc)
dayjs.extend(utc);
const LABEL = 'label';
const RELATION = 'relation';
@ -1110,6 +1111,11 @@ class BNote extends AbstractBeccaEntity {
.map(row => new BNoteRevision(row));
}
getNoteAttachments() {
return sql.getRows("SELECT * FROM note_attachments WHERE noteId = ? AND isDeleted = 0", [this.noteId])
.map(row => new BNoteAttachment(row));
}
/**
* @returns {string[][]} - array of notePaths (each represented by array of noteIds constituting the particular note path)
*/

139
src/becca/entities/bnote_attachment.js Normal file
View File

@ -0,0 +1,139 @@
"use strict";
const protectedSessionService = require('../../services/protected_session');
const utils = require('../../services/utils');
const sql = require('../../services/sql');
const dateUtils = require('../../services/date_utils');
const becca = require('../becca');
const entityChangesService = require('../../services/entity_changes');
const AbstractBeccaEntity = require("./abstract_becca_entity");
/**
* NoteAttachment represent data related/attached to the note. Conceptually similar to attributes, but intended for
* larger amounts of data and generally not accessible to the user.
*
* @extends AbstractBeccaEntity
*/
class BNoteAttachment extends AbstractBeccaEntity {
static get entityName() { return "note_attachments"; }
static get primaryKeyName() { return "noteAttachmentId"; }
static get hashedProperties() { return ["noteAttachmentId", "noteId", "name", "content", "utcDateModified"]; }
constructor(row) {
super();
/** @type {string} */
this.noteAttachmentId = row.noteAttachmentId;
/** @type {string} */
this.noteId = row.noteId;
/** @type {string} */
this.name = row.name;
/** @type {string} */
this.mime = row.mime;
/** @type {boolean} */
this.isProtected = !!row.isProtected;
/** @type {string} */
this.utcDateModified = row.utcDateModified;
}
getNote() {
return becca.notes[this.noteId];
}
/** @returns {boolean} true if the note has string content (not binary) */
isStringNote() {
return utils.isStringNote(this.type, this.mime);
}
/** @returns {*} */
getContent(silentNotFoundError = false) {
const res = sql.getRow(`SELECT content FROM note_attachment_contents WHERE noteAttachmentId = ?`, [this.noteAttachmentId]);
if (!res) {
if (silentNotFoundError) {
return undefined;
}
else {
throw new Error(`Cannot find note attachment content for noteAttachmentId=${this.noteAttachmentId}`);
}
}
let content = res.content;
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
content = protectedSessionService.decrypt(content);
}
else {
content = "";
}
}
if (this.isStringNote()) {
return content === null
? ""
: content.toString("UTF-8");
}
else {
return content;
}
}
setContent(content) {
const pojo = {
noteAttachmentId: this.noteAttachmentId,
content: content,
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content);
}
else {
throw new Error(`Cannot update content of noteAttachmentId=${this.noteAttachmentId} since we're out of protected session.`);
}
}
sql.upsert("note_attachment_contents", "noteAttachmentId", pojo);
const hash = utils.hash(`${this.noteAttachmentId}|${pojo.content.toString()}`);
entityChangesService.addEntityChange({
entityName: 'note_attachment_contents',
entityId: this.noteAttachmentId,
hash: hash,
isErased: false,
utcDateChanged: this.getUtcDateChanged(),
isSynced: true
});
}
beforeSaving() {
super.beforeSaving();
this.utcDateModified = dateUtils.utcNowDateTime();
}
getPojo() {
return {
noteAttachmentId: this.noteAttachmentId,
noteId: this.noteId,
mime: this.mime,
isProtected: this.isProtected,
name: this.name,
utcDateModified: this.utcDateModified,
content: this.content,
contentLength: this.contentLength
};
}
getPojoToSave() {
const pojo = this.getPojo();
delete pojo.content; // not getting persisted
return pojo;
}
}
module.exports = BNoteAttachment;

4
src/becca/entities/bnote_revision.js
View File

@ -106,7 +106,7 @@ class BNoteRevision extends AbstractBeccaEntity {
}
}
setContent(content, ignoreMissingProtectedSession = false) {
setContent(content) {
const pojo = {
noteRevisionId: this.noteRevisionId,
content: content,
@ -117,7 +117,7 @@ class BNoteRevision extends AbstractBeccaEntity {
if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content);
}
else if (!ignoreMissingProtectedSession) {
else {
throw new Error(`Cannot update content of noteRevisionId=${this.noteRevisionId} since we're out of protected session.`);
}
}

3
src/becca/entity_constructor.js
View File

@ -1,5 +1,6 @@
const BNote = require('./entities/bnote');
const BNoteRevision = require('./entities/bnote_revision');
const BNoteAttachment = require("./entities/bnote_attachment");
const BBranch = require('./entities/bbranch');
const BAttribute = require('./entities/battribute');
const BRecentNote = require('./entities/brecent_note');
@ -13,6 +14,8 @@ const ENTITY_NAME_TO_ENTITY = {
"note_contents": BNote,
"note_revisions": BNoteRevision,
"note_revision_contents": BNoteRevision,
"note_attachments": BNoteAttachment,
"note_attachment_contents": BNoteAttachment,
"recent_notes": BRecentNote,
"etapi_tokens": BEtapiToken,
"options": BOption

2
src/public/app/services/froca_updater.js
View File

@ -36,7 +36,7 @@ async function processEntityChanges(entityChanges) {
loadResults.addOption(ec.entity.name);
}
else if (ec.entityName === 'etapi_tokens') {
else if (['etapi_tokens', 'note_attachments', 'note_attachment_contents'].includes(ec.entityName)) {
// NOOP
}
else {

16
src/routes/api/files.js
View File

@ -31,6 +31,22 @@ function updateFile(req) {
note.setLabel('originalFileName', file.originalname);
if (note.mime === 'application/pdf') {
const pdfjsLib = require("pdfjs-dist");
(async () =>
{
let doc = await pdfjsLib.getDocument({data: file.buffer}).promise;
let page1 = await doc.getPage(1);
let content = await page1.getTextContent();
let strings = content.items.map(function (item) {
return item.str;
});
console.log(strings);
})();
}
return {
uploaded: true
};

8
src/routes/api/sync.js
View File

@ -114,6 +114,14 @@ function forceNoteSync(req) {
entityChangesService.moveEntityChangeToTop('note_revision_contents', noteRevisionId);
}
for (const noteAttachmentId of sql.getColumn("SELECT noteAttachmentId FROM note_attachments WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE note_attachments SET utcDateModified = ? WHERE noteAttachmentId = ?`, [now, noteAttachmentId]);
entityChangesService.moveEntityChangeToTop('note_attachments', noteAttachmentId);
sql.execute(`UPDATE note_attachment_contents SET utcDateModified = ? WHERE noteAttachmentId = ?`, [now, noteAttachmentId]);
entityChangesService.moveEntityChangeToTop('note_attachment_contents', noteAttachmentId);
}
log.info(`Forcing note sync for ${noteId}`);
// not awaiting for the job to finish (will probably take a long time)

4
src/services/app_info.js
View File

@ -4,8 +4,8 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 212;
const SYNC_VERSION = 29;
const APP_DB_VERSION = 213;
const SYNC_VERSION = 30;
const CLIPPER_PROTOCOL_VERSION = "1.0";
module.exports = {

10
src/services/consistency_checks.js
View File

@ -396,15 +396,14 @@ class ConsistencyChecks {
SELECT note_revisions.noteRevisionId
FROM note_revisions
LEFT JOIN note_revision_contents USING (noteRevisionId)
WHERE note_revision_contents.noteRevisionId IS NULL
AND note_revisions.isProtected = 0`,
WHERE note_revision_contents.noteRevisionId IS NULL`,
({noteRevisionId}) => {
if (this.autoFix) {
noteRevisionService.eraseNoteRevisions([noteRevisionId]);
this.reloadNeeded = true;
logFix(`Note revision content '${noteRevisionId}' was created and set to erased since it did not exist.`);
logFix(`Note revision content '${noteRevisionId}' was set to erased since it did not exist.`);
} else {
logError(`Note revision content '${noteRevisionId}' does not exist`);
}
@ -597,6 +596,9 @@ class ConsistencyChecks {
this.runEntityChangeChecks("notes", "noteId");
this.runEntityChangeChecks("note_contents", "noteId");
this.runEntityChangeChecks("note_revisions", "noteRevisionId");
this.runEntityChangeChecks("note_revision_contents", "noteRevisionId");
this.runEntityChangeChecks("note_attachments", "noteAttachmentId");
this.runEntityChangeChecks("note_attachment_contents", "noteAttachmentId");
this.runEntityChangeChecks("branches", "branchId");
this.runEntityChangeChecks("attributes", "attributeId");
this.runEntityChangeChecks("etapi_tokens", "etapiTokenId");
@ -692,7 +694,7 @@ class ConsistencyChecks {
return `${tableName}: ${count}`;
}
const tables = [ "notes", "note_revisions", "branches", "attributes", "etapi_tokens" ];
const tables = [ "notes", "note_revisions", "note_attachments", "branches", "attributes", "etapi_tokens" ];
log.info(`Table counts: ${tables.map(tableName => getTableRowCount(tableName)).join(", ")}`);
}

2
src/services/entity_changes.js
View File

@ -128,6 +128,8 @@ function fillAllEntityChanges() {
fillEntityChanges("branches", "branchId");
fillEntityChanges("note_revisions", "noteRevisionId");
fillEntityChanges("note_revision_contents", "noteRevisionId");
fillEntityChanges("note_attachments", "noteAttachmentId");
fillEntityChanges("note_attachment_contents", "noteAttachmentId");
fillEntityChanges("attributes", "attributeId");
fillEntityChanges("etapi_tokens", "etapiTokenId");
fillEntityChanges("options", "name", 'isSynced = 1');

43
src/services/notes.js
View File

@ -21,6 +21,7 @@ const dayjs = require("dayjs");
const htmlSanitizer = require("./html_sanitizer");
const ValidationError = require("../errors/validation_error");
const noteTypesService = require("./note_types");
const BNoteAttachment = require("../becca/entities/bnote_attachment.js");
function getNewNotePosition(parentNoteId) {
const note = becca.notes[parentNoteId];
@ -666,6 +667,16 @@ function undeleteBranch(branchId, deleteId, taskContext) {
new BAttribute(attribute).save();
}
const noteAttachments = sql.getRows(`
SELECT * FROM note_attachments
WHERE isDeleted = 1
AND deleteId = ?
AND noteId = ?`, [deleteId, note.noteId]);
for (const noteAttachment of noteAttachments) {
new BNoteAttachment(noteAttachment).save();
}
const childBranchIds = sql.getColumn(`
SELECT branches.branchId
FROM branches
@ -738,6 +749,11 @@ function eraseNotes(noteIdsToErase) {
noteRevisionService.eraseNoteRevisions(noteRevisionIdsToErase);
const noteAttachmentIdsToErase = sql.getManyRows(`SELECT noteAttachmentId FROM note_attachments WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.noteAttachmentId);
eraseNoteAttachments(noteAttachmentIdsToErase);
log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
}
@ -773,6 +789,20 @@ function eraseAttributes(attributeIdsToErase) {
log.info(`Erased attributes: ${JSON.stringify(attributeIdsToErase)}`);
}
function eraseNoteAttachments(noteAttachmentIdsToErase) {
if (noteAttachmentIdsToErase.length === 0) {
return;
}
log.info(`Removing note attachments: ${JSON.stringify(noteAttachmentIdsToErase)}`);
sql.executeMany(`DELETE FROM note_attachments WHERE noteAttachmentId IN (???)`, noteAttachmentIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_attachments' AND entityId IN (???)`, noteAttachmentIdsToErase);
sql.executeMany(`DELETE FROM note_attachment_contents WHERE noteAttachmentId IN (???)`, noteAttachmentIdsToErase);
sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_attachment_contents' AND entityId IN (???)`, noteAttachmentIdsToErase);
}
function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
// this is important also so that the erased entity changes are sent to the connected clients
sql.transactional(() => {
@ -907,9 +937,22 @@ function duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapp
attr.save();
}
for (const noteAttachment of origNote.getNoteAttachments()) {
const duplNoteAttachment = new BNoteAttachment({
...noteAttachment,
noteAttachmentId: undefined,
noteId: newNote.noteId
});
duplNoteAttachment.save();
duplNoteAttachment.setContent(noteAttachment.getContent());
}
for (const childBranch of origNote.getChildBranches()) {
duplicateSubtreeInner(childBranch.getNote(), childBranch, newNote.noteId, noteIdMapping);
}
return newNote;
}

2
src/services/sync.js
View File

@ -321,7 +321,7 @@ function getEntityChangeRow(entityName, entityId) {
throw new Error(`Entity ${entityName} ${entityId} not found.`);
}
if (['note_contents', 'note_revision_contents'].includes(entityName) && entity.content !== null) {
if (['note_contents', 'note_revision_contents', 'note_attachment_contents'].includes(entityName) && entity.content !== null) {
if (typeof entity.content === 'string') {
entity.content = Buffer.from(entity.content, 'UTF-8');
}

15
src/services/sync_update.js
View File

@ -64,7 +64,7 @@ function updateNormalEntity(remoteEntityChange, remoteEntityRow, instanceId) {
|| localEntityChange.utcDateChanged < remoteEntityChange.utcDateChanged
|| localEntityChange.hash !== remoteEntityChange.hash // sync error, we should still update
) {
if (['note_contents', 'note_revision_contents'].includes(remoteEntityChange.entityName)) {
if (['note_contents', 'note_revision_contents', 'note_attachment_contents'].includes(remoteEntityChange.entityName)) {
remoteEntityRow.content = handleContent(remoteEntityRow.content);
}
@ -109,7 +109,18 @@ function handleContent(content) {
function eraseEntity(entityChange, instanceId) {
const {entityName, entityId} = entityChange;
if (!["notes", "note_contents", "branches", "attributes", "note_revisions", "note_revision_contents"].includes(entityName)) {
const entityNames = [
"notes",
"note_contents",
"branches",
"attributes",
"note_revisions",
"note_revision_contents",
"note_attachments",
"note_attachment_contents"
];
if (!entityNames.includes(entityName)) {
log.error(`Cannot erase entity '${entityName}', id '${entityId}'`);
return;
}