Merge branch 'generateanonymize-script'

# Conflicts:
#	package-lock.json
#	src/services/builtin_attributes.js

.gitpod.yml

@@ -2,7 +2,7 @@ image:
   file: .gitpod.dockerfile
 
 tasks:
-    - before: nvm install 16.13.1 && nvm use 16.13.1
+    - before: nvm install 16.13.2 && nvm use 16.13.2
       init: npm install
       command: npm run start-server
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16.13.1-alpine
+FROM node:16.13.2-alpine
 
 # Create app directory
 WORKDIR /usr/src/app

bin/build-linux-x64.sh

@@ -27,6 +27,8 @@ cp images/app-icons/png/128x128.png $BUILD_DIR/icon.png
 # removing software WebGL binaries because they are pretty huge and not necessary
 rm -r $BUILD_DIR/swiftshader
 
+cp bin/tpl/anonymize-database.sql $BUILD_DIR/
+
 cp bin/tpl/trilium-portable.sh $BUILD_DIR/
 chmod 755 $BUILD_DIR/trilium-portable.sh
 

bin/build-mac-x64.sh

@@ -23,6 +23,8 @@ rm -rf $BUILD_DIR
 # Mac build has by default useless directory level
 mv "./dist/Trilium Notes-darwin-x64" $BUILD_DIR
 
+cp bin/tpl/anonymize-database.sql $BUILD_DIR/
+
 echo "Zipping mac x64 electron distribution..."
 
 VERSION=`jq -r ".version" package.json`

bin/build-server.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 PKG_DIR=dist/trilium-linux-x64-server
-NODE_VERSION=16.13.1
+NODE_VERSION=16.13.2
 
 if [ "$1" != "DONTCOPY" ]
 then
@@ -20,12 +20,16 @@ rm -r $PKG_DIR/node/lib/node_modules/npm
 rm -r $PKG_DIR/node/include/node
 
 rm -r $PKG_DIR/node_modules/electron*
+rm -r $PKG_DIR/webpack*
+rm -r $PKG_DIR/electron.js
 
 cp -r bin/better-sqlite3/linux-server-better_sqlite3.node $PKG_DIR/node_modules/better-sqlite3/build/Release/better_sqlite3.node
 
 printf "#!/bin/sh\n./node/bin/node src/www" > $PKG_DIR/trilium.sh
 chmod 755 $PKG_DIR/trilium.sh
 
+cp bin/tpl/anonymize-database.sql $PKG_DIR/
+
 VERSION=`jq -r ".version" package.json`
 
 cd dist

bin/build-win-x64.sh

@@ -25,6 +25,8 @@ mv "./dist/Trilium Notes-win32-x64" $BUILD_DIR
 # removing software WebGL binaries because they are pretty huge and not necessary
 rm -r $BUILD_DIR/swiftshader
 
+cp bin/tpl/anonymize-database.sql $BUILD_DIR/
+
 cp bin/tpl/trilium-portable.bat $BUILD_DIR/
 cp bin/tpl/trilium-no-cert-check.bat $BUILD_DIR/
 cp bin/tpl/trilium-safe-mode.bat $BUILD_DIR/

bin/copy-trilium.sh

@@ -5,7 +5,7 @@ if [[ $# -eq 0 ]] ; then
     exit 1
 fi
 
-n exec 16.13.1 npm run webpack
+n exec 16.13.2 npm run webpack
 
 DIR=$1
 
@@ -27,7 +27,7 @@ cp -r electron.js $DIR/
 cp webpack-* $DIR/
 
 # run in subshell (so we return to original dir)
-(cd $DIR && n exec 16.13.1 npm install --only=prod)
+(cd $DIR && n exec 16.13.2 npm install --only=prod)
 
 # cleanup of useless files in dependencies
 rm -r $DIR/node_modules/image-q/demo

bin/create-anonymization-script.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+const anonymizationService = require('../src/services/anonymization');
+const fs = require('fs');
+const path = require('path');
+
+fs.writeFileSync(path.resolve(__dirname, 'tpl', 'anonymize-database.sql'), anonymizationService.getAnonymizationScript());

bin/tpl/anonymize-database.sql

@@ -0,0 +1,17 @@
+
+UPDATE etapi_tokens SET tokenHash = 'API token hash value';
+UPDATE notes SET title = 'title';
+UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL;
+UPDATE note_revisions SET title = 'title';
+UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
+
+UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');
+UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN ('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');
+UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL;
+UPDATE options SET value = 'anonymized' WHERE name IN
+                    ('documentId', 'documentSecret', 'encryptedDataKey', 
+                     'passwordVerificationHash', 'passwordVerificationSalt', 
+                     'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 
+                      AND value != '';
+
+VACUUM;

src/becca/entities/etapi_token.js

@@ -2,7 +2,6 @@
 
 const dateUtils = require('../../services/date_utils');
 const AbstractEntity = require("./abstract_entity");
-const sql = require("../../services/sql.js");
 
 /**
  * EtapiToken is an entity representing token used to authenticate against Trilium REST API from client applications.

src/etapi/auth.js

@@ -1,7 +1,7 @@
 const becca = require("../becca/becca");
 const eu = require("./etapi_utils");
-const passwordEncryptionService = require("../services/password_encryption.js");
+const passwordEncryptionService = require("../services/password_encryption");
-const etapiTokenService = require("../services/etapi_tokens.js");
+const etapiTokenService = require("../services/etapi_tokens");
 
 function register(router) {
     eu.NOT_AUTHENTICATED_ROUTE(router, 'post', '/etapi/auth/login', (req, res, next) => {

src/etapi/etapi_utils.js

@@ -2,8 +2,8 @@ const cls = require("../services/cls");
 const sql = require("../services/sql");
 const log = require("../services/log");
 const becca = require("../becca/becca");
-const etapiTokenService = require("../services/etapi_tokens.js");
+const etapiTokenService = require("../services/etapi_tokens");
-const config = require("../services/config.js");
+const config = require("../services/config");
 const GENERIC_CODE = "GENERIC";
 
 const noAuthentication = config.General && config.General.noAuthentication === true;

src/routes/api/database.js

@@ -3,10 +3,11 @@
 const sql = require('../../services/sql');
 const log = require('../../services/log');
 const backupService = require('../../services/backup');
+const anonymizationService = require('../../services/anonymization');
 const consistencyChecksService = require('../../services/consistency_checks');
 
 async function anonymize() {
-    return await backupService.anonymize();
+    return await anonymizationService.createAnonymizedCopy();
 }
 
 async function backupDatabase() {

src/services/anonymization.js

@@ -0,0 +1,60 @@
+const BUILTIN_ATTRIBUTES = require("./builtin_attributes");
+const fs = require("fs-extra");
+const dataDir = require("./data_dir");
+const dateUtils = require("./date_utils");
+const Database = require("better-sqlite3");
+const sql = require("./sql");
+
+function getAnonymizationScript() {
+    // we want to delete all non-builtin attributes because they can contain sensitive names and values
+// on the other hand builtin/system attrs should not contain any sensitive info
+    const builtinAttrNames = BUILTIN_ATTRIBUTES
+        .map(attr => "'" + attr.name + "'").join(', ');
+
+    const anonymizeScript = `
+UPDATE etapi_tokens SET tokenHash = 'API token hash value';
+UPDATE notes SET title = 'title';
+UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL;
+UPDATE note_revisions SET title = 'title';
+UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
+
+UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrNames});
+UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrNames});
+UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL;
+UPDATE options SET value = 'anonymized' WHERE name IN
+                    ('documentId', 'documentSecret', 'encryptedDataKey', 
+                     'passwordVerificationHash', 'passwordVerificationSalt', 
+                     'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 
+                      AND value != '';
+
+VACUUM;
+`;
+
+    return anonymizeScript;
+}
+
+async function createAnonymizedCopy() {
+    if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
+        fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
+    }
+
+    const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
+
+    await sql.copyDatabase(anonymizedFile);
+
+    const db = new Database(anonymizedFile);
+
+    db.exec(getAnonymizationScript());
+
+    db.close();
+
+    return {
+        success: true,
+        anonymizedFilePath: anonymizedFile
+    };
+}
+
+module.exports = {
+    getAnonymizationScript,
+    createAnonymizedCopy
+}

src/services/attributes.js

@@ -82,7 +82,7 @@ function createAttribute(attribute) {
 function getAttributeNames(type, nameLike) {
     nameLike = nameLike.toLowerCase();
 
-    const names = sql.getColumn(
+    let names = sql.getColumn(
         `SELECT DISTINCT name 
              FROM attributes 
              WHERE isDeleted = 0
@@ -95,6 +95,13 @@ function getAttributeNames(type, nameLike) {
         }
     }
 
+    names = names.filter(name => ![
+        'internalLink',
+        'imageLink',
+        'includeNoteLink',
+        'relationMapLink'
+    ].includes(name));
+
     names.sort((a, b) => {
         const aPrefix = a.toLowerCase().startsWith(nameLike);
         const bPrefix = b.toLowerCase().startsWith(nameLike);
@@ -122,14 +129,7 @@ function isAttributeDangerous(type, name) {
 }
 
 function getBuiltinAttributeNames() {
-    return BUILTIN_ATTRIBUTES
+    return BUILTIN_ATTRIBUTES;
-        .map(attr => attr.name)
-        .concat([
-            'internalLink',
-            'imageLink',
-            'includeNoteLink',
-            'relationMapLink'
-        ]);
 }
 
 function sanitizeAttributeName(origName) {

src/services/backup.js

@@ -6,9 +6,8 @@ const fs = require('fs-extra');
 const dataDir = require('./data_dir');
 const log = require('./log');
 const syncMutexService = require('./sync_mutex');
-const attributeService = require('./attributes');
 const cls = require('./cls');
-const Database = require('better-sqlite3');
+const sql = require('./sql');
 
 function regularBackup() {
     cls.init(() => {
@@ -41,23 +40,12 @@ function periodBackup(optionName, backupType, periodInSeconds) {
     }
 }
 
-async function copyFile(backupFile) {
-    const sql = require('./sql');
-
-    try {
-        fs.unlinkSync(backupFile);
-    } catch (e) {
-    } // unlink throws exception if the file did not exist
-
-    await sql.dbConnection.backup(backupFile);
-}
-
 async function backupNow(name) {
     // we don't want to back up DB in the middle of sync with potentially inconsistent DB state
     return await syncMutexService.doExclusively(async () => {
         const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`;
 
-        await copyFile(backupFile);
+        await sql.copyDatabase(backupFile);
 
         log.info("Created backup at " + backupFile);
 
@@ -65,53 +53,11 @@ async function backupNow(name) {
     });
 }
 
-async function anonymize() {
-    if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
-        fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
-    }
-
-    const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
-
-    await copyFile(anonymizedFile);
-
-    const db = new Database(anonymizedFile);
-
-    db.prepare("UPDATE etapi_tokens SET tokenHash = 'API token hash value'").run();
-    db.prepare("UPDATE notes SET title = 'title'").run();
-    db.prepare("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL").run();
-    db.prepare("UPDATE note_revisions SET title = 'title'").run();
-    db.prepare("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL").run();
-
-    // we want to delete all non-builtin attributes because they can contain sensitive names and values
-    // on the other hand builtin/system attrs should not contain any sensitive info
-    const builtinAttrs = attributeService
-        .getBuiltinAttributeNames()
-        .map(name => "'" + name + "'").join(', ');
-
-    db.prepare(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`).run();
-    db.prepare(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`).run();
-    db.prepare("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL").run();
-    db.prepare(`UPDATE options SET value = 'anonymized' WHERE name IN 
-                    ('documentId', 'documentSecret', 'encryptedDataKey', 
-                     'passwordVerificationHash', 'passwordVerificationSalt', 
-                     'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 
-                      AND value != ''`).run();
-    db.prepare("VACUUM").run();
-
-    db.close();
-
-    return {
-        success: true,
-        anonymizedFilePath: anonymizedFile
-    };
-}
-
 if (!fs.existsSync(dataDir.BACKUP_DIR)) {
     fs.mkdirSync(dataDir.BACKUP_DIR, 0o700);
 }
 
 module.exports = {
     backupNow,
-    anonymize,
     regularBackup
 };

src/services/builtin_attributes.js

@@ -48,6 +48,10 @@ module.exports = [
     { type: 'label', name: 'shareRoot' },
 
     // relation names
+    { type: 'relation', name: 'internalLink' },
+    { type: 'relation', name: 'imageLink' },
+    { type: 'relation', name: 'relationMapLink' },
+    { type: 'relation', name: 'includeMapLink' },
     { type: 'relation', name: 'runOnNoteCreation', isDangerous: true },
     { type: 'relation', name: 'runOnNoteTitleChange', isDangerous: true },
     { type: 'relation', name: 'runOnNoteChange', isDangerous: true },
@@ -57,7 +61,7 @@ module.exports = [
     { type: 'relation', name: 'template' },
     { type: 'relation', name: 'widget', isDangerous: true },
     { type: 'relation', name: 'renderNote', isDangerous: true },
-    { type: 'relation', name: 'shareCss', isDangerous: false },
+    { type: 'relation', name: 'shareCss' },
-    { type: 'relation', name: 'shareJs', isDangerous: false },
+    { type: 'relation', name: 'shareJs' },
-    { type: 'relation', name: 'shareFavicon', isDangerous: false },
+    { type: 'relation', name: 'shareFavicon' },
 ];

src/services/sql.js

@@ -8,6 +8,7 @@ const log = require('./log');
 const Database = require('better-sqlite3');
 const dataDir = require('./data_dir');
 const cls = require('./cls');
+const fs = require("fs-extra");
 
 const dbConnection = new Database(dataDir.DOCUMENT_PATH);
 dbConnection.pragma('journal_mode = WAL');
@@ -276,6 +277,15 @@ function fillParamList(paramIds, truncate = true) {
     s.run(paramIds);
 }
 
+async function copyDatabase(targetFilePath) {
+    try {
+        fs.unlinkSync(targetFilePath);
+    } catch (e) {
+    } // unlink throws exception if the file did not exist
+
+    await dbConnection.backup(targetFilePath);
+}
+
 module.exports = {
     dbConnection,
     insert,
@@ -347,5 +357,6 @@ module.exports = {
     executeScript,
     transactional,
     upsert,
-    fillParamList
+    fillParamList,
+    copyDatabase
 };