gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-06-06 18:08:33 +02:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'generateanonymize-script'

Browse Source 
# Conflicts:
#	package-lock.json
#	src/services/builtin_attributes.js
This commit is contained in:
zadam 2022-01-17 23:48:30 +01:00
commit 4fc3305080
18 changed files with 142 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitpod.yml
View File

@ -2,7 +2,7 @@ image:
file: .gitpod.dockerfile
tasks:
- before: nvm install 16.13.1 && nvm use 16.13.1
- before: nvm install 16.13.2 && nvm use 16.13.2
init: npm install
command: npm run start-server

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:16.13.1-alpine
FROM node:16.13.2-alpine
# Create app directory
WORKDIR /usr/src/app

2
bin/build-linux-x64.sh
View File

@ -27,6 +27,8 @@ cp images/app-icons/png/128x128.png $BUILD_DIR/icon.png
# removing software WebGL binaries because they are pretty huge and not necessary
rm -r $BUILD_DIR/swiftshader
cp bin/tpl/anonymize-database.sql $BUILD_DIR/
cp bin/tpl/trilium-portable.sh $BUILD_DIR/
chmod 755 $BUILD_DIR/trilium-portable.sh

2
bin/build-mac-x64.sh
View File

@ -23,6 +23,8 @@ rm -rf $BUILD_DIR
# Mac build has by default useless directory level
mv "./dist/Trilium Notes-darwin-x64" $BUILD_DIR
cp bin/tpl/anonymize-database.sql $BUILD_DIR/
echo "Zipping mac x64 electron distribution..."
VERSION=`jq -r ".version" package.json`

6
bin/build-server.sh
View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash
PKG_DIR=dist/trilium-linux-x64-server
NODE_VERSION=16.13.1
NODE_VERSION=16.13.2
if [ "$1" != "DONTCOPY" ]
then
@ -20,12 +20,16 @@ rm -r $PKG_DIR/node/lib/node_modules/npm
rm -r $PKG_DIR/node/include/node
rm -r $PKG_DIR/node_modules/electron*
rm -r $PKG_DIR/webpack*
rm -r $PKG_DIR/electron.js
cp -r bin/better-sqlite3/linux-server-better_sqlite3.node $PKG_DIR/node_modules/better-sqlite3/build/Release/better_sqlite3.node
printf "#!/bin/sh\n./node/bin/node src/www" > $PKG_DIR/trilium.sh
chmod 755 $PKG_DIR/trilium.sh
cp bin/tpl/anonymize-database.sql $PKG_DIR/
VERSION=`jq -r ".version" package.json`
cd dist

2
bin/build-win-x64.sh
View File

@ -25,6 +25,8 @@ mv "./dist/Trilium Notes-win32-x64" $BUILD_DIR
# removing software WebGL binaries because they are pretty huge and not necessary
rm -r $BUILD_DIR/swiftshader
cp bin/tpl/anonymize-database.sql $BUILD_DIR/
cp bin/tpl/trilium-portable.bat $BUILD_DIR/
cp bin/tpl/trilium-no-cert-check.bat $BUILD_DIR/
cp bin/tpl/trilium-safe-mode.bat $BUILD_DIR/

4
bin/copy-trilium.sh
View File

@ -5,7 +5,7 @@ if [[ $# -eq 0 ]] ; then
exit 1
fi
n exec 16.13.1 npm run webpack
n exec 16.13.2 npm run webpack
DIR=$1
@ -27,7 +27,7 @@ cp -r electron.js $DIR/
cp webpack-* $DIR/
# run in subshell (so we return to original dir)
(cd $DIR && n exec 16.13.1 npm install --only=prod)
(cd $DIR && n exec 16.13.2 npm install --only=prod)
# cleanup of useless files in dependencies
rm -r $DIR/node_modules/image-q/demo

7
bin/create-anonymization-script.js Normal file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env node
const anonymizationService = require('../src/services/anonymization');
const fs = require('fs');
const path = require('path');
fs.writeFileSync(path.resolve(__dirname, 'tpl', 'anonymize-database.sql'), anonymizationService.getAnonymizationScript());

17
bin/tpl/anonymize-database.sql Normal file
View File

@ -0,0 +1,17 @@
UPDATE etapi_tokens SET tokenHash = 'API token hash value';
UPDATE notes SET title = 'title';
UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL;
UPDATE note_revisions SET title = 'title';
UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');
UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN ('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');
UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL;
UPDATE options SET value = 'anonymized' WHERE name IN
('documentId', 'documentSecret', 'encryptedDataKey',
'passwordVerificationHash', 'passwordVerificationSalt',
'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy')
AND value != '';
VACUUM;

7
src/becca/entities/etapi_token.js
View File

@ -2,14 +2,13 @@
const dateUtils = require('../../services/date_utils');
const AbstractEntity = require("./abstract_entity");
const sql = require("../../services/sql.js");
/**
* EtapiToken is an entity representing token used to authenticate against Trilium REST API from client applications.
* Used by:
* - Trilium Sender
* - ETAPI clients
*
*
* The format user is presented with is "<etapiTokenId>_<tokenHash>". This is also called "authToken" to distinguish it
* from tokenHash and token.
*/
@ -42,10 +41,10 @@ class EtapiToken extends AbstractEntity {
this.utcDateModified = row.utcDateModified || this.utcDateCreated;
/** @type {boolean} */
this.isDeleted = !!row.isDeleted;
this.becca.etapiTokens[this.etapiTokenId] = this;
}
init() {
if (this.etapiTokenId) {
this.becca.etapiTokens[this.etapiTokenId] = this;

4
src/etapi/auth.js
View File

@ -1,7 +1,7 @@
const becca = require("../becca/becca");
const eu = require("./etapi_utils");
const passwordEncryptionService = require("../services/password_encryption.js");
const etapiTokenService = require("../services/etapi_tokens.js");
const passwordEncryptionService = require("../services/password_encryption");
const etapiTokenService = require("../services/etapi_tokens");
function register(router) {
eu.NOT_AUTHENTICATED_ROUTE(router, 'post', '/etapi/auth/login', (req, res, next) => {

10
src/etapi/etapi_utils.js
View File

@ -2,8 +2,8 @@ const cls = require("../services/cls");
const sql = require("../services/sql");
const log = require("../services/log");
const becca = require("../becca/becca");
const etapiTokenService = require("../services/etapi_tokens.js");
const config = require("../services/config.js");
const etapiTokenService = require("../services/etapi_tokens");
const config = require("../services/config");
const GENERIC_CODE = "GENERIC";
const noAuthentication = config.General && config.General.noAuthentication === true;
@ -11,7 +11,7 @@ const noAuthentication = config.General && config.General.noAuthentication === t
class EtapiError extends Error {
constructor(statusCode, code, message) {
super();
this.statusCode = statusCode;
this.code = code;
this.message = message;
@ -72,7 +72,7 @@ function NOT_AUTHENTICATED_ROUTE(router, method, path, routeHandler) {
function getAndCheckNote(noteId) {
const note = becca.getNote(noteId);
if (note) {
return note;
}
@ -118,7 +118,7 @@ function validateAndPatch(target, source, allowedProperties) {
}
}
}
// validation passed, let's patch
for (const propName of Object.keys(source)) {
target[propName] = source[propName];

3
src/routes/api/database.js
View File

@ -3,10 +3,11 @@
const sql = require('../../services/sql');
const log = require('../../services/log');
const backupService = require('../../services/backup');
const anonymizationService = require('../../services/anonymization');
const consistencyChecksService = require('../../services/consistency_checks');
async function anonymize() {
return await backupService.anonymize();
return await anonymizationService.createAnonymizedCopy();
}
async function backupDatabase() {

60
src/services/anonymization.js Normal file
View File

@ -0,0 +1,60 @@
const BUILTIN_ATTRIBUTES = require("./builtin_attributes");
const fs = require("fs-extra");
const dataDir = require("./data_dir");
const dateUtils = require("./date_utils");
const Database = require("better-sqlite3");
const sql = require("./sql");
function getAnonymizationScript() {
// we want to delete all non-builtin attributes because they can contain sensitive names and values
// on the other hand builtin/system attrs should not contain any sensitive info
const builtinAttrNames = BUILTIN_ATTRIBUTES
.map(attr => "'" + attr.name + "'").join(', ');
const anonymizeScript = `
UPDATE etapi_tokens SET tokenHash = 'API token hash value';
UPDATE notes SET title = 'title';
UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL;
UPDATE note_revisions SET title = 'title';
UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrNames});
UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrNames});
UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL;
UPDATE options SET value = 'anonymized' WHERE name IN
('documentId', 'documentSecret', 'encryptedDataKey',
'passwordVerificationHash', 'passwordVerificationSalt',
'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy')
AND value != '';
VACUUM;
`;
return anonymizeScript;
}
async function createAnonymizedCopy() {
if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
}
const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
await sql.copyDatabase(anonymizedFile);
const db = new Database(anonymizedFile);
db.exec(getAnonymizationScript());
db.close();
return {
success: true,
anonymizedFilePath: anonymizedFile
};
}
module.exports = {
getAnonymizationScript,
createAnonymizedCopy
}

18
src/services/attributes.js
View File

@ -82,7 +82,7 @@ function createAttribute(attribute) {
function getAttributeNames(type, nameLike) {
nameLike = nameLike.toLowerCase();
const names = sql.getColumn(
let names = sql.getColumn(
`SELECT DISTINCT name
FROM attributes
WHERE isDeleted = 0
@ -95,6 +95,13 @@ function getAttributeNames(type, nameLike) {
}
}
names = names.filter(name => ![
'internalLink',
'imageLink',
'includeNoteLink',
'relationMapLink'
].includes(name));
names.sort((a, b) => {
const aPrefix = a.toLowerCase().startsWith(nameLike);
const bPrefix = b.toLowerCase().startsWith(nameLike);
@ -122,14 +129,7 @@ function isAttributeDangerous(type, name) {
}
function getBuiltinAttributeNames() {
return BUILTIN_ATTRIBUTES
.map(attr => attr.name)
.concat([
'internalLink',
'imageLink',
'includeNoteLink',
'relationMapLink'
]);
return BUILTIN_ATTRIBUTES;
}
function sanitizeAttributeName(origName) {

60
src/services/backup.js
View File

@ -6,9 +6,8 @@ const fs = require('fs-extra');
const dataDir = require('./data_dir');
const log = require('./log');
const syncMutexService = require('./sync_mutex');
const attributeService = require('./attributes');
const cls = require('./cls');
const Database = require('better-sqlite3');
const sql = require('./sql');
function regularBackup() {
cls.init(() => {
@ -41,23 +40,12 @@ function periodBackup(optionName, backupType, periodInSeconds) {
}
}
async function copyFile(backupFile) {
const sql = require('./sql');
try {
fs.unlinkSync(backupFile);
} catch (e) {
} // unlink throws exception if the file did not exist
await sql.dbConnection.backup(backupFile);
}
async function backupNow(name) {
// we don't want to backup DB in the middle of sync with potentially inconsistent DB state
// we don't want to back up DB in the middle of sync with potentially inconsistent DB state
return await syncMutexService.doExclusively(async () => {
const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`;
await copyFile(backupFile);
await sql.copyDatabase(backupFile);
log.info("Created backup at " + backupFile);
@ -65,53 +53,11 @@ async function backupNow(name) {
});
}
async function anonymize() {
if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
}
const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
await copyFile(anonymizedFile);
const db = new Database(anonymizedFile);
db.prepare("UPDATE etapi_tokens SET tokenHash = 'API token hash value'").run();
db.prepare("UPDATE notes SET title = 'title'").run();
db.prepare("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL").run();
db.prepare("UPDATE note_revisions SET title = 'title'").run();
db.prepare("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL").run();
// we want to delete all non-builtin attributes because they can contain sensitive names and values
// on the other hand builtin/system attrs should not contain any sensitive info
const builtinAttrs = attributeService
.getBuiltinAttributeNames()
.map(name => "'" + name + "'").join(', ');
db.prepare(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`).run();
db.prepare(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`).run();
db.prepare("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL").run();
db.prepare(`UPDATE options SET value = 'anonymized' WHERE name IN
('documentId', 'documentSecret', 'encryptedDataKey',
'passwordVerificationHash', 'passwordVerificationSalt',
'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy')
AND value != ''`).run();
db.prepare("VACUUM").run();
db.close();
return {
success: true,
anonymizedFilePath: anonymizedFile
};
}
if (!fs.existsSync(dataDir.BACKUP_DIR)) {
fs.mkdirSync(dataDir.BACKUP_DIR, 0o700);
}
module.exports = {
backupNow,
anonymize,
regularBackup
};

10
src/services/builtin_attributes.js
View File

@ -48,6 +48,10 @@ module.exports = [
{ type: 'label', name: 'shareRoot' },
// relation names
{ type: 'relation', name: 'internalLink' },
{ type: 'relation', name: 'imageLink' },
{ type: 'relation', name: 'relationMapLink' },
{ type: 'relation', name: 'includeMapLink' },
{ type: 'relation', name: 'runOnNoteCreation', isDangerous: true },
{ type: 'relation', name: 'runOnNoteTitleChange', isDangerous: true },
{ type: 'relation', name: 'runOnNoteChange', isDangerous: true },
@ -57,7 +61,7 @@ module.exports = [
{ type: 'relation', name: 'template' },
{ type: 'relation', name: 'widget', isDangerous: true },
{ type: 'relation', name: 'renderNote', isDangerous: true },
{ type: 'relation', name: 'shareCss', isDangerous: false },
{ type: 'relation', name: 'shareJs', isDangerous: false },
{ type: 'relation', name: 'shareFavicon', isDangerous: false },
{ type: 'relation', name: 'shareCss' },
{ type: 'relation', name: 'shareJs' },
{ type: 'relation', name: 'shareFavicon' },
];

13
src/services/sql.js
View File

@ -8,6 +8,7 @@ const log = require('./log');
const Database = require('better-sqlite3');
const dataDir = require('./data_dir');
const cls = require('./cls');
const fs = require("fs-extra");
const dbConnection = new Database(dataDir.DOCUMENT_PATH);
dbConnection.pragma('journal_mode = WAL');
@ -276,6 +277,15 @@ function fillParamList(paramIds, truncate = true) {
s.run(paramIds);
}
async function copyDatabase(targetFilePath) {
try {
fs.unlinkSync(targetFilePath);
} catch (e) {
} // unlink throws exception if the file did not exist
await dbConnection.backup(targetFilePath);
}
module.exports = {
dbConnection,
insert,
@ -347,5 +357,6 @@ module.exports = {
executeScript,
transactional,
upsert,
fillParamList
fillParamList,
copyDatabase
};