From 66a6c7655293f2c184efed5537f805250817c08a Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 15 Jan 2022 22:09:51 +0100 Subject: [PATCH] generate anonymization script WIP --- bin/anonymize-database.sql | 17 ++++++ bin/generate-anonymization-script.js | 31 +++++++++++ src/services/attributes.js | 82 ++++------------------------ src/services/builtin_attributes.js | 66 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 72 deletions(-) create mode 100644 bin/anonymize-database.sql create mode 100644 bin/generate-anonymization-script.js create mode 100644 src/services/builtin_attributes.js diff --git a/bin/anonymize-database.sql b/bin/anonymize-database.sql new file mode 100644 index 000000000..b2057628a --- /dev/null +++ b/bin/anonymize-database.sql @@ -0,0 +1,17 @@ + +UPDATE etapi_tokens SET tokenHash = 'API token hash value'; +UPDATE notes SET title = 'title'; +UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL; +UPDATE note_revisions SET title = 'title'; +UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL; + +UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon'); +UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN ('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon'); +UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL; +UPDATE options SET value = 'anonymized' WHERE + ('documentId', 'documentSecret', 'encryptedDataKey', + 'passwordVerificationHash', 'passwordVerificationSalt', + 'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') + AND value != ''; + +VACUUM; diff --git a/bin/generate-anonymization-script.js b/bin/generate-anonymization-script.js new file mode 100644 index 000000000..e5db962a7 --- /dev/null +++ b/bin/generate-anonymization-script.js @@ -0,0 +1,31 @@ +#!/usr/bin/env node + +const BUILTIN_ATTRIBUTES = require('../src/services/builtin_attributes'); +const fs = require('fs'); +const path = require('path'); + +// we want to delete all non-builtin attributes because they can contain sensitive names and values +// on the other hand builtin/system attrs should not contain any sensitive info +const builtinAttrNames = BUILTIN_ATTRIBUTES + .map(attr => "'" + attr.name + "'").join(', '); + +const anonymizeScript = ` +UPDATE etapi_tokens SET tokenHash = 'API token hash value'; +UPDATE notes SET title = 'title'; +UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL; +UPDATE note_revisions SET title = 'title'; +UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL; + +UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrNames}); +UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrNames}); +UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL; +UPDATE options SET value = 'anonymized' WHERE + ('documentId', 'documentSecret', 'encryptedDataKey', + 'passwordVerificationHash', 'passwordVerificationSalt', + 'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') + AND value != ''; + +VACUUM; +`; + +fs.writeFileSync(path.resolve(__dirname, 'anonymize-database.sql'), anonymizeScript); diff --git a/src/services/attributes.js b/src/services/attributes.js index 89a637e00..5fcc121bd 100644 --- a/src/services/attributes.js +++ b/src/services/attributes.js @@ -5,72 +5,10 @@ const sql = require('./sql'); const becca = require('../becca/becca'); const Attribute = require('../becca/entities/attribute'); const {formatAttrForSearch} = require("./attribute_formatter"); +const BUILTIN_ATTRIBUTES = require("./builtin_attributes"); const ATTRIBUTE_TYPES = [ 'label', 'relation' ]; -const BUILTIN_ATTRIBUTES = [ - // label names - { type: 'label', name: 'inbox' }, - { type: 'label', name: 'disableVersioning' }, - { type: 'label', name: 'calendarRoot' }, - { type: 'label', name: 'archived' }, - { type: 'label', name: 'excludeFromExport' }, - { type: 'label', name: 'disableInclusion' }, - { type: 'label', name: 'appCss' }, - { type: 'label', name: 'appTheme' }, - { type: 'label', name: 'hidePromotedAttributes' }, - { type: 'label', name: 'readOnly' }, - { type: 'label', name: 'autoReadOnlyDisabled' }, - { type: 'label', name: 'hoistedCssClass' }, - { type: 'label', name: 'cssClass' }, - { type: 'label', name: 'iconClass' }, - { type: 'label', name: 'keyboardShortcut' }, - { type: 'label', name: 'run', isDangerous: true }, - { type: 'label', name: 'runOnInstance', isDangerous: false }, - { type: 'label', name: 'runAtHour', isDangerous: false }, - { type: 'label', name: 'customRequestHandler', isDangerous: true }, - { type: 'label', name: 'customResourceProvider', isDangerous: true }, - { type: 'label', name: 'widget', isDangerous: true }, - { type: 'label', name: 'noteInfoWidgetDisabled' }, - { type: 'label', name: 'linkMapWidgetDisabled' }, - { type: 'label', name: 'noteRevisionsWidgetDisabled' }, - { type: 'label', name: 'whatLinksHereWidgetDisabled' }, - { type: 'label', name: 'similarNotesWidgetDisabled' }, - { type: 'label', name: 'workspace' }, - { type: 'label', name: 'workspaceIconClass' }, - { type: 'label', name: 'workspaceTabBackgroundColor' }, - { type: 'label', name: 'searchHome' }, - { type: 'label', name: 'hoistedInbox' }, - { type: 'label', name: 'hoistedSearchHome' }, - { type: 'label', name: 'sqlConsoleHome' }, - { type: 'label', name: 'datePattern' }, - { type: 'label', name: 'pageSize' }, - { type: 'label', name: 'viewType' }, - { type: 'label', name: 'mapRootNoteId' }, - { type: 'label', name: 'bookmarked' }, - { type: 'label', name: 'bookmarkFolder' }, - { type: 'label', name: 'sorted' }, - { type: 'label', name: 'top' }, - { type: 'label', name: 'fullContentWidth' }, - { type: 'label', name: 'shareHiddenFromTree' }, - { type: 'label', name: 'shareAlias' }, - { type: 'label', name: 'shareOmitDefaultCss' }, - - // relation names - { type: 'relation', name: 'runOnNoteCreation', isDangerous: true }, - { type: 'relation', name: 'runOnNoteTitleChange', isDangerous: true }, - { type: 'relation', name: 'runOnNoteChange', isDangerous: true }, - { type: 'relation', name: 'runOnChildNoteCreation', isDangerous: true }, - { type: 'relation', name: 'runOnAttributeCreation', isDangerous: true }, - { type: 'relation', name: 'runOnAttributeChange', isDangerous: true }, - { type: 'relation', name: 'template' }, - { type: 'relation', name: 'widget', isDangerous: true }, - { type: 'relation', name: 'renderNote', isDangerous: true }, - { type: 'relation', name: 'shareCss', isDangerous: false }, - { type: 'relation', name: 'shareJs', isDangerous: false }, - { type: 'relation', name: 'shareFavicon', isDangerous: false }, -]; - /** @returns {Note[]} */ function getNotesWithLabel(name, value) { const query = formatAttrForSearch({type: 'label', name, value}, true); @@ -144,7 +82,7 @@ function createAttribute(attribute) { function getAttributeNames(type, nameLike) { nameLike = nameLike.toLowerCase(); - const names = sql.getColumn( + let names = sql.getColumn( `SELECT DISTINCT name FROM attributes WHERE isDeleted = 0 @@ -157,6 +95,13 @@ function getAttributeNames(type, nameLike) { } } + names = names.filter(name => ![ + 'internalLink', + 'imageLink', + 'includeNoteLink', + 'relationMapLink' + ].includes(name)); + names.sort((a, b) => { const aPrefix = a.toLowerCase().startsWith(nameLike); const bPrefix = b.toLowerCase().startsWith(nameLike); @@ -184,14 +129,7 @@ function isAttributeDangerous(type, name) { } function getBuiltinAttributeNames() { - return BUILTIN_ATTRIBUTES - .map(attr => attr.name) - .concat([ - 'internalLink', - 'imageLink', - 'includeNoteLink', - 'relationMapLink' - ]); + return BUILTIN_ATTRIBUTES; } function sanitizeAttributeName(origName) { diff --git a/src/services/builtin_attributes.js b/src/services/builtin_attributes.js new file mode 100644 index 000000000..11b2811fe --- /dev/null +++ b/src/services/builtin_attributes.js @@ -0,0 +1,66 @@ +module.exports = [ + // label names + { type: 'label', name: 'inbox' }, + { type: 'label', name: 'disableVersioning' }, + { type: 'label', name: 'calendarRoot' }, + { type: 'label', name: 'archived' }, + { type: 'label', name: 'excludeFromExport' }, + { type: 'label', name: 'disableInclusion' }, + { type: 'label', name: 'appCss' }, + { type: 'label', name: 'appTheme' }, + { type: 'label', name: 'hidePromotedAttributes' }, + { type: 'label', name: 'readOnly' }, + { type: 'label', name: 'autoReadOnlyDisabled' }, + { type: 'label', name: 'hoistedCssClass' }, + { type: 'label', name: 'cssClass' }, + { type: 'label', name: 'iconClass' }, + { type: 'label', name: 'keyboardShortcut' }, + { type: 'label', name: 'run', isDangerous: true }, + { type: 'label', name: 'runOnInstance', isDangerous: false }, + { type: 'label', name: 'runAtHour', isDangerous: false }, + { type: 'label', name: 'customRequestHandler', isDangerous: true }, + { type: 'label', name: 'customResourceProvider', isDangerous: true }, + { type: 'label', name: 'widget', isDangerous: true }, + { type: 'label', name: 'noteInfoWidgetDisabled' }, + { type: 'label', name: 'linkMapWidgetDisabled' }, + { type: 'label', name: 'noteRevisionsWidgetDisabled' }, + { type: 'label', name: 'whatLinksHereWidgetDisabled' }, + { type: 'label', name: 'similarNotesWidgetDisabled' }, + { type: 'label', name: 'workspace' }, + { type: 'label', name: 'workspaceIconClass' }, + { type: 'label', name: 'workspaceTabBackgroundColor' }, + { type: 'label', name: 'searchHome' }, + { type: 'label', name: 'hoistedInbox' }, + { type: 'label', name: 'hoistedSearchHome' }, + { type: 'label', name: 'sqlConsoleHome' }, + { type: 'label', name: 'datePattern' }, + { type: 'label', name: 'pageSize' }, + { type: 'label', name: 'viewType' }, + { type: 'label', name: 'mapRootNoteId' }, + { type: 'label', name: 'bookmarked' }, + { type: 'label', name: 'bookmarkFolder' }, + { type: 'label', name: 'sorted' }, + { type: 'label', name: 'top' }, + { type: 'label', name: 'fullContentWidth' }, + { type: 'label', name: 'shareHiddenFromTree' }, + { type: 'label', name: 'shareAlias' }, + { type: 'label', name: 'shareOmitDefaultCss' }, + + // relation names + { type: 'relation', name: 'internalLink' }, + { type: 'relation', name: 'imageLink' }, + { type: 'relation', name: 'relationMapLink' }, + { type: 'relation', name: 'includeMapLink' }, + { type: 'relation', name: 'runOnNoteCreation', isDangerous: true }, + { type: 'relation', name: 'runOnNoteTitleChange', isDangerous: true }, + { type: 'relation', name: 'runOnNoteChange', isDangerous: true }, + { type: 'relation', name: 'runOnChildNoteCreation', isDangerous: true }, + { type: 'relation', name: 'runOnAttributeCreation', isDangerous: true }, + { type: 'relation', name: 'runOnAttributeChange', isDangerous: true }, + { type: 'relation', name: 'template' }, + { type: 'relation', name: 'widget', isDangerous: true }, + { type: 'relation', name: 'renderNote', isDangerous: true }, + { type: 'relation', name: 'shareCss' }, + { type: 'relation', name: 'shareJs' }, + { type: 'relation', name: 'shareFavicon' }, +];