Merge branch 'beta'

# Conflicts:
#	docs/backend_api/BAttachment.html
#	docs/backend_api/BNote.html
#	docs/backend_api/BackendScriptApi.html
#	package-lock.json
#	package.json
This commit is contained in:
zadam 2023-10-19 01:13:45 +02:00
commit 52244ddc99
22 changed files with 198 additions and 68 deletions

View File

@ -2,7 +2,7 @@ image:
file: .gitpod.dockerfile
tasks:
- before: nvm install 18.18.0 && nvm use 18.18.0
- before: nvm install 18.18.2 && nvm use 18.18.2
init: npm install
command: npm run start-server

View File

@ -1,5 +1,5 @@
# !!! Don't try to build this Dockerfile directly, run it through bin/build-docker.sh script !!!
FROM node:18.18.0-alpine
FROM node:18.18.2-alpine
# Create app directory
WORKDIR /usr/src/app

View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash
PKG_DIR=dist/trilium-linux-x64-server
NODE_VERSION=18.18.0
NODE_VERSION=18.18.2
if [ "$1" != "DONTCOPY" ]
then

View File

@ -5,7 +5,7 @@ if [[ $# -eq 0 ]] ; then
exit 1
fi
n exec 18.18.0 npm run webpack
n exec 18.18.2 npm run webpack
DIR=$1
@ -27,7 +27,7 @@ cp -r electron.js $DIR/
cp webpack-* $DIR/
# run in subshell (so we return to original dir)
(cd $DIR && n exec 18.18.0 npm install --only=prod)
(cd $DIR && n exec 18.18.2 npm install --only=prod)
# cleanup of useless files in dependencies
rm -r $DIR/node_modules/image-q/demo

View File

@ -47,6 +47,7 @@ const SpacedUpdate = require("./spaced_update");
const specialNotesService = require("./special_notes");
const branchService = require("./branches");
const exportService = require("./export/zip");
const syncMutex = require("./sync_mutex.js");
/**
@ -628,6 +629,20 @@ function BackendScriptApi(currentNote, apiParams) {
}
};
/**
* Sync process can make data intermittently inconsistent. Scripts which require strong data consistency
* can use this function to wait for a possible sync process to finish and prevent new sync process from starting
* while it is running.
*
* Because this is an async process, the inner callback doesn't have automatic transaction handling, so in case
* you need to make some DB changes, you need to surround your call with api.transactional(...)
*
* @method
* @param {function} callback - function to be executed while sync process is not running
* @returns {Promise} - resolves once the callback is finished (callback is awaited)
*/
this.runOutsideOfSync = syncMutex.doExclusively;
/**
* This object contains "at your risk" and "no BC guarantees" objects for advanced use cases.
*

View File

@ -217,7 +217,7 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
};
}
const {elements, files, appState} = content;
const {elements, files, appState = {}} = content;
appState.theme = this.themeStyle;

View File

@ -20,6 +20,10 @@ const TPL = `
<p>You can decide yourself if you want to provide a fully or lightly anonymized database. Even fully anonymized DB is very useful, however in some cases lightly anonymized database can speed up the process of bug identification and fixing.</p>
<button class="anonymize-light-button btn">Save lightly anonymized database</button>
<h5>Existing anonymized databases</h5>
<ul class="existing-anonymized-databases"></ul>
</div>`;
export default class DatabaseAnonymizationOptions extends OptionsWidget {
@ -38,6 +42,8 @@ export default class DatabaseAnonymizationOptions extends OptionsWidget {
else {
toastService.showMessage(`Created fully anonymized database in ${resp.anonymizedFilePath}`, 10000);
}
this.refresh();
});
this.$anonymizeLightButton.on('click', async () => {
@ -51,6 +57,24 @@ export default class DatabaseAnonymizationOptions extends OptionsWidget {
else {
toastService.showMessage(`Created lightly anonymized database in ${resp.anonymizedFilePath}`, 10000);
}
this.refresh();
});
this.$existingAnonymizedDatabases = this.$widget.find(".existing-anonymized-databases");
}
optionsLoaded(options) {
server.get("database/anonymized-databases").then(anonymizedDatabases => {
this.$existingAnonymizedDatabases.empty();
if (!anonymizedDatabases.length) {
anonymizedDatabases = [{filePath: "no anonymized database yet"}];
}
for (const {filePath} of anonymizedDatabases) {
this.$existingAnonymizedDatabases.append($("<li>").text(filePath));
}
});
}
}

View File

@ -37,6 +37,12 @@ const TPL = `
<button class="backup-database-button btn">Backup database now</button>
</div>
<div class="options-section">
<h4>Existing backups</h4>
<ul class="existing-backup-list"></ul>
</div>
`;
export default class BackupOptions extends OptionsWidget {
@ -49,6 +55,8 @@ export default class BackupOptions extends OptionsWidget {
const {backupFile} = await server.post('database/backup-database');
toastService.showMessage(`Database has been backed up to ${backupFile}`, 10000);
this.refresh();
});
this.$dailyBackupEnabled = this.$widget.find(".daily-backup-enabled");
@ -63,11 +71,25 @@ export default class BackupOptions extends OptionsWidget {
this.$monthlyBackupEnabled.on('change', () =>
this.updateCheckboxOption('monthlyBackupEnabled', this.$monthlyBackupEnabled));
this.$existingBackupList = this.$widget.find(".existing-backup-list");
}
optionsLoaded(options) {
this.setCheckboxState(this.$dailyBackupEnabled, options.dailyBackupEnabled);
this.setCheckboxState(this.$weeklyBackupEnabled, options.weeklyBackupEnabled);
this.setCheckboxState(this.$monthlyBackupEnabled, options.monthlyBackupEnabled);
server.get("database/backups").then(backupFiles => {
this.$existingBackupList.empty();
if (!backupFiles.length) {
backupFiles = [{filePath: "no backup yet", ctime: ''}];
}
for (const {filePath, ctime} of backupFiles) {
this.$existingBackupList.append($("<li>").text(`${filePath} ${ctime ? ` - ${ctime}` : ''}`));
}
});
}
}

View File

@ -6,8 +6,8 @@ const backupService = require('../../services/backup');
const anonymizationService = require('../../services/anonymization');
const consistencyChecksService = require('../../services/consistency_checks');
async function anonymize(req) {
return await anonymizationService.createAnonymizedCopy(req.params.type);
function getExistingBackups() {
return backupService.getExistingBackups();
}
async function backupDatabase() {
@ -22,6 +22,18 @@ function vacuumDatabase() {
log.info("Database has been vacuumed.");
}
function findAndFixConsistencyIssues() {
consistencyChecksService.runOnDemandChecks(true);
}
function getExistingAnonymizedDatabases() {
return anonymizationService.getExistingAnonymizedDatabases();
}
async function anonymize(req) {
return await anonymizationService.createAnonymizedCopy(req.params.type);
}
function checkIntegrity() {
const results = sql.getRows("PRAGMA integrity_check");
@ -32,14 +44,12 @@ function checkIntegrity() {
};
}
function findAndFixConsistencyIssues() {
consistencyChecksService.runOnDemandChecks(true);
}
module.exports = {
getExistingBackups,
backupDatabase,
vacuumDatabase,
findAndFixConsistencyIssues,
getExistingAnonymizedDatabases,
anonymize,
checkIntegrity
};

View File

@ -289,9 +289,11 @@ function register(app) {
apiRoute(GET, '/api/sql/schema', sqlRoute.getSchema);
apiRoute(PST, '/api/sql/execute/:noteId', sqlRoute.execute);
route(PST, '/api/database/anonymize/:type', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.anonymize, apiResultHandler, false);
apiRoute(GET, '/api/database/anonymized-databases', databaseRoute.getExistingAnonymizedDatabases);
// backup requires execution outside of transaction
route(PST, '/api/database/backup-database', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.backupDatabase, apiResultHandler, false);
apiRoute(GET, '/api/database/backups', databaseRoute.getExistingBackups);
// VACUUM requires execution outside of transaction
route(PST, '/api/database/vacuum-database', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.vacuumDatabase, apiResultHandler, false);

View File

@ -4,6 +4,7 @@ const dataDir = require("./data_dir");
const dateUtils = require("./date_utils");
const Database = require("better-sqlite3");
const sql = require("./sql");
const path = require("path");
function getFullAnonymizationScript() {
// we want to delete all non-builtin attributes because they can contain sensitive names and values
@ -70,7 +71,21 @@ async function createAnonymizedCopy(type) {
};
}
function getExistingAnonymizedDatabases() {
if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
return [];
}
return fs.readdirSync(dataDir.ANONYMIZED_DB_DIR)
.filter(fileName => fileName.includes("anonymized"))
.map(fileName => ({
fileName: fileName,
filePath: path.resolve(dataDir.ANONYMIZED_DB_DIR, fileName)
}));
}
module.exports = {
getFullAnonymizationScript,
createAnonymizedCopy
createAnonymizedCopy,
getExistingAnonymizedDatabases
}

View File

@ -19,6 +19,7 @@ const SpacedUpdate = require("./spaced_update");
const specialNotesService = require("./special_notes");
const branchService = require("./branches");
const exportService = require("./export/zip");
const syncMutex = require("./sync_mutex.js");
/**
@ -600,6 +601,20 @@ function BackendScriptApi(currentNote, apiParams) {
}
};
/**
* Sync process can make data intermittently inconsistent. Scripts which require strong data consistency
* can use this function to wait for a possible sync process to finish and prevent new sync process from starting
* while it is running.
*
* Because this is an async process, the inner callback doesn't have automatic transaction handling, so in case
* you need to make some DB changes, you need to surround your call with api.transactional(...)
*
* @method
* @param {function} callback - function to be executed while sync process is not running
* @returns {Promise} - resolves once the callback is finished (callback is awaited)
*/
this.runOutsideOfSync = syncMutex.doExclusively;
/**
* This object contains "at your risk" and "no BC guarantees" objects for advanced use cases.
*

View File

@ -8,6 +8,22 @@ const log = require('./log');
const syncMutexService = require('./sync_mutex');
const cls = require('./cls');
const sql = require('./sql');
const path = require('path');
function getExistingBackups() {
if (!fs.existsSync(dataDir.BACKUP_DIR)) {
return [];
}
return fs.readdirSync(dataDir.BACKUP_DIR)
.filter(fileName => fileName.includes("backup"))
.map(fileName => {
const filePath = path.resolve(dataDir.BACKUP_DIR, fileName);
const stat = fs.statSync(filePath)
return {fileName, filePath, ctime: stat.ctime};
});
}
function regularBackup() {
cls.init(() => {
@ -58,6 +74,7 @@ if (!fs.existsSync(dataDir.BACKUP_DIR)) {
}
module.exports = {
getExistingBackups,
backupNow,
regularBackup
};

View File

@ -1 +1 @@
module.exports = { buildDate:"2023-09-29T00:54:45+02:00", buildRevision: "e5555beea9a1638fefa218118e0596f4cfc1f4d0" };
module.exports = { buildDate:"2023-10-07T23:02:47+03:00", buildRevision: "3d15aeae58224ac8716dd58938458e89af9bf7a0" };

View File

@ -68,6 +68,7 @@ module.exports = [
{ type: 'label', name: 'executeDescription'},
{ type: 'label', name: 'newNotesOnTop'},
{ type: 'label', name: 'clipperInbox'},
{ type: 'label', name: 'webViewSrc', isDangerous: true },
// relation names
{ type: 'relation', name: 'internalLink' },

View File

@ -7,7 +7,7 @@ const BBranch = require('../becca/entities/bbranch');
const becca = require("../becca/becca");
const log = require("./log");
function cloneNoteToParentNote(noteId, parentNoteId, prefix) {
function cloneNoteToParentNote(noteId, parentNoteId, prefix = null) {
if (!(noteId in becca.notes) || !(parentNoteId in becca.notes)) {
return { success: false, message: 'Note cannot be cloned because either the cloned note or the intended parent is deleted.' };
}

View File

@ -311,7 +311,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
return /^(?:[a-z]+:)?\/\//i.test(url);
}
content = removeTrilumTags(content);
content = removeTriliumTags(content);
content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
if (noteTitle.trim() === text.trim()) {
@ -393,7 +393,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
return content;
}
function removeTrilumTags(content) {
function removeTriliumTags(content) {
const tagsToRemove = [
'<h1 data-trilium-h1>([^<]*)<\/h1>',
'<title data-trilium-title>([^<]*)<\/title>'

View File

@ -58,10 +58,6 @@ function exec(opts) {
request.on('error', err => reject(generateError(opts, err)));
request.on('response', response => {
if (![200, 201, 204].includes(response.statusCode)) {
reject(generateError(opts, `${response.statusCode} ${response.statusMessage}`));
}
if (opts.cookieJar && response.headers['set-cookie']) {
opts.cookieJar.header = response.headers['set-cookie'];
}
@ -71,16 +67,29 @@ function exec(opts) {
response.on('data', chunk => responseStr += chunk);
response.on('end', () => {
if ([200, 201, 204].includes(response.statusCode)) {
try {
const jsonObj = responseStr.trim() ? JSON.parse(responseStr) : null;
resolve(jsonObj);
}
catch (e) {
} catch (e) {
log.error(`Failed to deserialize sync response: ${responseStr}`);
reject(generateError(opts, e.message));
}
} else {
let errorMessage;
try {
const jsonObj = JSON.parse(responseStr);
errorMessage = jsonObj?.message || '';
} catch (e) {
errorMessage = responseStr.substr(0, Math.min(responseStr.length, 100));
}
reject(generateError(opts, `${response.statusCode} ${response.statusMessage} ${errorMessage}`));
}
});
});

View File

@ -22,6 +22,10 @@ class SearchResult {
const note = becca.notes[this.noteId];
if (note.noteId.toLowerCase() === fulltextQuery) {
this.score += 100;
}
if (note.title.toLowerCase() === fulltextQuery) {
this.score += 100; // high reward for exact match #3470
}

View File

@ -1,5 +1,5 @@
/**
* Sync makes process can make data intermittently inconsistent. Processes which require strong data consistency
* Sync process can make data intermittently inconsistent. Processes which require strong data consistency
* (like consistency checks) can use this mutex to make sure sync isn't currently running.
*/

View File

@ -68,43 +68,28 @@ function updateEntity(remoteEC, remoteEntityRow, instanceId, updateContext) {
function updateNormalEntity(remoteEC, remoteEntityRow, instanceId, updateContext) {
const localEC = sql.getRow(`SELECT * FROM entity_changes WHERE entityName = ? AND entityId = ?`, [remoteEC.entityName, remoteEC.entityId]);
if (!localEC?.isErased && remoteEC.isErased) {
eraseEntity(remoteEC, instanceId);
updateContext.erased++;
return true;
} else if (localEC?.isErased && !remoteEC.isErased) {
// on this side, we can't unerase the entity, so force the entity to be erased on the other side.
entityChangesService.putEntityChangeForOtherInstances(localEC);
return false;
} else if (localEC?.isErased && remoteEC.isErased) {
updateContext.alreadyErased++;
return false;
}
if (!localEC || localEC.utcDateChanged <= remoteEC.utcDateChanged) {
if (remoteEC.isErased) {
if (localEC?.isErased) {
eraseEntity(remoteEC); // make sure it's erased anyway
updateContext.alreadyErased++;
return false; // we won't save entitychange in this case
} else {
eraseEntity(remoteEC);
updateContext.erased++;
}
} else {
if (!remoteEntityRow) {
throw new Error(`Empty entity row for: ${JSON.stringify(remoteEC)}`);
}
if (remoteEC.entityName === 'blobs' && remoteEntityRow.content !== null) {
// we always use a Buffer object which is different from normal saving - there we use a simple string type for
// "string notes". The problem is that in general, it's not possible to detect whether a blob content
// is string note or note (syncs can arrive out of order)
remoteEntityRow.content = Buffer.from(remoteEntityRow.content, 'base64');
if (remoteEntityRow.content.byteLength === 0) {
// there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
// (possibly not a problem anymore with the newer better-sqlite3)
remoteEntityRow.content = "";
}
}
preProcessContent(remoteEC, remoteEntityRow);
sql.replace(remoteEC.entityName, remoteEntityRow);
updateContext.updated[remoteEC.entityName] = updateContext.updated[remoteEC.entityName] || [];
updateContext.updated[remoteEC.entityName].push(remoteEC.entityId);
}
if (!localEC || localEC.utcDateChanged < remoteEC.utcDateChanged || localEC.hash !== remoteEC.hash) {
entityChangesService.putEntityChangeWithInstanceId(remoteEC, instanceId);
@ -121,6 +106,21 @@ function updateNormalEntity(remoteEC, remoteEntityRow, instanceId, updateContext
return false;
}
function preProcessContent(remoteEC, remoteEntityRow) {
if (remoteEC.entityName === 'blobs' && remoteEntityRow.content !== null) {
// we always use a Buffer object which is different from normal saving - there we use a simple string type for
// "string notes". The problem is that in general, it's not possible to detect whether a blob content
// is string note or note (syncs can arrive out of order)
remoteEntityRow.content = Buffer.from(remoteEntityRow.content, 'base64');
if (remoteEntityRow.content.byteLength === 0) {
// there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
// (possibly not a problem anymore with the newer better-sqlite3)
remoteEntityRow.content = "";
}
}
}
function updateNoteReordering(remoteEC, remoteEntityRow, instanceId) {
if (!remoteEntityRow) {
throw new Error(`Empty note_reordering body for: ${JSON.stringify(remoteEC)}`);
@ -135,7 +135,7 @@ function updateNoteReordering(remoteEC, remoteEntityRow, instanceId) {
return true;
}
function eraseEntity(entityChange, instanceId) {
function eraseEntity(entityChange) {
const {entityName, entityId} = entityChange;
const entityNames = [
@ -155,8 +155,6 @@ function eraseEntity(entityChange, instanceId) {
const primaryKeyName = entityConstructor.getEntityFromEntityName(entityName).primaryKeyName;
sql.execute(`DELETE FROM ${entityName} WHERE ${primaryKeyName} = ?`, [entityId]);
entityChangesService.putEntityChangeWithInstanceId(entityChange, instanceId);
}
function logUpdateContext(updateContext) {

View File

@ -68,8 +68,6 @@
// https://stackoverflow.com/a/73731646/944162
function isMobile() {
if ('maxTouchPoints' in navigator) return navigator.maxTouchPoints > 0;
const mQ = matchMedia?.('(pointer:coarse)');
if (mQ?.media === '(pointer:coarse)') return !!mQ.matches;