gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix partial syncs

Browse Source
This commit is contained in:
zadam 2021-01-14 23:22:13 +01:00
parent cbc1f63732
commit 199b49c1f7
5 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/app.js
View File

@ -23,6 +23,7 @@ app.use(helmet({
contentSecurityPolicy: false contentSecurityPolicy: false
})); }));
app.use(bodyParser.text({limit: '500mb'}));
app.use(bodyParser.json({limit: '500mb'})); app.use(bodyParser.json({limit: '500mb'}));
app.use(bodyParser.urlencoded({extended: false})); app.use(bodyParser.urlencoded({extended: false}));
app.use(cookieParser()); app.use(cookieParser());

2
src/services/content_hash.js
View File

@ -34,7 +34,7 @@ function getSectorHashes(tableName, primaryKeyName, whereBranch) {
function getEntityHashes() { function getEntityHashes() {
const startTime = new Date(); const startTime = new Date();
const hashRows = sql.getRows(`SELECT entityName, entityId, hash FROM entity_changes`); const hashRows = sql.getRows(`SELECT entityName, entityId, hash FROM entity_changes WHERE isSynced = 1`);
// sorting is faster in memory // sorting is faster in memory
// sorting by entityId is enough, hashes will be segmented by entityName later on anyway // sorting by entityId is enough, hashes will be segmented by entityName later on anyway

2
src/services/entity_changes.js
View File

@ -1,5 +1,4 @@
const sql = require('./sql'); const sql = require('./sql');
const repository = require('./repository');
const sourceIdService = require('./source_id'); const sourceIdService = require('./source_id');
const dateUtils = require('./date_utils'); const dateUtils = require('./date_utils');
const log = require('./log'); const log = require('./log');
@ -39,6 +38,7 @@ function moveEntityChangeToTop(entityName, entityId) {
function addEntityChangesForSector(entityName, entityPrimaryKey, sector) { function addEntityChangesForSector(entityName, entityPrimaryKey, sector) {
const startTime = Date.now(); const startTime = Date.now();
const repository = require('./repository');
sql.transactional(() => { sql.transactional(() => {
const entityIds = sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]); const entityIds = sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]);

8
src/services/sync.js
View File

@ -253,6 +253,8 @@ async function checkContentHash(syncContext) {
return failedChecks.length > 0; return failedChecks.length > 0;
} }
const PAGE_SIZE = 1000000;
async function syncRequest(syncContext, method, requestPath, body) { async function syncRequest(syncContext, method, requestPath, body) {
body = body ? JSON.stringify(body) : ''; body = body ? JSON.stringify(body) : '';
@ -261,7 +263,7 @@ async function syncRequest(syncContext, method, requestPath, body) {
let response; let response;
const requestId = utils.randomString(10); const requestId = utils.randomString(10);
const pageCount = Math.min(1, Math.ceil(body.length / 1000000)); const pageCount = Math.max(1, Math.ceil(body.length / PAGE_SIZE));
for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) {
const opts = { const opts = {
@ -274,13 +276,11 @@ async function syncRequest(syncContext, method, requestPath, body) {
pageCount, pageCount,
requestId requestId
}, },
body, body: body.substr(pageIndex * PAGE_SIZE, Math.min(PAGE_SIZE, body.length - pageIndex * PAGE_SIZE)),
proxy: proxyToggle ? syncOptions.getSyncProxy() : null proxy: proxyToggle ? syncOptions.getSyncProxy() : null
}; };
response = await utils.timeLimit(request.exec(opts), timeout); response = await utils.timeLimit(request.exec(opts), timeout);
console.log("response", response);
} }
return response; return response;

23
src/services/sync_update.js
View File

@ -23,35 +23,36 @@ function updateEntity(entityChange, entity, sourceId) {
} }
} }
function updateNormalEntity(entityChange, entity, sourceId) { function updateNormalEntity(remoteEntityChange, entity, sourceId) {
const {utcDateChanged, hash, isErased} = sql.getRow(` const localEntityChange = sql.getRow(`
SELECT utcDateChanged, hash, isErased SELECT utcDateChanged, hash, isErased
FROM entity_changes FROM entity_changes
WHERE entityName = ? AND entityId = ?`, [entityChange.entityName, entityChange.entityId]); WHERE entityName = ? AND entityId = ?`, [remoteEntityChange.entityName, remoteEntityChange.entityId]);
if (!isErased && entityChange.isErased) { if (localEntityChange && !localEntityChange.isErased && remoteEntityChange.isErased) {
sql.transactional(() => { sql.transactional(() => {
const primaryKey = entityConstructor.getEntityFromEntityName(entityName).primaryKeyName; const primaryKey = entityConstructor.getEntityFromEntityName(entityName).primaryKeyName;
sql.execute(`DELETE FROM ${entityChange.entityName} WHERE ${primaryKey} = ?`, entityChange.entityId); sql.execute(`DELETE FROM ${remoteEntityChange.entityName} WHERE ${primaryKey} = ?`, remoteEntityChange.entityId);
entityChangesService.addEntityChange(entityChange, sourceId); entityChangesService.addEntityChange(remoteEntityChange, sourceId);
}); });
return true; return true;
} }
if (utcDateChanged < entityChange.utcDateChanged if (!localEntityChange
|| hash !== entityChange.hash // sync error, we should still update || localEntityChange.utcDateChanged < remoteEntityChange.utcDateChanged
|| localEntityChange.hash !== remoteEntityChange.hash // sync error, we should still update
) { ) {
if (['note_contents', 'note_revision_contents'].includes(entityChange.entityName)) { if (['note_contents', 'note_revision_contents'].includes(remoteEntityChange.entityName)) {
entity.content = handleContent(entity.content); entity.content = handleContent(entity.content);
} }
sql.transactional(() => { sql.transactional(() => {
sql.replace(entityChange.entityName, entity); sql.replace(remoteEntityChange.entityName, entity);
entityChangesService.addEntityChange(entityChange, sourceId); entityChangesService.addEntityChange(remoteEntityChange, sourceId);
}); });
return true; return true;