gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

WIP partial sync requests

Browse Source
This commit is contained in:
zadam 2021-01-10 21:56:40 +01:00
parent cd653b9f0c
commit 7f8b19aee4
6 changed files with 111 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
package-lock.json generated
View File

@ -1569,16 +1569,16 @@
"integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="
}, },
"browserslist": { "browserslist": {
"version": "4.16.0", "version": "4.16.1",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.0.tgz", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.1.tgz",
"integrity": "sha512-/j6k8R0p3nxOC6kx5JGAxsnhc9ixaWJfYc+TNTzxg6+ARaESAvQGV7h0uNOB4t+pLQJZWzcrMxXOxjgsCj3dqQ==", "integrity": "sha512-UXhDrwqsNcpTYJBTZsbGATDxZbiVDsx6UjpmRUmtnP10pr8wAYr5LgFoEFw9ixriQH2mv/NX2SfGzE/o8GndLA==",
"dev": true, "dev": true,
"requires": { "requires": {
"caniuse-lite": "^1.0.30001165", "caniuse-lite": "^1.0.30001173",
"colorette": "^1.2.1", "colorette": "^1.2.1",
"electron-to-chromium": "^1.3.621", "electron-to-chromium": "^1.3.634",
"escalade": "^3.1.1", "escalade": "^3.1.1",
"node-releases": "^1.1.67" "node-releases": "^1.1.69"
} }
}, },
"buffer": { "buffer": {
@ -1792,9 +1792,9 @@
"dev": true "dev": true
}, },
"caniuse-lite": { "caniuse-lite": {
"version": "1.0.30001170", "version": "1.0.30001173",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001170.tgz", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001173.tgz",
"integrity": "sha512-Dd4d/+0tsK0UNLrZs3CvNukqalnVTRrxb5mcQm8rHL49t7V5ZaTygwXkrq+FB+dVDf++4ri8eJnFEJAB8332PA==", "integrity": "sha512-R3aqmjrICdGCTAnSXtNyvWYMK3YtV5jwudbq0T7nN9k4kmE4CBuwPqyJ+KBzepSTh0huivV2gLbSMEzTTmfeYw==",
"dev": true "dev": true
}, },
"caseless": { "caseless": {
@ -3209,9 +3209,9 @@
} }
}, },
"electron-to-chromium": { "electron-to-chromium": {
"version": "1.3.633", "version": "1.3.635",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.633.tgz", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.635.tgz",
"integrity": "sha512-bsVCsONiVX1abkWdH7KtpuDAhsQ3N3bjPYhROSAXE78roJKet0Y5wznA14JE9pzbwSZmSMAW6KiKYf1RvbTJkA==", "integrity": "sha512-RRriZOLs9CpW6KTLmgBqyUdnY0QNqqWs0HOtuQGGEMizOTNNn1P7sGRBxARnUeLejOsgwjDyRqT3E/CSst02ZQ==",
"dev": true "dev": true
}, },
"electron-window-state": { "electron-window-state": {
@ -4871,9 +4871,9 @@
} }
}, },
"loader-runner": { "loader-runner": {
"version": "4.1.0", "version": "4.2.0",
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.1.0.tgz", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.2.0.tgz",
"integrity": "sha512-oR4lB4WvwFoC70ocraKhn5nkKSs23t57h9udUgw8o0iH8hMXeEoRuUgfcvgUwAJ1ZpRqBvcou4N2SMvM1DwMrA==", "integrity": "sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw==",
"dev": true "dev": true
}, },
"locate-path": { "locate-path": {
@ -5413,9 +5413,9 @@
} }
}, },
"node-releases": { "node-releases": {
"version": "1.1.67", "version": "1.1.69",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.67.tgz", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.69.tgz",
"integrity": "sha512-V5QF9noGFl3EymEwUYzO+3NTDpGfQB4ve6Qfnzf3UNydMhjQRVPR1DZTuvWiLzaFJYw2fmDwAfnRNEVb64hSIg==", "integrity": "sha512-DGIjo79VDEyAnRlfSqYTsy+yoHd2IOjJiKUozD2MV2D85Vso6Bug56mb9tT/fY5Urt0iqk01H7x+llAruDR2zA==",
"dev": true "dev": true
}, },
"noop-logger": { "noop-logger": {
@ -5596,9 +5596,9 @@
} }
}, },
"open": { "open": {
"version": "7.3.0", "version": "7.3.1",
"resolved": "https://registry.npmjs.org/open/-/open-7.3.0.tgz", "resolved": "https://registry.npmjs.org/open/-/open-7.3.1.tgz",
"integrity": "sha512-mgLwQIx2F/ye9SmbrUkurZCnkoXyXyu9EbHtJZrICjVAJfyMArdHp3KkixGdZx1ZHFPNIwl0DDM1dFFqXbTLZw==", "integrity": "sha512-f2wt9DCBKKjlFbjzGb8MOAW8LH8F0mrs1zc7KTjAJ9PZNQbfenzWbNP1VZJvw6ICMG9r14Ah6yfwPn7T7i646A==",
"requires": { "requires": {
"is-docker": "^2.0.0", "is-docker": "^2.0.0",
"is-wsl": "^2.1.1" "is-wsl": "^2.1.1"
@ -7160,17 +7160,17 @@
} }
}, },
"terser-webpack-plugin": { "terser-webpack-plugin": {
"version": "5.0.3", "version": "5.1.0",
"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.0.3.tgz", "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.0.tgz",
"integrity": "sha512-zFdGk8Lh9ZJGPxxPE6jwysOlATWB8GMW8HcfGULWA/nPal+3VdATflQvSBSLQJRCmYZnfFJl6vkRTiwJGNgPiQ==", "integrity": "sha512-7Hw5b45IslUGsR3rh1WhKlt2EHHIemwrus2Y++8f+36SGBVXruvwuDU1/bgkM44i/x6F24yJk1d+3r+JGtHaOg==",
"dev": true, "dev": true,
"requires": { "requires": {
"jest-worker": "^26.6.1", "jest-worker": "^26.6.2",
"p-limit": "^3.0.2", "p-limit": "^3.1.0",
"schema-utils": "^3.0.0", "schema-utils": "^3.0.0",
"serialize-javascript": "^5.0.1", "serialize-javascript": "^5.0.1",
"source-map": "^0.6.1", "source-map": "^0.6.1",
"terser": "^5.3.8" "terser": "^5.5.1"
}, },
"dependencies": { "dependencies": {
"p-limit": { "p-limit": {
@ -7600,9 +7600,9 @@
"integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==" "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w=="
}, },
"webpack": { "webpack": {
"version": "5.11.1", "version": "5.12.1",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.11.1.tgz", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.12.1.tgz",
"integrity": "sha512-tNUIdAmYJv+nupRs/U/gqmADm6fgrf5xE+rSlSsf2PgsGO7j2WG7ccU6AWNlOJlHFl+HnmXlBmHIkiLf+XA9mQ==", "integrity": "sha512-Bh6hPzUvTLuGZg33xsZLEtAkaEJf9ux29WwGj4IeAGUCy7RE8zhqe4aHN4UqA8yHmHzvhORFH2p9ohB6h6R3yg==",
"dev": true, "dev": true,
"requires": { "requires": {
"@types/eslint-scope": "^3.7.0", "@types/eslint-scope": "^3.7.0",
@ -7620,7 +7620,7 @@
"glob-to-regexp": "^0.4.1", "glob-to-regexp": "^0.4.1",
"graceful-fs": "^4.2.4", "graceful-fs": "^4.2.4",
"json-parse-better-errors": "^1.0.2", "json-parse-better-errors": "^1.0.2",
"loader-runner": "^4.1.0", "loader-runner": "^4.2.0",
"mime-types": "^2.1.27", "mime-types": "^2.1.27",
"neo-async": "^2.6.2", "neo-async": "^2.6.2",
"pkg-dir": "^5.0.0", "pkg-dir": "^5.0.0",

2
package.json
View File

@ -86,7 +86,7 @@
"jsdoc": "3.6.6", "jsdoc": "3.6.6",
"lorem-ipsum": "2.0.3", "lorem-ipsum": "2.0.3",
"rcedit": "3.0.0", "rcedit": "3.0.0",
"webpack": "5.11.1", "webpack": "5.12.1",
"webpack-cli": "4.3.1" "webpack-cli": "4.3.1"
}, },
"optionalDependencies": { "optionalDependencies": {

42
src/routes/api/sync.js
View File

@ -138,14 +138,54 @@ function getChanged(req) {
return ret; return ret;
} }
const partialRequests = {};
function update(req) { function update(req) {
const {sourceId, entities} = req.body; let {body} = req;
const pageCount = parseInt(req.get('pageCount'));
const pageIndex = parseInt(req.get('pageIndex'));
if (pageCount !== 1) {
const requestId = req.get('requestId');
if (pageIndex === 0) {
partialRequests[requestId] = {
createdAt: Date.now(),
payload: ''
};
}
if (!partialRequests[requestId]) {
throw new Error(`Partial request ${requestId}, index ${pageIndex} of ${pageCount} of pages does not have expected record.`);
}
partialRequests[requestId].payload += req.body;
if (pageIndex !== pageCount - 1) {
return;
}
else {
body = JSON.parse(partialRequests[requestId].payload);
delete partialRequests[requestId];
}
}
const {sourceId, entities} = body;
for (const {entityChange, entity} of entities) { for (const {entityChange, entity} of entities) {
syncUpdateService.updateEntity(entityChange, entity, sourceId); syncUpdateService.updateEntity(entityChange, entity, sourceId);
} }
} }
setInterval(() => {
for (const key in partialRequests) {
if (partialRequests[key].createdAt - Date.now() > 5 * 60 * 1000) {
delete partialRequests[key];
}
}
}, 60 * 1000);
function syncFinished() { function syncFinished() {
// after first sync finishes, the application is ready to be used // after first sync finishes, the application is ready to be used
// this is meaningless but at the same time harmless (idempotent) for further syncs // this is meaningless but at the same time harmless (idempotent) for further syncs

2
src/services/app_info.js
View File

@ -5,7 +5,7 @@ const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir'); const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 178; const APP_DB_VERSION = 178;
const SYNC_VERSION = 18; const SYNC_VERSION = 19;
const CLIPPER_PROTOCOL_VERSION = "1.0"; const CLIPPER_PROTOCOL_VERSION = "1.0";
module.exports = { module.exports = {

16
src/services/request.js
View File

@ -16,15 +16,25 @@ function exec(opts) {
opts.proxy = null; opts.proxy = null;
} }
if (!opts.paging) {
opts.paging = {
pageCount: 1,
pageIndex: 0
};
}
const proxyAgent = getProxyAgent(opts); const proxyAgent = getProxyAgent(opts);
const parsedTargetUrl = url.parse(opts.url); const parsedTargetUrl = url.parse(opts.url);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
try { try {
const headers = { const headers = Object.assign({
Cookie: (opts.cookieJar && opts.cookieJar.header) || "", Cookie: (opts.cookieJar && opts.cookieJar.header) || "",
'Content-Type': 'application/json' 'Content-Type': opts.paging.pageCount === 1 ? 'application/json' : 'text/plain',
}; pageCount: opts.pageCount,
pageIndex: opts.pageIndex,
requestId: opts.requestId
}, opts.headers || {});
if (opts.auth) { if (opts.auth) {
const token = Buffer.from(opts.auth.user + ":" + opts.auth.pass).toString('base64'); const token = Buffer.from(opts.auth.user + ":" + opts.auth.pass).toString('base64');

18
src/services/sync.js
View File

@ -253,19 +253,33 @@ async function checkContentHash(syncContext) {
return failedChecks.length > 0; return failedChecks.length > 0;
} }
async function syncRequest(syncContext, method, requestPath, body) { async function syncRequest(syncContext, method, requestPath, body = '') {
const timeout = syncOptions.getSyncTimeout(); const timeout = syncOptions.getSyncTimeout();
let response;
const requestId = utils.randomString(10);
const pageCount = Math.ceil(body.length / 1000000);
for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) {
const opts = { const opts = {
method, method,
url: syncOptions.getSyncServerHost() + requestPath, url: syncOptions.getSyncServerHost() + requestPath,
cookieJar: syncContext.cookieJar, cookieJar: syncContext.cookieJar,
timeout: timeout, timeout: timeout,
paging: {
pageIndex,
pageCount,
requestId
},
body, body,
proxy: proxyToggle ? syncOptions.getSyncProxy() : null proxy: proxyToggle ? syncOptions.getSyncProxy() : null
}; };
return await utils.timeLimit(request.exec(opts), timeout); response = await utils.timeLimit(request.exec(opts), timeout);
}
} }
function getEntityChangeRow(entityName, entityId) { function getEntityChangeRow(entityName, entityId) {