mirror of
https://github.com/zadam/trilium.git
synced 2025-10-29 02:28:57 +01:00
WIP partial sync requests
This commit is contained in:
zadam
parent
cd653b9f0c
commit
7f8b19aee4
62
package-lock.json
generated
62
package-lock.json
generated
@ -1569,16 +1569,16 @@
|
||||
"integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="
|
||||
},
|
||||
"browserslist": {
|
||||
"version": "4.16.0",
|
||||
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.0.tgz",
|
||||
"integrity": "sha512-/j6k8R0p3nxOC6kx5JGAxsnhc9ixaWJfYc+TNTzxg6+ARaESAvQGV7h0uNOB4t+pLQJZWzcrMxXOxjgsCj3dqQ==",
|
||||
"version": "4.16.1",
|
||||
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.1.tgz",
|
||||
"integrity": "sha512-UXhDrwqsNcpTYJBTZsbGATDxZbiVDsx6UjpmRUmtnP10pr8wAYr5LgFoEFw9ixriQH2mv/NX2SfGzE/o8GndLA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"caniuse-lite": "^1.0.30001165",
|
||||
"caniuse-lite": "^1.0.30001173",
|
||||
"colorette": "^1.2.1",
|
||||
"electron-to-chromium": "^1.3.621",
|
||||
"electron-to-chromium": "^1.3.634",
|
||||
"escalade": "^3.1.1",
|
||||
"node-releases": "^1.1.67"
|
||||
"node-releases": "^1.1.69"
|
||||
}
|
||||
},
|
||||
"buffer": {
|
||||
@ -1792,9 +1792,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"caniuse-lite": {
|
||||
"version": "1.0.30001170",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001170.tgz",
|
||||
"integrity": "sha512-Dd4d/+0tsK0UNLrZs3CvNukqalnVTRrxb5mcQm8rHL49t7V5ZaTygwXkrq+FB+dVDf++4ri8eJnFEJAB8332PA==",
|
||||
"version": "1.0.30001173",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001173.tgz",
|
||||
"integrity": "sha512-R3aqmjrICdGCTAnSXtNyvWYMK3YtV5jwudbq0T7nN9k4kmE4CBuwPqyJ+KBzepSTh0huivV2gLbSMEzTTmfeYw==",
|
||||
"dev": true
|
||||
},
|
||||
"caseless": {
|
||||
@ -3209,9 +3209,9 @@
|
||||
}
|
||||
},
|
||||
"electron-to-chromium": {
|
||||
"version": "1.3.633",
|
||||
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.633.tgz",
|
||||
"integrity": "sha512-bsVCsONiVX1abkWdH7KtpuDAhsQ3N3bjPYhROSAXE78roJKet0Y5wznA14JE9pzbwSZmSMAW6KiKYf1RvbTJkA==",
|
||||
"version": "1.3.635",
|
||||
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.635.tgz",
|
||||
"integrity": "sha512-RRriZOLs9CpW6KTLmgBqyUdnY0QNqqWs0HOtuQGGEMizOTNNn1P7sGRBxARnUeLejOsgwjDyRqT3E/CSst02ZQ==",
|
||||
"dev": true
|
||||
},
|
||||
"electron-window-state": {
|
||||
@ -4871,9 +4871,9 @@
|
||||
}
|
||||
},
|
||||
"loader-runner": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.1.0.tgz",
|
||||
"integrity": "sha512-oR4lB4WvwFoC70ocraKhn5nkKSs23t57h9udUgw8o0iH8hMXeEoRuUgfcvgUwAJ1ZpRqBvcou4N2SMvM1DwMrA==",
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.2.0.tgz",
|
||||
"integrity": "sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw==",
|
||||
"dev": true
|
||||
},
|
||||
"locate-path": {
|
||||
@ -5413,9 +5413,9 @@
|
||||
}
|
||||
},
|
||||
"node-releases": {
|
||||
"version": "1.1.67",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.67.tgz",
|
||||
"integrity": "sha512-V5QF9noGFl3EymEwUYzO+3NTDpGfQB4ve6Qfnzf3UNydMhjQRVPR1DZTuvWiLzaFJYw2fmDwAfnRNEVb64hSIg==",
|
||||
"version": "1.1.69",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.69.tgz",
|
||||
"integrity": "sha512-DGIjo79VDEyAnRlfSqYTsy+yoHd2IOjJiKUozD2MV2D85Vso6Bug56mb9tT/fY5Urt0iqk01H7x+llAruDR2zA==",
|
||||
"dev": true
|
||||
},
|
||||
"noop-logger": {
|
||||
@ -5596,9 +5596,9 @@
|
||||
}
|
||||
},
|
||||
"open": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/open/-/open-7.3.0.tgz",
|
||||
"integrity": "sha512-mgLwQIx2F/ye9SmbrUkurZCnkoXyXyu9EbHtJZrICjVAJfyMArdHp3KkixGdZx1ZHFPNIwl0DDM1dFFqXbTLZw==",
|
||||
"version": "7.3.1",
|
||||
"resolved": "https://registry.npmjs.org/open/-/open-7.3.1.tgz",
|
||||
"integrity": "sha512-f2wt9DCBKKjlFbjzGb8MOAW8LH8F0mrs1zc7KTjAJ9PZNQbfenzWbNP1VZJvw6ICMG9r14Ah6yfwPn7T7i646A==",
|
||||
"requires": {
|
||||
"is-docker": "^2.0.0",
|
||||
"is-wsl": "^2.1.1"
|
||||
@ -7160,17 +7160,17 @@
|
||||
}
|
||||
},
|
||||
"terser-webpack-plugin": {
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.0.3.tgz",
|
||||
"integrity": "sha512-zFdGk8Lh9ZJGPxxPE6jwysOlATWB8GMW8HcfGULWA/nPal+3VdATflQvSBSLQJRCmYZnfFJl6vkRTiwJGNgPiQ==",
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.0.tgz",
|
||||
"integrity": "sha512-7Hw5b45IslUGsR3rh1WhKlt2EHHIemwrus2Y++8f+36SGBVXruvwuDU1/bgkM44i/x6F24yJk1d+3r+JGtHaOg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"jest-worker": "^26.6.1",
|
||||
"p-limit": "^3.0.2",
|
||||
"jest-worker": "^26.6.2",
|
||||
"p-limit": "^3.1.0",
|
||||
"schema-utils": "^3.0.0",
|
||||
"serialize-javascript": "^5.0.1",
|
||||
"source-map": "^0.6.1",
|
||||
"terser": "^5.3.8"
|
||||
"terser": "^5.5.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"p-limit": {
|
||||
@ -7600,9 +7600,9 @@
|
||||
"integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w=="
|
||||
},
|
||||
"webpack": {
|
||||
"version": "5.11.1",
|
||||
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.11.1.tgz",
|
||||
"integrity": "sha512-tNUIdAmYJv+nupRs/U/gqmADm6fgrf5xE+rSlSsf2PgsGO7j2WG7ccU6AWNlOJlHFl+HnmXlBmHIkiLf+XA9mQ==",
|
||||
"version": "5.12.1",
|
||||
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.12.1.tgz",
|
||||
"integrity": "sha512-Bh6hPzUvTLuGZg33xsZLEtAkaEJf9ux29WwGj4IeAGUCy7RE8zhqe4aHN4UqA8yHmHzvhORFH2p9ohB6h6R3yg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/eslint-scope": "^3.7.0",
|
||||
@ -7620,7 +7620,7 @@
|
||||
"glob-to-regexp": "^0.4.1",
|
||||
"graceful-fs": "^4.2.4",
|
||||
"json-parse-better-errors": "^1.0.2",
|
||||
"loader-runner": "^4.1.0",
|
||||
"loader-runner": "^4.2.0",
|
||||
"mime-types": "^2.1.27",
|
||||
"neo-async": "^2.6.2",
|
||||
"pkg-dir": "^5.0.0",
|
||||
|
||||
@ -86,7 +86,7 @@
|
||||
"jsdoc": "3.6.6",
|
||||
"lorem-ipsum": "2.0.3",
|
||||
"rcedit": "3.0.0",
|
||||
"webpack": "5.11.1",
|
||||
"webpack": "5.12.1",
|
||||
"webpack-cli": "4.3.1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
|
||||
@ -138,14 +138,54 @@ function getChanged(req) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
const partialRequests = {};
|
||||
|
||||
function update(req) {
|
||||
const {sourceId, entities} = req.body;
|
||||
let {body} = req;
|
||||
|
||||
const pageCount = parseInt(req.get('pageCount'));
|
||||
const pageIndex = parseInt(req.get('pageIndex'));
|
||||
|
||||
if (pageCount !== 1) {
|
||||
const requestId = req.get('requestId');
|
||||
|
||||
if (pageIndex === 0) {
|
||||
partialRequests[requestId] = {
|
||||
createdAt: Date.now(),
|
||||
payload: ''
|
||||
};
|
||||
}
|
||||
|
||||
if (!partialRequests[requestId]) {
|
||||
throw new Error(`Partial request ${requestId}, index ${pageIndex} of ${pageCount} of pages does not have expected record.`);
|
||||
}
|
||||
|
||||
partialRequests[requestId].payload += req.body;
|
||||
|
||||
if (pageIndex !== pageCount - 1) {
|
||||
return;
|
||||
}
|
||||
else {
|
||||
body = JSON.parse(partialRequests[requestId].payload);
|
||||
delete partialRequests[requestId];
|
||||
}
|
||||
}
|
||||
|
||||
const {sourceId, entities} = body;
|
||||
|
||||
for (const {entityChange, entity} of entities) {
|
||||
syncUpdateService.updateEntity(entityChange, entity, sourceId);
|
||||
}
|
||||
}
|
||||
|
||||
setInterval(() => {
|
||||
for (const key in partialRequests) {
|
||||
if (partialRequests[key].createdAt - Date.now() > 5 * 60 * 1000) {
|
||||
delete partialRequests[key];
|
||||
}
|
||||
}
|
||||
}, 60 * 1000);
|
||||
|
||||
function syncFinished() {
|
||||
// after first sync finishes, the application is ready to be used
|
||||
// this is meaningless but at the same time harmless (idempotent) for further syncs
|
||||
|
||||
@ -5,7 +5,7 @@ const packageJson = require('../../package');
|
||||
const {TRILIUM_DATA_DIR} = require('./data_dir');
|
||||
|
||||
const APP_DB_VERSION = 178;
|
||||
const SYNC_VERSION = 18;
|
||||
const SYNC_VERSION = 19;
|
||||
const CLIPPER_PROTOCOL_VERSION = "1.0";
|
||||
|
||||
module.exports = {
|
||||
|
||||
@ -16,15 +16,25 @@ function exec(opts) {
|
||||
opts.proxy = null;
|
||||
}
|
||||
|
||||
if (!opts.paging) {
|
||||
opts.paging = {
|
||||
pageCount: 1,
|
||||
pageIndex: 0
|
||||
};
|
||||
}
|
||||
|
||||
const proxyAgent = getProxyAgent(opts);
|
||||
const parsedTargetUrl = url.parse(opts.url);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const headers = {
|
||||
const headers = Object.assign({
|
||||
Cookie: (opts.cookieJar && opts.cookieJar.header) || "",
|
||||
'Content-Type': 'application/json'
|
||||
};
|
||||
'Content-Type': opts.paging.pageCount === 1 ? 'application/json' : 'text/plain',
|
||||
pageCount: opts.pageCount,
|
||||
pageIndex: opts.pageIndex,
|
||||
requestId: opts.requestId
|
||||
}, opts.headers || {});
|
||||
|
||||
if (opts.auth) {
|
||||
const token = Buffer.from(opts.auth.user + ":" + opts.auth.pass).toString('base64');
|
||||
|
||||
@ -253,19 +253,33 @@ async function checkContentHash(syncContext) {
|
||||
return failedChecks.length > 0;
|
||||
}
|
||||
|
||||
async function syncRequest(syncContext, method, requestPath, body) {
|
||||
async function syncRequest(syncContext, method, requestPath, body = '') {
|
||||
const timeout = syncOptions.getSyncTimeout();
|
||||
|
||||
const opts = {
|
||||
method,
|
||||
url: syncOptions.getSyncServerHost() + requestPath,
|
||||
cookieJar: syncContext.cookieJar,
|
||||
timeout: timeout,
|
||||
body,
|
||||
proxy: proxyToggle ? syncOptions.getSyncProxy() : null
|
||||
};
|
||||
let response;
|
||||
|
||||
const requestId = utils.randomString(10);
|
||||
const pageCount = Math.ceil(body.length / 1000000);
|
||||
|
||||
for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) {
|
||||
const opts = {
|
||||
method,
|
||||
url: syncOptions.getSyncServerHost() + requestPath,
|
||||
cookieJar: syncContext.cookieJar,
|
||||
timeout: timeout,
|
||||
paging: {
|
||||
pageIndex,
|
||||
pageCount,
|
||||
requestId
|
||||
},
|
||||
body,
|
||||
proxy: proxyToggle ? syncOptions.getSyncProxy() : null
|
||||
};
|
||||
|
||||
response = await utils.timeLimit(request.exec(opts), timeout);
|
||||
}
|
||||
|
||||
|
||||
return await utils.timeLimit(request.exec(opts), timeout);
|
||||
}
|
||||
|
||||
function getEntityChangeRow(entityName, entityId) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user