From 7f8b19aee4d0272c8c4a6b5a3d3704007e8d028e Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 10 Jan 2021 21:56:40 +0100 Subject: [PATCH] WIP partial sync requests --- package-lock.json | 62 ++++++++++++++++++++-------------------- package.json | 2 +- src/routes/api/sync.js | 42 ++++++++++++++++++++++++++- src/services/app_info.js | 2 +- src/services/request.js | 16 +++++++++-- src/services/sync.js | 34 +++++++++++++++------- 6 files changed, 111 insertions(+), 47 deletions(-) diff --git a/package-lock.json b/package-lock.json index aa759f4a7..5a4153f72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1569,16 +1569,16 @@ "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" }, "browserslist": { - "version": "4.16.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.0.tgz", - "integrity": "sha512-/j6k8R0p3nxOC6kx5JGAxsnhc9ixaWJfYc+TNTzxg6+ARaESAvQGV7h0uNOB4t+pLQJZWzcrMxXOxjgsCj3dqQ==", + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.1.tgz", + "integrity": "sha512-UXhDrwqsNcpTYJBTZsbGATDxZbiVDsx6UjpmRUmtnP10pr8wAYr5LgFoEFw9ixriQH2mv/NX2SfGzE/o8GndLA==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001165", + "caniuse-lite": "^1.0.30001173", "colorette": "^1.2.1", - "electron-to-chromium": "^1.3.621", + "electron-to-chromium": "^1.3.634", "escalade": "^3.1.1", - "node-releases": "^1.1.67" + "node-releases": "^1.1.69" } }, "buffer": { @@ -1792,9 +1792,9 @@ "dev": true }, "caniuse-lite": { - "version": "1.0.30001170", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001170.tgz", - "integrity": "sha512-Dd4d/+0tsK0UNLrZs3CvNukqalnVTRrxb5mcQm8rHL49t7V5ZaTygwXkrq+FB+dVDf++4ri8eJnFEJAB8332PA==", + "version": "1.0.30001173", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001173.tgz", + "integrity": "sha512-R3aqmjrICdGCTAnSXtNyvWYMK3YtV5jwudbq0T7nN9k4kmE4CBuwPqyJ+KBzepSTh0huivV2gLbSMEzTTmfeYw==", "dev": true }, "caseless": { @@ -3209,9 +3209,9 @@ } }, "electron-to-chromium": { - "version": "1.3.633", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.633.tgz", - "integrity": "sha512-bsVCsONiVX1abkWdH7KtpuDAhsQ3N3bjPYhROSAXE78roJKet0Y5wznA14JE9pzbwSZmSMAW6KiKYf1RvbTJkA==", + "version": "1.3.635", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.635.tgz", + "integrity": "sha512-RRriZOLs9CpW6KTLmgBqyUdnY0QNqqWs0HOtuQGGEMizOTNNn1P7sGRBxARnUeLejOsgwjDyRqT3E/CSst02ZQ==", "dev": true }, "electron-window-state": { @@ -4871,9 +4871,9 @@ } }, "loader-runner": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.1.0.tgz", - "integrity": "sha512-oR4lB4WvwFoC70ocraKhn5nkKSs23t57h9udUgw8o0iH8hMXeEoRuUgfcvgUwAJ1ZpRqBvcou4N2SMvM1DwMrA==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.2.0.tgz", + "integrity": "sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw==", "dev": true }, "locate-path": { @@ -5413,9 +5413,9 @@ } }, "node-releases": { - "version": "1.1.67", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.67.tgz", - "integrity": "sha512-V5QF9noGFl3EymEwUYzO+3NTDpGfQB4ve6Qfnzf3UNydMhjQRVPR1DZTuvWiLzaFJYw2fmDwAfnRNEVb64hSIg==", + "version": "1.1.69", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.69.tgz", + "integrity": "sha512-DGIjo79VDEyAnRlfSqYTsy+yoHd2IOjJiKUozD2MV2D85Vso6Bug56mb9tT/fY5Urt0iqk01H7x+llAruDR2zA==", "dev": true }, "noop-logger": { @@ -5596,9 +5596,9 @@ } }, "open": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/open/-/open-7.3.0.tgz", - "integrity": "sha512-mgLwQIx2F/ye9SmbrUkurZCnkoXyXyu9EbHtJZrICjVAJfyMArdHp3KkixGdZx1ZHFPNIwl0DDM1dFFqXbTLZw==", + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/open/-/open-7.3.1.tgz", + "integrity": "sha512-f2wt9DCBKKjlFbjzGb8MOAW8LH8F0mrs1zc7KTjAJ9PZNQbfenzWbNP1VZJvw6ICMG9r14Ah6yfwPn7T7i646A==", "requires": { "is-docker": "^2.0.0", "is-wsl": "^2.1.1" @@ -7160,17 +7160,17 @@ } }, "terser-webpack-plugin": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.0.3.tgz", - "integrity": "sha512-zFdGk8Lh9ZJGPxxPE6jwysOlATWB8GMW8HcfGULWA/nPal+3VdATflQvSBSLQJRCmYZnfFJl6vkRTiwJGNgPiQ==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.0.tgz", + "integrity": "sha512-7Hw5b45IslUGsR3rh1WhKlt2EHHIemwrus2Y++8f+36SGBVXruvwuDU1/bgkM44i/x6F24yJk1d+3r+JGtHaOg==", "dev": true, "requires": { - "jest-worker": "^26.6.1", - "p-limit": "^3.0.2", + "jest-worker": "^26.6.2", + "p-limit": "^3.1.0", "schema-utils": "^3.0.0", "serialize-javascript": "^5.0.1", "source-map": "^0.6.1", - "terser": "^5.3.8" + "terser": "^5.5.1" }, "dependencies": { "p-limit": { @@ -7600,9 +7600,9 @@ "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==" }, "webpack": { - "version": "5.11.1", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.11.1.tgz", - "integrity": "sha512-tNUIdAmYJv+nupRs/U/gqmADm6fgrf5xE+rSlSsf2PgsGO7j2WG7ccU6AWNlOJlHFl+HnmXlBmHIkiLf+XA9mQ==", + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.12.1.tgz", + "integrity": "sha512-Bh6hPzUvTLuGZg33xsZLEtAkaEJf9ux29WwGj4IeAGUCy7RE8zhqe4aHN4UqA8yHmHzvhORFH2p9ohB6h6R3yg==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.0", @@ -7620,7 +7620,7 @@ "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.4", "json-parse-better-errors": "^1.0.2", - "loader-runner": "^4.1.0", + "loader-runner": "^4.2.0", "mime-types": "^2.1.27", "neo-async": "^2.6.2", "pkg-dir": "^5.0.0", diff --git a/package.json b/package.json index 9e63a9f54..3bc6959f6 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ "jsdoc": "3.6.6", "lorem-ipsum": "2.0.3", "rcedit": "3.0.0", - "webpack": "5.11.1", + "webpack": "5.12.1", "webpack-cli": "4.3.1" }, "optionalDependencies": { diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js index 9efb24cc9..58e56361e 100644 --- a/src/routes/api/sync.js +++ b/src/routes/api/sync.js @@ -138,14 +138,54 @@ function getChanged(req) { return ret; } +const partialRequests = {}; + function update(req) { - const {sourceId, entities} = req.body; + let {body} = req; + + const pageCount = parseInt(req.get('pageCount')); + const pageIndex = parseInt(req.get('pageIndex')); + + if (pageCount !== 1) { + const requestId = req.get('requestId'); + + if (pageIndex === 0) { + partialRequests[requestId] = { + createdAt: Date.now(), + payload: '' + }; + } + + if (!partialRequests[requestId]) { + throw new Error(`Partial request ${requestId}, index ${pageIndex} of ${pageCount} of pages does not have expected record.`); + } + + partialRequests[requestId].payload += req.body; + + if (pageIndex !== pageCount - 1) { + return; + } + else { + body = JSON.parse(partialRequests[requestId].payload); + delete partialRequests[requestId]; + } + } + + const {sourceId, entities} = body; for (const {entityChange, entity} of entities) { syncUpdateService.updateEntity(entityChange, entity, sourceId); } } +setInterval(() => { + for (const key in partialRequests) { + if (partialRequests[key].createdAt - Date.now() > 5 * 60 * 1000) { + delete partialRequests[key]; + } + } +}, 60 * 1000); + function syncFinished() { // after first sync finishes, the application is ready to be used // this is meaningless but at the same time harmless (idempotent) for further syncs diff --git a/src/services/app_info.js b/src/services/app_info.js index 688b49c51..23464fe6d 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -5,7 +5,7 @@ const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); const APP_DB_VERSION = 178; -const SYNC_VERSION = 18; +const SYNC_VERSION = 19; const CLIPPER_PROTOCOL_VERSION = "1.0"; module.exports = { diff --git a/src/services/request.js b/src/services/request.js index 1b9b3cd2b..ff5278d29 100644 --- a/src/services/request.js +++ b/src/services/request.js @@ -16,15 +16,25 @@ function exec(opts) { opts.proxy = null; } + if (!opts.paging) { + opts.paging = { + pageCount: 1, + pageIndex: 0 + }; + } + const proxyAgent = getProxyAgent(opts); const parsedTargetUrl = url.parse(opts.url); return new Promise((resolve, reject) => { try { - const headers = { + const headers = Object.assign({ Cookie: (opts.cookieJar && opts.cookieJar.header) || "", - 'Content-Type': 'application/json' - }; + 'Content-Type': opts.paging.pageCount === 1 ? 'application/json' : 'text/plain', + pageCount: opts.pageCount, + pageIndex: opts.pageIndex, + requestId: opts.requestId + }, opts.headers || {}); if (opts.auth) { const token = Buffer.from(opts.auth.user + ":" + opts.auth.pass).toString('base64'); diff --git a/src/services/sync.js b/src/services/sync.js index d6c616ca4..23c38cc46 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -253,19 +253,33 @@ async function checkContentHash(syncContext) { return failedChecks.length > 0; } -async function syncRequest(syncContext, method, requestPath, body) { +async function syncRequest(syncContext, method, requestPath, body = '') { const timeout = syncOptions.getSyncTimeout(); - const opts = { - method, - url: syncOptions.getSyncServerHost() + requestPath, - cookieJar: syncContext.cookieJar, - timeout: timeout, - body, - proxy: proxyToggle ? syncOptions.getSyncProxy() : null - }; + let response; + + const requestId = utils.randomString(10); + const pageCount = Math.ceil(body.length / 1000000); + + for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { + const opts = { + method, + url: syncOptions.getSyncServerHost() + requestPath, + cookieJar: syncContext.cookieJar, + timeout: timeout, + paging: { + pageIndex, + pageCount, + requestId + }, + body, + proxy: proxyToggle ? syncOptions.getSyncProxy() : null + }; + + response = await utils.timeLimit(request.exec(opts), timeout); + } + - return await utils.timeLimit(request.exec(opts), timeout); } function getEntityChangeRow(entityName, entityId) {