diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js index 58e56361e..214acd2e8 100644 --- a/src/routes/api/sync.js +++ b/src/routes/api/sync.js @@ -162,6 +162,8 @@ function update(req) { partialRequests[requestId].payload += req.body; + log.info(`Receiving partial request ${requestId}, page index ${pageIndex} out of ${pageCount} pages.`); + if (pageIndex !== pageCount - 1) { return; } @@ -180,7 +182,9 @@ function update(req) { setInterval(() => { for (const key in partialRequests) { - if (partialRequests[key].createdAt - Date.now() > 5 * 60 * 1000) { + if (Date.now() - partialRequests[key].createdAt > 5 * 60 * 1000) { + log.info(`Cleaning up unfinished partial requests for ${key}`); + delete partialRequests[key]; } } diff --git a/src/services/request.js b/src/services/request.js index ff5278d29..46542c2ba 100644 --- a/src/services/request.js +++ b/src/services/request.js @@ -19,7 +19,8 @@ function exec(opts) { if (!opts.paging) { opts.paging = { pageCount: 1, - pageIndex: 0 + pageIndex: 0, + requestId: 'n/a' }; } @@ -28,13 +29,13 @@ function exec(opts) { return new Promise((resolve, reject) => { try { - const headers = Object.assign({ + const headers = { Cookie: (opts.cookieJar && opts.cookieJar.header) || "", 'Content-Type': opts.paging.pageCount === 1 ? 'application/json' : 'text/plain', - pageCount: opts.pageCount, - pageIndex: opts.pageIndex, - requestId: opts.requestId - }, opts.headers || {}); + pageCount: opts.paging.pageCount, + pageIndex: opts.paging.pageIndex, + requestId: opts.paging.requestId + }; if (opts.auth) { const token = Buffer.from(opts.auth.user + ":" + opts.auth.pass).toString('base64'); @@ -85,7 +86,7 @@ function exec(opts) { }); }); - request.end(opts.body ? JSON.stringify(opts.body) : undefined); + request.end(opts.body); } catch (e) { reject(generateError(opts, e.message)); diff --git a/src/services/sync.js b/src/services/sync.js index 23c38cc46..4701633e2 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -253,13 +253,15 @@ async function checkContentHash(syncContext) { return failedChecks.length > 0; } -async function syncRequest(syncContext, method, requestPath, body = '') { +async function syncRequest(syncContext, method, requestPath, body) { + body = body ? JSON.stringify(body) : ''; + const timeout = syncOptions.getSyncTimeout(); let response; const requestId = utils.randomString(10); - const pageCount = Math.ceil(body.length / 1000000); + const pageCount = Math.min(1, Math.ceil(body.length / 1000000)); for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) { const opts = { @@ -277,9 +279,11 @@ async function syncRequest(syncContext, method, requestPath, body = '') { }; response = await utils.timeLimit(request.exec(opts), timeout); + + console.log("response", response); } - + return response; } function getEntityChangeRow(entityName, entityId) {