partial sync requests WIP

src/routes/api/sync.js

@@ -162,6 +162,8 @@ function update(req) {
 
         partialRequests[requestId].payload += req.body;
 
+        log.info(`Receiving partial request ${requestId}, page index ${pageIndex} out of ${pageCount} pages.`);
+
         if (pageIndex !== pageCount - 1) {
             return;
         }
@@ -180,7 +182,9 @@ function update(req) {
 
 setInterval(() => {
     for (const key in partialRequests) {
-        if (partialRequests[key].createdAt - Date.now() > 5 * 60 * 1000) {
+        if (Date.now() - partialRequests[key].createdAt > 5 * 60 * 1000) {
+            log.info(`Cleaning up unfinished partial requests for ${key}`);
+
             delete partialRequests[key];
         }
     }

src/services/request.js

@@ -19,7 +19,8 @@ function exec(opts) {
     if (!opts.paging) {
         opts.paging = {
             pageCount: 1,
-            pageIndex: 0
+            pageIndex: 0,
+            requestId: 'n/a'
         };
     }
 
@@ -28,13 +29,13 @@ function exec(opts) {
 
     return new Promise((resolve, reject) => {
         try {
-            const headers = Object.assign({
+            const headers = {
                 Cookie: (opts.cookieJar && opts.cookieJar.header) || "",
                 'Content-Type': opts.paging.pageCount === 1 ? 'application/json' : 'text/plain',
-                pageCount: opts.pageCount,
-                pageIndex: opts.pageIndex,
-                requestId: opts.requestId
-            }, opts.headers || {});
+                pageCount: opts.paging.pageCount,
+                pageIndex: opts.paging.pageIndex,
+                requestId: opts.paging.requestId
+            };
 
             if (opts.auth) {
                 const token = Buffer.from(opts.auth.user + ":" + opts.auth.pass).toString('base64');
@@ -85,7 +86,7 @@ function exec(opts) {
                 });
             });
 
-            request.end(opts.body ? JSON.stringify(opts.body) : undefined);
+            request.end(opts.body);
         }
         catch (e) {
             reject(generateError(opts, e.message));

src/services/sync.js

@@ -253,13 +253,15 @@ async function checkContentHash(syncContext) {
     return failedChecks.length > 0;
 }
 
-async function syncRequest(syncContext, method, requestPath, body = '') {
+async function syncRequest(syncContext, method, requestPath, body) {
+    body = body ? JSON.stringify(body) : '';
+
     const timeout = syncOptions.getSyncTimeout();
 
     let response;
 
     const requestId = utils.randomString(10);
-    const pageCount = Math.ceil(body.length / 1000000);
+    const pageCount = Math.min(1, Math.ceil(body.length / 1000000));
 
     for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) {
         const opts = {
@@ -277,9 +279,11 @@ async function syncRequest(syncContext, method, requestPath, body = '') {
         };
 
         response = await utils.timeLimit(request.exec(opts), timeout);
+
+        console.log("response", response);
     }
 
-
+    return response;
 }
 
 function getEntityChangeRow(entityName, entityId) {