fixes for sync

app.js

@@ -99,7 +99,7 @@ app.use('/api/login', loginApiRoute);
 
 // catch 404 and forward to error handler
 app.use((req, res, next) => {
-    const err = new Error('Not Found');
+    const err = new Error('Router not found for request ' + req.url);
     err.status = 404;
     next(err);
 });

routes/api/login.js

@@ -4,7 +4,7 @@ const express = require('express');
 const router = express.Router();
 const sql = require('../../services/sql');
 const utils = require('../../services/utils');
-const crypto = require('crypto');
+const migration = require('../../services/migration');
 
 router.post('', async (req, res, next) => {
     const timestamp = req.body.timestamp;
@@ -16,7 +16,7 @@ router.post('', async (req, res, next) => {
         res.send({ message: 'Auth request time is out of sync' });
     }
 
-    const dbVersion = res.body.dbVersion;
+    const dbVersion = req.body.dbVersion;
 
     if (dbVersion !== migration.APP_DB_VERSION) {
         res.status(400);
@@ -24,10 +24,7 @@ router.post('', async (req, res, next) => {
     }
 
     const documentSecret = await sql.getOption('document_secret');
-
+    const expectedHash = utils.hmac(documentSecret, timestamp);
-    const hmac = crypto.createHmac('sha256', documentSecret);
-    hmac.update(timestamp);
-    const expectedHash = hmac.digest('base64');
 
     const givenHash = req.body.hash;
 

routes/api/sync.js

@@ -17,7 +17,9 @@ router.post('/now', auth.checkApiAuth, async (req, res, next) => {
 router.get('/changed/:since', auth.checkApiAuth, async (req, res, next) => {
     const since = parseInt(req.params.since);
 
-    res.send(await sync.getChangedSince(since));
+    const result = await sync.getChangedSince(since);
+
+    res.send(result);
 });
 
 router.put('/changed', auth.checkApiAuth, async (req, res, next) => {

services/auth.js

@@ -6,8 +6,7 @@ async function checkAuth(req, res, next) {
     if (!req.session.loggedIn) {
         res.redirect("login");
     }
-
+    else if (await migration.isDbUpToDate()) {
-    if (await migration.isDbUpToDate()) {
         next();
     }
     else {
@@ -26,20 +25,19 @@ async function checkAuthWithoutMigration(req, res, next) {
 
 async function checkApiAuth(req, res, next) {
     if (!req.session.loggedIn) {
-        res.sendStatus(401);
+        res.status(401).send({});
     }
-
+    else if (await migration.isDbUpToDate()) {
-    if (await migration.isDbUpToDate()) {
         next();
     }
     else {
-        res.sendStatus(409); // need better response than that
+        res.status(409).send({}); // need better response than that
     }
 }
 
 async function checkApiAuthWithoutMigration(req, res, next) {
     if (!req.session.loggedIn) {
-        res.sendStatus(401);
+        res.status(401).send({});
     }
     else {
         next();

services/log.js

@@ -16,11 +16,13 @@ const logger = require('simple-node-logger').createRollingFileLogger({
 
 function info(message) {
     logger.info(message);
+
+    console.log(message);
 }
 
 function error(message) {
     // we're using .info() instead of .error() because simple-node-logger emits weird error for error()
-    logger.info(message);
+    info(message);
 }
 
 const requestBlacklist = [ "/api/audit", "/libraries", "/javascripts", "/images", "/stylesheets" ];

services/sync.js

@@ -17,13 +17,21 @@ let syncInProgress = false;
 async function pullSync(cookieJar, syncLog) {
     const lastSyncedPull = parseInt(await sql.getOption('last_synced_pull'));
 
-    const resp = await rp({
+    let resp;
-        uri: SYNC_SERVER + '/api/sync/changed/' + lastSyncedPull,
+
-        headers: {
+    try {
-            auth: 'sync'
+        resp = await rp({
-        },
+            uri: SYNC_SERVER + '/api/sync/changed/' + lastSyncedPull,
-        json: true
+            headers: {
-    });
+                auth: 'sync'
+            },
+            jar: cookieJar,
+            json: true
+        });
+    }
+    catch (e) {
+        throw new Error("Can't pull changed, inner exception: " + e.stack);
+    }
 
     try {
         await sql.beginTransaction();
@@ -31,19 +39,29 @@ async function pullSync(cookieJar, syncLog) {
         await putChanged(resp, syncLog);
 
         for (const noteId of resp.notes) {
-            const note = await rp({
+            let note;
-                uri: SYNC_SERVER + "/api/sync/note/" + noteId + "/" + lastSyncedPull,
-                headers: {
-                    auth: 'sync'
-                },
-                json: true,
-                jar: cookieJar
-            });
 
+            try {
+                note = await rp({
+                    uri: SYNC_SERVER + "/api/sync/note/" + noteId + "/" + lastSyncedPull,
+                    headers: {
+                        auth: 'sync'
+                    },
+                    json: true,
+                    jar: cookieJar
+                });
+            }
+            catch (e) {
+                throw new Error("Can't pull note " + noteId + ", inner exception: " + e.stack);
+            }
 
             await putNote(note, syncLog);
         }
 
+        if (resp.notes.length > 0) {
+            await sql.addAudit(audit_category.SYNC);
+        }
+
         await sql.setOption('last_synced_pull', resp.syncTimestamp);
 
         await sql.commit();
@@ -64,16 +82,22 @@ async function pushSync(cookieJar, syncLog) {
     if (changed.tree.length > 0 || changed.audit_log.length > 0) {
         logSync("Sending " + changed.tree.length + " tree changes and " + changed.audit_log.length + " audit changes", syncLog);
 
-        await rp({
+        try {
-            method: 'PUT',
+            await rp({
-            uri: SYNC_SERVER + '/api/sync/changed',
+                method: 'PUT',
-            headers: {
+                uri: SYNC_SERVER + '/api/sync/changed',
-                auth: 'sync'
+                headers: {
-            },
+                    auth: 'sync'
-            body: changed,
+                },
-            json: true,
+                body: changed,
-            jar: cookieJar
+                json: true,
-        });
+                timeout: 300 * 1000, // this can take long time
+                jar: cookieJar
+            });
+        }
+        catch (e) {
+            throw new Error("Can't send tree changes and audit, inner exception: " + e.stack);
+        }
     }
 
     for (const noteId of changed.notes) {
@@ -81,16 +105,22 @@ async function pushSync(cookieJar, syncLog) {
 
         const note = await getNoteSince(noteId);
 
-        await rp({
+        try {
-            method: 'PUT',
+            await rp({
-            uri: SYNC_SERVER + '/api/sync/note',
+                method: 'PUT',
-            headers: {
+                uri: SYNC_SERVER + '/api/sync/note',
-                auth: 'sync'
+                headers: {
-            },
+                    auth: 'sync'
-            body: note,
+                },
-            json: true,
+                body: note,
-            jar: cookieJar
+                json: true,
-        });
+                timeout: 60 * 1000,
+                jar: cookieJar
+            });
+        }
+        catch (e) {
+            throw new Error("Can't send note update, inner exception: " + e.stack);
+        }
     }
 
     await sql.setOption('last_synced_push', syncStarted);
@@ -100,39 +130,47 @@ async function login() {
     const timestamp = utils.nowTimestamp();
 
     const documentSecret = await sql.getOption('document_secret');
-
+    const hash = utils.hmac(documentSecret, timestamp);
-    const hmac = crypto.createHmac('sha256', Buffer.from(documentSecret.toString(), 'ASCII'));
-    hmac.update(timestamp.toString());
-    const hash = hmac.digest('base64');
 
     const cookieJar = rp.jar();
 
-    await rp({
+    try {
-        method: 'POST',
+        await rp({
-        uri: SYNC_SERVER + '/api/login',
+            method: 'POST',
-        body: {
+            uri: SYNC_SERVER + '/api/login',
-            timestamp: timestamp,
+            body: {
-            dbVersion: migration.APP_DB_VERSION,
+                timestamp: timestamp,
-            hash: hash
+                dbVersion: migration.APP_DB_VERSION,
-        },
+                hash: hash
-        json: true,
+            },
-        jar: cookieJar
+            json: true,
-    });
+            timeout: 5 * 1000,
+            jar: cookieJar
+        });
 
-    return cookieJar;
+        return cookieJar;
+    }
+    catch (e) {
+        throw new Error("Can't login to API for sync, inner exception: " + e.stack);
+    }
 }
 
 async function sync() {
-    if (syncInProgress) {
+    const syncLog = [];
-        return;
+
-    }
+    // if (syncInProgress) {
+    //     syncLog.push("Sync already in progress");
+    //
+    //     return syncLog;
+    // }
 
     syncInProgress = true;
-    const syncLog = [];
 
     try {
         if (!await migration.isDbUpToDate()) {
-            return;
+            syncLog.push("DB not up to date");
+
+            return syncLog;
         }
 
         const cookieJar = await login();
@@ -154,18 +192,19 @@ async function sync() {
 function logSync(message, syncLog) {
     log.info(message);
 
-    if (syncLog !== null) {
+    if (syncLog) {
         syncLog.push(message);
     }
+
+    console.log(message);
 }
 
 async function getChangedSince(since) {
     return {
-        'documentId': await getDocumentId(),
         'syncTimestamp': utils.nowTimestamp(),
         'tree': await sql.getResults("select * from notes_tree where date_modified >= ?", [since]),
         'notes': await sql.getFlattenedResults('note_id', "select note_id from notes where date_modified >= ?", [since]),
-        'audit_log': await sql.getResults("select * from audit_log where date_modified >= ?", [since])
+        'audit_log': await sql.getResults("select * from audit_log where category != 'SYNC' and date_modified >= ?", [since])
     };
 }
 
@@ -189,7 +228,7 @@ async function putChanged(changed, syncLog) {
     for (const audit of changed.audit_log) {
         await sql.insert("audit_log", audit, true);
 
-        logSync("Update/sync audit_log for noteId=" + audit.note_id, syncLog);
+        logSync("Update/sync audit_log for category=" + audit.category + ", noteId=" + audit.note_id, syncLog);
     }
 
     if (changed.tree.length > 0 || changed.audit_log.length > 0) {
@@ -200,34 +239,37 @@ async function putChanged(changed, syncLog) {
 }
 
 async function putNote(note, syncLog) {
-    const origNote = await sql.getSingleResult();
+    const origNote = await sql.getSingleResult("select * from notes where note_id = ?", [note.detail.note_id]);
 
-    if (origNote !== null && origNote.date_modified >= note.detail.date_modified) {
+    try {
-        // version we have in DB is actually newer than the one we're getting from sync
+        if (origNote !== null && origNote.date_modified >= note.detail.date_modified) {
-        // so we'll leave the current state as it is. The synced version should be stored in the history
+            // version we have in DB is actually newer than the one we're getting from sync
+            // so we'll leave the current state as it is. The synced version should be stored in the history
+        }
+        else {
+            await sql.insert("notes", note.detail, true);
+        }
+
+        await sql.remove("images", note.detail.note_id);
+
+        for (const image of note.images) {
+            await sql.insert("images", image);
+        }
+
+        for (const history of note.history) {
+            delete history['id'];
+
+            await sql.insert("notes_history", history, true);
+        }
+
+        logSync("Update/sync note " + note.detail.note_id, syncLog);
     }
-    else {
+    catch (e) {
-        await sql.insert("notes", note.detail, true);
+        throw new Error("Update note " + note.detail.note_id + " failed, inner exception: " + e.stack);
     }
-
-    await sql.remove("images", note.detail.note_id);
-
-    for (const image of note.images) {
-        await sql.insert("images", image);
-    }
-
-    for (const history of note.history) {
-        delete history['id'];
-
-        await sql.insert("notes_history", history);
-    }
-
-    await sql.addAudit(audit_category.SYNC);
-
-    logSync("Update/sync note " + note.detail.note_id, syncLog);
 }
 
-if (SYNC_SERVER) {
+if (SYNC_SERVER && false) {
     log.info("Setting up sync");
 
     setInterval(sync, 60000);

services/utils.js

@@ -34,11 +34,18 @@ function fromBase64(encodedText) {
     return Buffer.from(encodedText, 'base64');
 }
 
+function hmac(secret, value) {
+    const hmac = crypto.createHmac('sha256', Buffer.from(secret.toString(), 'ASCII'));
+    hmac.update(value.toString());
+    return hmac.digest('base64');
+}
+
 module.exports = {
     randomSecureToken,
     randomString,
     nowTimestamp,
     newNoteId,
     toBase64,
-    fromBase64
+    fromBase64,
+    hmac
 };