From 6edaf0ed00f6e11f20bb20c6e2969d047199d7f9 Mon Sep 17 00:00:00 2001 From: azivner Date: Fri, 15 Dec 2017 21:14:10 -0500 Subject: [PATCH] split hash checks per table with recent notes not reported into frontend as error --- routes/api/sync.js | 2 +- services/content_hash.js | 67 +++++++++++++++++++++++++++++----------- services/sync.js | 35 +++++++++++++-------- services/sync_table.js | 2 +- 4 files changed, 73 insertions(+), 33 deletions(-) diff --git a/routes/api/sync.js b/routes/api/sync.js index 91c3cde75..ae1d54d01 100644 --- a/routes/api/sync.js +++ b/routes/api/sync.js @@ -11,7 +11,7 @@ const content_hash = require('../../services/content_hash'); router.get('/check', auth.checkApiAuth, async (req, res, next) => { res.send({ - 'content_hash': await content_hash.getContentHash(), + 'hashes': await content_hash.getHashes(), 'max_sync_id': await sql.getSingleValue('SELECT MAX(id) FROM sync') }); }); diff --git a/services/content_hash.js b/services/content_hash.js index e0494d34b..43f28f900 100644 --- a/services/content_hash.js +++ b/services/content_hash.js @@ -2,7 +2,9 @@ const sql = require('./sql'); const utils = require('./utils'); const options = require('./options'); -function updateHash(hash, rows) { +function getHash(rows) { + let hash = ''; + for (const row of rows) { hash = utils.hash(hash + JSON.stringify(row)); } @@ -10,29 +12,58 @@ function updateHash(hash, rows) { return hash; } -async function getContentHash() { - let hash = ''; +async function getHashes() { + const optionsQuestionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(','); - hash = updateHash(hash, await sql.getResults("SELECT note_id, note_title, note_text, date_modified, is_protected, " + - "is_deleted FROM notes ORDER BY note_id")); + return { + notes: getHash(await sql.getResults(`SELECT + note_id, + note_title, + note_text, + date_modified, + is_protected, + is_deleted + FROM notes + ORDER BY note_id`)), - hash = updateHash(hash, await sql.getResults("SELECT note_tree_id, note_id, note_pid, note_pos, date_modified, " + - "is_deleted, prefix FROM notes_tree ORDER BY note_tree_id")); + notes_tree: getHash(await sql.getResults(`SELECT + note_tree_id, + note_id, + note_pid, + note_pos, + date_modified, + is_deleted, + prefix + FROM notes_tree + ORDER BY note_tree_id`)), - hash = updateHash(hash, await sql.getResults("SELECT note_history_id, note_id, note_title, note_text, " + - "date_modified_from, date_modified_to FROM notes_history ORDER BY note_history_id")); + notes_history: getHash(await sql.getResults(`SELECT + note_history_id, + note_id, + note_title, + note_text, + date_modified_from, + date_modified_to + FROM notes_history + ORDER BY note_history_id`)), - hash = updateHash(hash, await sql.getResults("SELECT note_tree_id, note_path, date_accessed, is_deleted FROM recent_notes " + - "ORDER BY note_path")); + recent_notes: getHash(await sql.getResults(`SELECT + note_tree_id, + note_path, + date_accessed, + is_deleted + FROM recent_notes + ORDER BY note_path`)), - const questionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(','); - - hash = updateHash(hash, await sql.getResults("SELECT opt_name, opt_value FROM options " + - "WHERE opt_name IN (" + questionMarks + ") ORDER BY opt_name", options.SYNCED_OPTIONS)); - - return hash; + options: getHash(await sql.getResults(`SELECT + opt_name, + opt_value + FROM options + WHERE opt_name IN (${optionsQuestionMarks}) + ORDER BY opt_name`, options.SYNCED_OPTIONS)) + }; } module.exports = { - getContentHash + getHashes }; \ No newline at end of file diff --git a/services/sync.js b/services/sync.js index a3eb10949..c3c05452c 100644 --- a/services/sync.js +++ b/services/sync.js @@ -127,7 +127,7 @@ async function pullSync(syncContext) { const resp = await syncRequest(syncContext, 'GET', "/api/sync/" + sync.entity_name + "/" + encodeURIComponent(sync.entity_id)); - if (!resp) { + if (!resp || !resp.entity) { log.error("Empty response to pull for " + sync.entity_name + ", id=" + sync.entity_id); } else if (sync.entity_name === 'notes') { @@ -254,21 +254,30 @@ async function checkContentHash(syncContext) { const resp = await syncRequest(syncContext, 'GET', '/api/sync/check'); - // if (await getLastSyncedPull() < resp.max_sync_id) { - // log.info("There are some outstanding pulls, skipping content check."); - // - // return; - // } + if (await getLastSyncedPull() < resp.max_sync_id) { + log.info("There are some outstanding pulls, skipping content check."); - const localContentHash = await content_hash.getContentHash(); - - if (resp.content_hash === localContentHash) { - log.info("Content hash check PASSED with value: " + localContentHash); + return; } - else { - await messaging.sendMessage({type: 'sync-hash-check-failed'}); - await event_log.addEvent("Content hash check FAILED. Local is " + localContentHash + ", remote is " + resp.content_hash); + const hashes = await content_hash.getHashes(); + let allChecksPassed = true; + + for (const key in hashes) { + if (hashes[key] !== resp.hashes[key]) { + allChecksPassed = true; + + await event_log.addEvent(`Content hash check for ${key} FAILED. Local is ${hashes[key]}, remote is ${resp.hashes[key]}`); + + if (key !== 'recent_notes') { + // let's not get alarmed about recent notes which get updated often and can cause failures in race conditions + await messaging.sendMessage({type: 'sync-hash-check-failed'}); + } + } + } + + if (allChecksPassed) { + log.info("Content hash checks PASSED"); } } diff --git a/services/sync_table.js b/services/sync_table.js index db1df019a..100ec4cc6 100644 --- a/services/sync_table.js +++ b/services/sync_table.js @@ -38,7 +38,7 @@ async function addEntitySync(entityName, entityId, sourceId) { if (!sync_setup.isSyncSetup) { // this is because the "server" instances shouldn't have outstanding pushes // useful when you fork the DB for new "client" instance, it won't try to sync the whole DB - await sql.execute("UPDATE options SET opt_value = (SELECT MAX(id) FROM sync) WHERE opt_name = 'last_synced_push'"); + await sql.execute("UPDATE options SET opt_value = (SELECT MAX(id) FROM sync) WHERE opt_name IN('last_synced_push', 'last_synced_pull')"); } }