From 49a53f7a45fe7e039d73fa00174e5e5f9d5b5ac4 Mon Sep 17 00:00:00 2001 From: azivner Date: Tue, 22 May 2018 00:15:54 -0400 Subject: [PATCH] added hash columns for faster sync check calculation --- db/migrations/0093__add_hash_field.sql | 9 ++ src/entities/api_token.js | 1 + src/entities/branch.js | 1 + src/entities/entity.js | 8 ++ src/entities/image.js | 1 + src/entities/label.js | 1 + src/entities/note.js | 1 + src/entities/note_image.js | 1 + src/entities/note_revision.js | 1 + src/entities/option.js | 11 +++ src/entities/recent_note.js | 1 + src/services/app_info.js | 2 +- src/services/content_hash.js | 125 +++++-------------------- 13 files changed, 61 insertions(+), 102 deletions(-) create mode 100644 db/migrations/0093__add_hash_field.sql create mode 100644 src/entities/option.js diff --git a/db/migrations/0093__add_hash_field.sql b/db/migrations/0093__add_hash_field.sql new file mode 100644 index 000000000..489ff1061 --- /dev/null +++ b/db/migrations/0093__add_hash_field.sql @@ -0,0 +1,9 @@ +ALTER TABLE notes ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE branches ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE note_revisions ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE recent_notes ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE options ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE note_images ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE images ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE labels ADD hash TEXT DEFAULT "" NOT NULL; +ALTER TABLE api_tokens ADD hash TEXT DEFAULT "" NOT NULL; \ No newline at end of file diff --git a/src/entities/api_token.js b/src/entities/api_token.js index ecf2a9838..0240f052a 100644 --- a/src/entities/api_token.js +++ b/src/entities/api_token.js @@ -6,6 +6,7 @@ const dateUtils = require('../services/date_utils'); class ApiToken extends Entity { static get tableName() { return "api_tokens"; } static get primaryKeyName() { return "apiTokenId"; } + static get syncedProperties() { return ["apiTokenId", "token", "dateCreated", "isDeleted"]; } beforeSaving() { super.beforeSaving(); diff --git a/src/entities/branch.js b/src/entities/branch.js index 463786c29..8e6b1dc8e 100644 --- a/src/entities/branch.js +++ b/src/entities/branch.js @@ -8,6 +8,7 @@ const sql = require('../services/sql'); class Branch extends Entity { static get tableName() { return "branches"; } static get primaryKeyName() { return "branchId"; } + static get syncedProperties() { return ["branchId", "noteId", "parentNoteId", "notePosition", "dateModified", "isDeleted", "prefix"]; } async getNote() { return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]); diff --git a/src/entities/entity.js b/src/entities/entity.js index 897b8dea4..acb5a5665 100644 --- a/src/entities/entity.js +++ b/src/entities/entity.js @@ -14,6 +14,14 @@ class Entity { if (!this[this.constructor.primaryKeyName]) { this[this.constructor.primaryKeyName] = utils.newEntityId(); } + + let contentToHash = ""; + + for (const propertyName of this.constructor.syncedProperties) { + contentToHash += "|" + this[propertyName]; + } + + this["hash"] = utils.hash(contentToHash).substr(0, 10); } async save() { diff --git a/src/entities/image.js b/src/entities/image.js index 437f6d859..ce16d0111 100644 --- a/src/entities/image.js +++ b/src/entities/image.js @@ -6,6 +6,7 @@ const dateUtils = require('../services/date_utils'); class Image extends Entity { static get tableName() { return "images"; } static get primaryKeyName() { return "imageId"; } + static get syncedProperties() { return ["imageId", "format", "checksum", "name", "isDeleted", "dateModified", "dateCreated"]; } beforeSaving() { super.beforeSaving(); diff --git a/src/entities/label.js b/src/entities/label.js index 59e4ad5ec..72f4f8fdc 100644 --- a/src/entities/label.js +++ b/src/entities/label.js @@ -8,6 +8,7 @@ const sql = require('../services/sql'); class Label extends Entity { static get tableName() { return "labels"; } static get primaryKeyName() { return "labelId"; } + static get syncedProperties() { return ["labelId", "noteId", "name", "value", "dateModified", "dateCreated"]; } async getNote() { return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]); diff --git a/src/entities/note.js b/src/entities/note.js index 16b13d744..b77ac1a17 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -8,6 +8,7 @@ const dateUtils = require('../services/date_utils'); class Note extends Entity { static get tableName() { return "notes"; } static get primaryKeyName() { return "noteId"; } + static get syncedProperties() { return ["noteId", "title", "content", "type", "dateModified", "isProtected", "isDeleted"]; } constructor(row) { super(row); diff --git a/src/entities/note_image.js b/src/entities/note_image.js index 8483f270c..c39e1bf04 100644 --- a/src/entities/note_image.js +++ b/src/entities/note_image.js @@ -7,6 +7,7 @@ const dateUtils = require('../services/date_utils'); class NoteImage extends Entity { static get tableName() { return "note_images"; } static get primaryKeyName() { return "noteImageId"; } + static get syncedProperties() { return ["noteImageId", "noteId", "imageId", "isDeleted", "dateModified", "dateCreated"]; } async getNote() { return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]); diff --git a/src/entities/note_revision.js b/src/entities/note_revision.js index d1f010bb1..4c956d1bd 100644 --- a/src/entities/note_revision.js +++ b/src/entities/note_revision.js @@ -7,6 +7,7 @@ const repository = require('../services/repository'); class NoteRevision extends Entity { static get tableName() { return "note_revisions"; } static get primaryKeyName() { return "noteRevisionId"; } + static get syncedProperties() { return ["noteRevisionId", "noteId", "title", "content", "dateModifiedFrom", "dateModifiedTo"]; } constructor(row) { super(row); diff --git a/src/entities/option.js b/src/entities/option.js new file mode 100644 index 000000000..68b3dc212 --- /dev/null +++ b/src/entities/option.js @@ -0,0 +1,11 @@ +"use strict"; + +const Entity = require('./entity'); + +class Option extends Entity { + static get tableName() { return "options"; } + static get primaryKeyName() { return "name"; } + static get syncedProperties() { return ["name", "value"]; } +} + +module.exports = Option; \ No newline at end of file diff --git a/src/entities/recent_note.js b/src/entities/recent_note.js index a5d77b7d4..4761a1651 100644 --- a/src/entities/recent_note.js +++ b/src/entities/recent_note.js @@ -5,6 +5,7 @@ const Entity = require('./entity'); class RecentNote extends Entity { static get tableName() { return "recent_notes"; } static get primaryKeyName() { return "branchId"; } + static get syncedProperties() { return ["branchId", "notePath", "dateAccessed", "isDeleted"]; } } module.exports = RecentNote; \ No newline at end of file diff --git a/src/services/app_info.js b/src/services/app_info.js index ee2ea6820..f6976a77f 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -3,7 +3,7 @@ const build = require('./build'); const packageJson = require('../../package'); -const APP_DB_VERSION = 92; +const APP_DB_VERSION = 93; module.exports = { appVersion: packageJson.version, diff --git a/src/services/content_hash.js b/src/services/content_hash.js index f8384f35b..c1097fbc4 100644 --- a/src/services/content_hash.js +++ b/src/services/content_hash.js @@ -5,117 +5,40 @@ const utils = require('./utils'); const log = require('./log'); const eventLogService = require('./event_log'); const messagingService = require('./messaging'); +const ApiToken = require('../entities/api_token'); +const Branch = require('../entities/branch'); +const Image = require('../entities/image'); +const Note = require('../entities/note'); +const NoteImage = require('../entities/note_image'); +const Label = require('../entities/label'); +const NoteRevision = require('../entities/note_revision'); +const RecentNote = require('../entities/recent_note'); +const Option = require('../entities/option'); -function getHash(rows) { - let hash = ''; +async function getHash(entityConstructor, whereBranch) { + let contentToHash = await sql.getValue(`SELECT GROUP_CONCAT(hash) FROM ${entityConstructor.tableName} ` + + (whereBranch ? `WHERE ${whereBranch} ` : '') + `ORDER BY ${entityConstructor.primaryKeyName}`); - for (const row of rows) { - hash = utils.hash(hash + JSON.stringify(row)); + if (!contentToHash) { // might be null in case of no rows + contentToHash = ""; } - return hash; + return utils.hash(contentToHash); } async function getHashes() { const startTime = new Date(); const hashes = { - notes: getHash(await sql.getRows(` - SELECT - noteId, - title, - content, - type, - dateModified, - isProtected, - isDeleted - FROM notes - ORDER BY noteId`)), - - branches: getHash(await sql.getRows(` - SELECT - branchId, - noteId, - parentNoteId, - notePosition, - dateModified, - isDeleted, - prefix - FROM branches - ORDER BY branchId`)), - - note_revisions: getHash(await sql.getRows(` - SELECT - noteRevisionId, - noteId, - title, - content, - dateModifiedFrom, - dateModifiedTo - FROM note_revisions - ORDER BY noteRevisionId`)), - - recent_notes: getHash(await sql.getRows(` - SELECT - branchId, - notePath, - dateAccessed, - isDeleted - FROM recent_notes - ORDER BY notePath`)), - - options: getHash(await sql.getRows(` - SELECT - name, - value - FROM options - WHERE isSynced = 1 - ORDER BY name`)), - - // we don't include image data on purpose because they are quite large, checksum is good enough - // to represent the data anyway - images: getHash(await sql.getRows(` - SELECT - imageId, - format, - checksum, - name, - isDeleted, - dateModified, - dateCreated - FROM images - ORDER BY imageId`)), - - note_images: getHash(await sql.getRows(` - SELECT - noteImageId, - noteId, - imageId, - isDeleted, - dateModified, - dateCreated - FROM note_images - ORDER BY noteImageId`)), - - labels: getHash(await sql.getRows(` - SELECT - labelId, - noteId, - name, - value, - dateModified, - dateCreated - FROM labels - ORDER BY labelId`)), - - api_tokens: getHash(await sql.getRows(` - SELECT - apiTokenId, - token, - dateCreated, - isDeleted - FROM api_tokens - ORDER BY apiTokenId`)) + notes: await getHash(Note), + branches: await getHash(Branch), + note_revisions: await getHash(NoteRevision), + recent_notes: await getHash(RecentNote), + options: await getHash(Option, "isSynced = 1"), + images: await getHash(Image), + note_images: await getHash(NoteImage), + labels: await getHash(Label), + api_tokens: await getHash(ApiToken) }; const elapseTimeMs = new Date().getTime() - startTime.getTime();