From 1990c9399ec82546f05889118aea7ab677214af0 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 21 Apr 2019 12:03:13 +0200 Subject: [PATCH] removed FTS5 completely --- bin/export-schema.sh | 2 +- db/migrations/0128__add_note_fulltext.sql | 1 - db/migrations/0129__fill_note_fulltext.js | 11 +-- db/migrations/0133__drop_note_fulltext.sql | 1 + db/schema.sql | 2 - docs/backend_api/Note.html | 104 ++++++++++----------- docs/backend_api/entities_note.js.html | 6 -- src/entities/note.js | 6 -- src/services/app_info.js | 2 +- src/services/consistency_checks.js | 37 -------- src/services/note_fulltext.js | 67 ------------- src/services/sync_update.js | 5 - 12 files changed, 56 insertions(+), 188 deletions(-) create mode 100644 db/migrations/0133__drop_note_fulltext.sql delete mode 100644 src/services/note_fulltext.js diff --git a/bin/export-schema.sh b/bin/export-schema.sh index a75983dd8..152bcd0e0 100755 --- a/bin/export-schema.sh +++ b/bin/export-schema.sh @@ -2,6 +2,6 @@ SCHEMA_FILE_PATH=db/schema.sql -sqlite3 ~/trilium-data/document.db .schema | grep -v "sqlite_sequence" | grep -v "note_fulltext_" > "$SCHEMA_FILE_PATH" +sqlite3 ~/trilium-data/document.db .schema | grep -v "sqlite_sequence" > "$SCHEMA_FILE_PATH" echo "DB schema exported to $SCHEMA_FILE_PATH" \ No newline at end of file diff --git a/db/migrations/0128__add_note_fulltext.sql b/db/migrations/0128__add_note_fulltext.sql index d9910230a..e69de29bb 100644 --- a/db/migrations/0128__add_note_fulltext.sql +++ b/db/migrations/0128__add_note_fulltext.sql @@ -1 +0,0 @@ -CREATE VIRTUAL TABLE note_fulltext USING fts5(noteId UNINDEXED, title, titleHash UNINDEXED, content, contentHash UNINDEXED); diff --git a/db/migrations/0129__fill_note_fulltext.js b/db/migrations/0129__fill_note_fulltext.js index 1709d10fb..3bd8337a0 100644 --- a/db/migrations/0129__fill_note_fulltext.js +++ b/db/migrations/0129__fill_note_fulltext.js @@ -1,10 +1 @@ -const repository = require('../../src/services/repository'); -const noteFulltextService = require('../../src/services/note_fulltext'); - -module.exports = async () => { - const notes = await repository.getEntities('SELECT * FROM notes WHERE isDeleted = 0 AND isProtected = 0'); - - for (const note of notes) { - await noteFulltextService.updateNoteFulltext(note); - } -}; \ No newline at end of file +module.exports = async () => {}; \ No newline at end of file diff --git a/db/migrations/0133__drop_note_fulltext.sql b/db/migrations/0133__drop_note_fulltext.sql new file mode 100644 index 000000000..775842874 --- /dev/null +++ b/db/migrations/0133__drop_note_fulltext.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS note_fulltext; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index c015f826b..fd41ad1c3 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,5 +1,3 @@ -CREATE VIRTUAL TABLE note_fulltext USING fts5(noteId UNINDEXED, title, titleHash UNINDEXED, content, contentHash UNINDEXED) -/* note_fulltext(noteId,title,titleHash,content,contentHash) */; CREATE TABLE IF NOT EXISTS "sync" ( `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, `entityName` TEXT NOT NULL, diff --git a/docs/backend_api/Note.html b/docs/backend_api/Note.html index 1cf2ff433..d0efab35d 100644 --- a/docs/backend_api/Note.html +++ b/docs/backend_api/Note.html @@ -396,7 +396,7 @@
Source:
@@ -581,7 +581,7 @@
Source:
@@ -746,7 +746,7 @@
Source:
@@ -922,7 +922,7 @@
Source:
@@ -1026,7 +1026,7 @@
Source:
@@ -1126,7 +1126,7 @@
Source:
@@ -1230,7 +1230,7 @@
Source:
@@ -1334,7 +1334,7 @@
Source:
@@ -1434,7 +1434,7 @@
Source:
@@ -1665,7 +1665,7 @@
Source:
@@ -1861,7 +1861,7 @@
Source:
@@ -2057,7 +2057,7 @@
Source:
@@ -2157,7 +2157,7 @@
Source:
@@ -2306,7 +2306,7 @@
Source:
@@ -2471,7 +2471,7 @@
Source:
@@ -2636,7 +2636,7 @@
Source:
@@ -2789,7 +2789,7 @@
Source:
@@ -2897,7 +2897,7 @@
Source:
@@ -3001,7 +3001,7 @@
Source:
@@ -3101,7 +3101,7 @@
Source:
@@ -3205,7 +3205,7 @@
Source:
@@ -3358,7 +3358,7 @@
Source:
@@ -3523,7 +3523,7 @@
Source:
@@ -3688,7 +3688,7 @@
Source:
@@ -3841,7 +3841,7 @@
Source:
@@ -3997,7 +3997,7 @@
Source:
@@ -4105,7 +4105,7 @@
Source:
@@ -4205,7 +4205,7 @@
Source:
@@ -4313,7 +4313,7 @@
Source:
@@ -4413,7 +4413,7 @@
Source:
@@ -4589,7 +4589,7 @@
Source:
@@ -4693,7 +4693,7 @@
Source:
@@ -4846,7 +4846,7 @@
Source:
@@ -4999,7 +4999,7 @@
Source:
@@ -5108,7 +5108,7 @@ Cache is note instance scoped.
Source:
@@ -5190,7 +5190,7 @@ Cache is note instance scoped.
Source:
@@ -5294,7 +5294,7 @@ Cache is note instance scoped.
Source:
@@ -5398,7 +5398,7 @@ Cache is note instance scoped.
Source:
@@ -5502,7 +5502,7 @@ Cache is note instance scoped.
Source:
@@ -5606,7 +5606,7 @@ Cache is note instance scoped.
Source:
@@ -5710,7 +5710,7 @@ Cache is note instance scoped.
Source:
@@ -5937,7 +5937,7 @@ Cache is note instance scoped.
Source:
@@ -6133,7 +6133,7 @@ Cache is note instance scoped.
Source:
@@ -6329,7 +6329,7 @@ Cache is note instance scoped.
Source:
@@ -6556,7 +6556,7 @@ Cache is note instance scoped.
Source:
@@ -6656,7 +6656,7 @@ Cache is note instance scoped.
Source:
@@ -6756,7 +6756,7 @@ Cache is note instance scoped.
Source:
@@ -6952,7 +6952,7 @@ Cache is note instance scoped.
Source:
@@ -7148,7 +7148,7 @@ Cache is note instance scoped.
Source:
@@ -7406,7 +7406,7 @@ Cache is note instance scoped.
Source:
@@ -7633,7 +7633,7 @@ Cache is note instance scoped.
Source:
@@ -7860,7 +7860,7 @@ Cache is note instance scoped.
Source:
diff --git a/docs/backend_api/entities_note.js.html b/docs/backend_api/entities_note.js.html index f1acd74b0..5411ae861 100644 --- a/docs/backend_api/entities_note.js.html +++ b/docs/backend_api/entities_note.js.html @@ -35,7 +35,6 @@ const repository = require('../services/repository'); const sql = require('../services/sql'); const utils = require('../services/utils'); const dateUtils = require('../services/date_utils'); -const noteFulltextService = require('../services/note_fulltext'); const syncTableService = require('../services/sync_table'); const LABEL = 'label'; @@ -114,7 +113,6 @@ class Note extends Entity { } this.content = res.content; - this.contentHash = res.contentHash; // used only for note_fulltext consistency check if (this.isProtected) { if (this.isContentAvailable) { @@ -759,10 +757,6 @@ class Note extends Entity { delete pojo.content; delete pojo.contentHash; } - - async afterSaving() { - noteFulltextService.triggerNoteFulltextUpdate(this.noteId); - } } module.exports = Note; diff --git a/src/entities/note.js b/src/entities/note.js index c7451c1fd..6b938158f 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -7,7 +7,6 @@ const repository = require('../services/repository'); const sql = require('../services/sql'); const utils = require('../services/utils'); const dateUtils = require('../services/date_utils'); -const noteFulltextService = require('../services/note_fulltext'); const syncTableService = require('../services/sync_table'); const LABEL = 'label'; @@ -86,7 +85,6 @@ class Note extends Entity { } this.content = res.content; - this.contentHash = res.contentHash; // used only for note_fulltext consistency check if (this.isProtected) { if (this.isContentAvailable) { @@ -731,10 +729,6 @@ class Note extends Entity { delete pojo.content; delete pojo.contentHash; } - - async afterSaving() { - noteFulltextService.triggerNoteFulltextUpdate(this.noteId); - } } module.exports = Note; \ No newline at end of file diff --git a/src/services/app_info.js b/src/services/app_info.js index 8677f0c62..3949bd2a9 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -4,7 +4,7 @@ const build = require('./build'); const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); -const APP_DB_VERSION = 132; +const APP_DB_VERSION = 133; const SYNC_VERSION = 8; module.exports = { diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index bf8e207cb..b09544d5e 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -5,7 +5,6 @@ const sqlInit = require('./sql_init'); const log = require('./log'); const messagingService = require('./messaging'); const syncMutexService = require('./sync_mutex'); -const noteFulltextService = require('./note_fulltext'); const repository = require('./repository'); const cls = require('./cls'); const syncTableService = require('./sync_table'); @@ -358,42 +357,6 @@ async function findLogicIssues() { logFix(`Removed link ${linkId} because target note ${targetNoteId} is also deleted.`); }); - - // this doesn't try to find notes for which the fulltext doesn't exist at all - reason is the "archived" label - // which is inheritable and not easy to filter out such rows in consistency check which would mean that it would - // find some false positives. - await findAndFixIssues(` - SELECT - noteId - FROM - notes - JOIN note_contents USING(noteId) - JOIN note_fulltext USING(noteId) - WHERE - notes.isDeleted = 0 - AND notes.isProtected = 0 - AND (note_fulltext.noteId IS NULL - OR note_fulltext.titleHash != notes.hash - OR note_fulltext.contentHash != note_contents.hash)`, - async ({noteId}) => { - noteFulltextService.triggerNoteFulltextUpdate(noteId); - - logFix(`Triggered fulltext update of note ${noteId} since it was out of sync.`); - }); - - await findAndFixIssues(` - SELECT - noteId - FROM - notes - JOIN note_fulltext USING(noteId) - WHERE - (notes.isDeleted = 1 OR notes.isProtected = 1)`, - async ({noteId}) => { - noteFulltextService.triggerNoteFulltextUpdate(noteId); - - logFix(`Triggered fulltext update of note ${noteId} since it was out of sync.`); - }); } async function runSyncRowChecks(entityName, key) { diff --git a/src/services/note_fulltext.js b/src/services/note_fulltext.js deleted file mode 100644 index 578541d13..000000000 --- a/src/services/note_fulltext.js +++ /dev/null @@ -1,67 +0,0 @@ -const sql = require('./sql'); -const repository = require('./repository'); -const html2plaintext = require('html2plaintext'); - -const noteIdQueue = []; - -async function updateNoteFulltext(note) { - if (!note) { - // this might happen when note content is being synced before note itself - return; - } - - if (note.isDeleted || note.isProtected || await note.hasLabel('archived')) { - await sql.execute(`DELETE - FROM note_fulltext - WHERE noteId = ?`, [note.noteId]); - } else { - let content = null; - let contentHash = null; - - if (['text', 'code'].includes(note.type)) { - content = await note.getContent(true); - - // might not be available during sync before note_contents is synced - if (content) { - if (note.type === 'text' && note.mime === 'text/html') { - content = html2plaintext(content); - } - - contentHash = note.contentHash; - } - } - - // optimistically try to update first ... - const res = await sql.execute(`UPDATE note_fulltext SET title = ?, titleHash = ?, content = ?, contentHash = ? WHERE noteId = ?`, [note.title, note.hash, content, contentHash, note.noteId]); - - // ... and insert only when the update did not work - if (res.stmt.changes === 0) { - await sql.execute(`INSERT INTO note_fulltext (title, titleHash, content, contentHash, noteId) - VALUES (?, ?, ?, ?, ?)`, [note.title, note.hash, content, contentHash, note.noteId]); - } - } -} - -async function triggerNoteFulltextUpdate(noteId) { - if (!noteIdQueue.includes(noteId)) { - noteIdQueue.push(noteId); - } - - while (noteIdQueue.length > 0) { - await sql.transactional(async () => { - if (noteIdQueue.length === 0) { - return; - } - - const noteId = noteIdQueue.shift(); - const note = await repository.getNote(noteId); - - await updateNoteFulltext(note); - }); - } -} - -module.exports = { - triggerNoteFulltextUpdate, - updateNoteFulltext -}; \ No newline at end of file diff --git a/src/services/sync_update.js b/src/services/sync_update.js index 7221318da..34a4659fc 100644 --- a/src/services/sync_update.js +++ b/src/services/sync_update.js @@ -3,7 +3,6 @@ const log = require('./log'); const eventLogService = require('./event_log'); const syncTableService = require('./sync_table'); const eventService = require('./events'); -const noteFulltextService = require('../services/note_fulltext'); async function updateEntity(sync, entity, sourceId) { const {entityName} = sync; @@ -62,8 +61,6 @@ async function updateNote(entity, sourceId) { await syncTableService.addNoteSync(entity.noteId, sourceId); }); - noteFulltextService.triggerNoteFulltextUpdate(entity.noteId); - log.info("Update/sync note " + entity.noteId); } } @@ -80,8 +77,6 @@ async function updateNoteContent(entity, sourceId) { await syncTableService.addNoteContentSync(entity.noteId, sourceId); }); - noteFulltextService.triggerNoteFulltextUpdate(entity.noteId); - log.info("Update/sync note content for noteId=" + entity.noteId); } }