From 16cdd9e13741154c2dcba817baabdb32e2bf17be Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 26 Jul 2025 18:31:16 +0300 Subject: [PATCH] feat(fs_sync): draft implementation --- .github/instructions/nx.instructions.md | 2 +- apps/client/src/stylesheets/style.css | 186 +++++ .../widgets/type_widgets/content_widget.ts | 2 + .../options/advanced/file_system_sync.ts | 623 +++++++++++++++ apps/server/src/app.ts | 7 + apps/server/src/assets/db/schema.sql | 53 ++ apps/server/src/becca/becca-interface.ts | 43 + apps/server/src/becca/becca_loader.ts | 26 +- .../src/becca/entities/bfile_note_mapping.ts | 233 ++++++ .../becca/entities/bfile_system_mapping.ts | 236 ++++++ apps/server/src/becca/entity_constructor.ts | 6 +- apps/server/src/migrations/migrations.ts | 58 ++ .../server/src/routes/api/file_system_sync.ts | 366 +++++++++ apps/server/src/routes/routes.ts | 4 + apps/server/src/services/app_info.ts | 2 +- .../services/file_system_content_converter.ts | 464 +++++++++++ apps/server/src/services/file_system_sync.ts | 749 ++++++++++++++++++ .../src/services/file_system_sync_init.ts | 129 +++ .../src/services/file_system_watcher.ts | 431 ++++++++++ apps/server/src/services/options_init.ts | 3 + packages/commons/src/lib/options_interface.ts | 1 + 21 files changed, 3620 insertions(+), 4 deletions(-) create mode 100644 apps/client/src/widgets/type_widgets/options/advanced/file_system_sync.ts create mode 100644 apps/server/src/becca/entities/bfile_note_mapping.ts create mode 100644 apps/server/src/becca/entities/bfile_system_mapping.ts create mode 100644 apps/server/src/routes/api/file_system_sync.ts create mode 100644 apps/server/src/services/file_system_content_converter.ts create mode 100644 apps/server/src/services/file_system_sync.ts create mode 100644 apps/server/src/services/file_system_sync_init.ts create mode 100644 apps/server/src/services/file_system_watcher.ts diff --git a/.github/instructions/nx.instructions.md b/.github/instructions/nx.instructions.md index a055adbc4..c6d01270f 100644 --- a/.github/instructions/nx.instructions.md +++ b/.github/instructions/nx.instructions.md @@ -4,7 +4,7 @@ applyTo: '**' // This file is automatically generated by Nx Console -You are in an nx workspace using Nx 21.3.5 and pnpm as the package manager. +You are in an nx workspace using Nx 21.3.7 and pnpm as the package manager. You have access to the Nx MCP server and the tools it provides. Use them. Follow these guidelines in order to best help the user: diff --git a/apps/client/src/stylesheets/style.css b/apps/client/src/stylesheets/style.css index 2296166f2..7844c58a4 100644 --- a/apps/client/src/stylesheets/style.css +++ b/apps/client/src/stylesheets/style.css @@ -2201,3 +2201,189 @@ footer.file-footer button { content: "\ec24"; transform: rotate(180deg); } + +/* File System Sync Modal Styles */ +.mapping-modal { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + z-index: 1050; + display: flex; + align-items: center; + justify-content: center; +} + +.mapping-modal .modal-backdrop { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0, 0, 0, 0.5); + z-index: 1051; +} + +.mapping-modal .modal-content { + position: relative; + background: var(--main-background-color); + border: 1px solid var(--main-border-color); + border-radius: 5px; + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3); + width: 90%; + max-width: 600px; + max-height: 80vh; + overflow-y: auto; + z-index: 1052; +} + +.mapping-modal .modal-header { + display: flex; + justify-content: between; + align-items: center; + padding: 1rem; + border-bottom: 1px solid var(--main-border-color); +} + +.mapping-modal .modal-title { + margin: 0; + font-size: 1.25rem; + flex: 1; +} + +.mapping-modal .modal-close { + background: none; + border: none; + font-size: 1.5rem; + cursor: pointer; + color: var(--muted-text-color); + padding: 0; + width: 30px; + height: 30px; + display: flex; + align-items: center; + justify-content: center; +} + +.mapping-modal .modal-close:hover { + color: var(--main-text-color); +} + +.mapping-modal .modal-body { + padding: 1rem; +} + +.mapping-modal .modal-footer { + padding: 1rem; + border-top: 1px solid var(--main-border-color); + display: flex; + justify-content: flex-end; + gap: 0.5rem; +} + +/* File System Sync Mapping Cards */ +.mapping-item.card { + border: 1px solid var(--main-border-color); + border-radius: 5px; + transition: box-shadow 0.2s ease; +} + +.mapping-item.card:hover { + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); +} + +.mapping-item .mapping-path { + font-family: monospace; + font-size: 0.9rem; + word-break: break-all; +} + +.mapping-item .mapping-details { + font-size: 0.85rem; + margin-top: 0.25rem; +} + +.mapping-item .mapping-status { + margin-top: 0.5rem; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.mapping-item .mapping-actions { + display: flex; + gap: 0.25rem; +} + +.mapping-item .mapping-actions .btn { + padding: 0.25rem 0.5rem; +} + +/* Status Badges */ +.status-badge.badge { + font-size: 0.75rem; + padding: 0.25rem 0.5rem; + border-radius: 3px; +} + +.status-badge.badge-success { + background-color: #28a745; + color: white; +} + +.status-badge.badge-danger { + background-color: #dc3545; + color: white; +} + +.status-badge.badge-secondary { + background-color: #6c757d; + color: white; +} + +/* Path Validation Styles */ +.path-validation-result { + margin-top: 0.5rem; + font-size: 0.875rem; +} + +.path-validation-result .text-success { + color: #28a745; +} + +.path-validation-result .text-warning { + color: #ffc107; +} + +.path-validation-result .text-danger { + color: #dc3545; +} + +/* Sync Status Section */ +.sync-status-container { + margin: 1rem 0; + padding: 1rem; + background: var(--accented-background-color); + border-radius: 5px; +} + +.sync-status-info .status-item, +.sync-status-info .active-mappings-count { + margin-bottom: 0.5rem; +} + +/* Form Enhancements */ +.mapping-form .form-group { + margin-bottom: 1rem; +} + +.mapping-form .subtree-options { + margin-left: 1.5rem; +} + +.mapping-form .help-block { + font-size: 0.875rem; + color: var(--muted-text-color); + margin-top: 0.25rem; +} diff --git a/apps/client/src/widgets/type_widgets/content_widget.ts b/apps/client/src/widgets/type_widgets/content_widget.ts index beb9de272..4a52e6013 100644 --- a/apps/client/src/widgets/type_widgets/content_widget.ts +++ b/apps/client/src/widgets/type_widgets/content_widget.ts @@ -27,6 +27,7 @@ import RevisionSnapshotsLimitOptions from "./options/other/revision_snapshots_li import NetworkConnectionsOptions from "./options/other/network_connections.js"; import HtmlImportTagsOptions from "./options/other/html_import_tags.js"; import AdvancedSyncOptions from "./options/advanced/sync.js"; +import FileSystemSyncOptions from "./options/advanced/file_system_sync.js"; import DatabaseIntegrityCheckOptions from "./options/advanced/database_integrity_check.js"; import VacuumDatabaseOptions from "./options/advanced/vacuum_database.js"; import DatabaseAnonymizationOptions from "./options/advanced/database_anonymization.js"; @@ -138,6 +139,7 @@ const CONTENT_WIDGETS: Record; +} + +// API Request/Response interfaces +interface PathValidationRequest { + filePath: string; +} + +interface PathValidationResponse { + exists: boolean; + stats?: { + isDirectory: boolean; + size: number; + modified: string; + }; +} + +interface CreateMappingRequest { + noteId: string; + filePath: string; + syncDirection: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium'; + contentFormat: 'auto' | 'markdown' | 'html' | 'raw'; + includeSubtree: boolean; + preserveHierarchy: boolean; + excludePatterns: string[] | null; +} + +interface UpdateMappingRequest extends CreateMappingRequest {} + +interface SyncMappingResponse { + success: boolean; + message?: string; +} + +interface ApiResponse { + success?: boolean; + message?: string; +} + +const TPL = /*html*/` +
+

File System Sync

+ +
+ +
+ Allows bidirectional synchronization between Trilium notes and files on your local file system. +
+
+ + +
+ + +`; + +const MAPPING_ITEM_TPL = /*html*/` +
+
+
+
+
+ +
+
+ • + • + +
+
+ + +
+
+
+ + + +
+
+ +
+
`; + +export default class FileSystemSyncOptions extends OptionsWidget { + private $fileSyncEnabledCheckbox!: JQuery; + private $fileSyncControls!: JQuery; + private $syncStatusText!: JQuery; + private $mappingsCount!: JQuery; + private $mappingsList!: JQuery; + private $createMappingButton!: JQuery; + private $refreshStatusButton!: JQuery; + + // Modal elements + private $mappingModal!: JQuery; + private $modalTitle!: JQuery; + private $noteSelector!: JQuery; + private $selectedNoteId!: JQuery; + private $filePathInput!: JQuery; + private $validatePathButton!: JQuery; + private $pathValidationResult!: JQuery; + private $syncDirectionSelect!: JQuery; + private $contentFormatSelect!: JQuery; + private $includeSubtreeCheckbox!: JQuery; + private $preserveHierarchyCheckbox!: JQuery; + private $subtreeOptions!: JQuery; + private $excludePatternsTextarea!: JQuery; + private $saveMappingButton!: JQuery; + private $cancelMappingButton!: JQuery; + private $modalClose!: JQuery; + + private currentEditingMappingId: string | null = null; + private mappings: FileSystemMapping[] = []; + + doRender() { + this.$widget = $(TPL); + this.initializeElements(); + this.setupEventHandlers(); + } + + private initializeElements() { + this.$fileSyncEnabledCheckbox = this.$widget.find(".file-sync-enabled-checkbox"); + this.$fileSyncControls = this.$widget.find(".file-sync-controls"); + this.$syncStatusText = this.$widget.find(".sync-status-text"); + this.$mappingsCount = this.$widget.find(".mappings-count"); + this.$mappingsList = this.$widget.find(".mappings-list"); + this.$createMappingButton = this.$widget.find(".create-mapping-button"); + this.$refreshStatusButton = this.$widget.find(".refresh-status-button"); + + // Modal elements + this.$mappingModal = this.$widget.find(".mapping-modal"); + this.$modalTitle = this.$mappingModal.find(".modal-title"); + this.$noteSelector = this.$mappingModal.find(".note-selector"); + this.$selectedNoteId = this.$mappingModal.find(".selected-note-id"); + this.$filePathInput = this.$mappingModal.find(".file-path-input"); + this.$validatePathButton = this.$mappingModal.find(".validate-path-button"); + this.$pathValidationResult = this.$mappingModal.find(".path-validation-result"); + this.$syncDirectionSelect = this.$mappingModal.find(".sync-direction-select"); + this.$contentFormatSelect = this.$mappingModal.find(".content-format-select"); + this.$includeSubtreeCheckbox = this.$mappingModal.find(".include-subtree-checkbox"); + this.$preserveHierarchyCheckbox = this.$mappingModal.find(".preserve-hierarchy-checkbox"); + this.$subtreeOptions = this.$mappingModal.find(".subtree-options"); + this.$excludePatternsTextarea = this.$mappingModal.find(".exclude-patterns-textarea"); + this.$saveMappingButton = this.$mappingModal.find(".save-mapping-button"); + this.$cancelMappingButton = this.$mappingModal.find(".cancel-mapping-button"); + this.$modalClose = this.$mappingModal.find(".modal-close"); + } + + private setupEventHandlers() { + this.$fileSyncEnabledCheckbox.on("change", async () => { + const isEnabled = this.$fileSyncEnabledCheckbox.prop("checked"); + + try { + if (isEnabled) { + await server.post("file-system-sync/enable"); + } else { + await server.post("file-system-sync/disable"); + } + + this.toggleControls(isEnabled); + if (isEnabled) { + await this.refreshStatus(); + } + + toastService.showMessage(`File system sync ${isEnabled ? 'enabled' : 'disabled'}`); + } catch (error) { + toastService.showError(`Failed to ${isEnabled ? 'enable' : 'disable'} file system sync`); + // Revert checkbox state + this.$fileSyncEnabledCheckbox.prop("checked", !isEnabled); + } + }); + + this.$createMappingButton.on("click", () => { + this.showMappingModal(); + }); + + this.$refreshStatusButton.on("click", () => { + this.refreshStatus(); + }); + + this.$validatePathButton.on("click", () => { + this.validatePath(); + }); + + this.$includeSubtreeCheckbox.on("change", () => { + const isChecked = this.$includeSubtreeCheckbox.prop("checked"); + this.$subtreeOptions.toggle(isChecked); + }); + + // Modal handlers + this.$saveMappingButton.on("click", () => { + this.saveMapping(); + }); + + this.$cancelMappingButton.on("click", () => { + this.hideMappingModal(); + }); + + this.$modalClose.on("click", () => { + this.hideMappingModal(); + }); + + this.$mappingModal.find(".modal-backdrop").on("click", () => { + this.hideMappingModal(); + }); + + // Note selector (simplified - in real implementation would integrate with note picker) + this.$noteSelector.on("click", () => { + // TODO: Integrate with Trilium's note picker dialog + toastService.showMessage("Note picker integration needed"); + }); + } + + private toggleControls(enabled: boolean) { + this.$fileSyncControls.toggle(enabled); + } + + private async refreshStatus() { + try { + const status = await server.get("file-system-sync/status"); + + this.$syncStatusText.text(status.initialized ? "Active" : "Inactive"); + + if (status.initialized) { + await this.loadMappings(); + } + } catch (error) { + this.$syncStatusText.text("Error"); + toastService.showError("Failed to get sync status"); + } + } + + private async loadMappings() { + try { + this.mappings = await server.get("file-system-sync/mappings"); + this.renderMappings(); + this.$mappingsCount.text(this.mappings.length.toString()); + } catch (error) { + toastService.showError("Failed to load mappings"); + } + } + + private renderMappings() { + this.$mappingsList.empty(); + + for (const mapping of this.mappings) { + const $item = $(MAPPING_ITEM_TPL); + $item.attr("data-mapping-id", mapping.mappingId); + + $item.find(".file-path").text(mapping.filePath); + $item.find(".note-title").text(`Note: ${mapping.noteId}`); // TODO: Get actual note title + $item.find(".sync-direction-text").text(this.formatSyncDirection(mapping.syncDirection)); + $item.find(".content-format-text").text(mapping.contentFormat); + + // Status badge + const $statusBadge = $item.find(".status-badge"); + if (mapping.syncErrors && mapping.syncErrors.length > 0) { + $statusBadge.addClass("badge badge-danger").text("Error"); + const $errorsDiv = $item.find(".sync-errors"); + const $errorList = $errorsDiv.find(".error-list"); + mapping.syncErrors.forEach(error => { + $errorList.append(`
  • ${error}
    • `); + }); + $errorsDiv.show(); + } else if (mapping.isActive) { + $statusBadge.addClass("badge badge-success").text("Active"); + } else { + $statusBadge.addClass("badge badge-secondary").text("Inactive"); + } + + // Last sync time + if (mapping.lastSyncTime) { + const lastSync = new Date(mapping.lastSyncTime).toLocaleString(); + $item.find(".last-sync").text(`Last sync: ${lastSync}`); + } else { + $item.find(".last-sync").text("Never synced"); + } + + // Action handlers + $item.find(".sync-mapping-button").on("click", () => { + this.syncMapping(mapping.mappingId); + }); + + $item.find(".edit-mapping-button").on("click", () => { + this.editMapping(mapping); + }); + + $item.find(".delete-mapping-button").on("click", () => { + this.deleteMapping(mapping.mappingId); + }); + + this.$mappingsList.append($item); + } + } + + private formatSyncDirection(direction: string): string { + switch (direction) { + case 'bidirectional': return 'Bidirectional'; + case 'trilium_to_disk': return 'Trilium → Disk'; + case 'disk_to_trilium': return 'Disk → Trilium'; + default: return direction; + } + } + + private showMappingModal(mapping?: FileSystemMapping) { + this.currentEditingMappingId = mapping?.mappingId || null; + + if (mapping) { + this.$modalTitle.text("Edit File System Mapping"); + this.populateMappingForm(mapping); + } else { + this.$modalTitle.text("Create File System Mapping"); + this.clearMappingForm(); + } + + this.$mappingModal.show(); + } + + private hideMappingModal() { + this.$mappingModal.hide(); + this.clearMappingForm(); + this.currentEditingMappingId = null; + } + + private populateMappingForm(mapping: FileSystemMapping) { + this.$selectedNoteId.val(mapping.noteId); + this.$noteSelector.val(`Note: ${mapping.noteId}`); // TODO: Show actual note title + this.$filePathInput.val(mapping.filePath); + this.$syncDirectionSelect.val(mapping.syncDirection); + this.$contentFormatSelect.val(mapping.contentFormat); + this.$includeSubtreeCheckbox.prop("checked", mapping.includeSubtree); + this.$preserveHierarchyCheckbox.prop("checked", mapping.preserveHierarchy); + this.$subtreeOptions.toggle(mapping.includeSubtree); + + if (mapping.excludePatterns) { + this.$excludePatternsTextarea.val(mapping.excludePatterns.join('\n')); + } + } + + private clearMappingForm() { + this.$selectedNoteId.val(''); + this.$noteSelector.val(''); + this.$filePathInput.val(''); + this.$syncDirectionSelect.val('bidirectional'); + this.$contentFormatSelect.val('auto'); + this.$includeSubtreeCheckbox.prop("checked", false); + this.$preserveHierarchyCheckbox.prop("checked", true); + this.$subtreeOptions.hide(); + this.$excludePatternsTextarea.val(''); + this.$pathValidationResult.empty(); + } + + private async validatePath() { + const filePath = this.$filePathInput.val() as string; + if (!filePath) { + this.$pathValidationResult.html('
    Please enter a file path
    '); + return; + } + + try { + const result = await server.post("file-system-sync/validate-path", { filePath } as PathValidationRequest); + + if (result.exists && result.stats) { + const type = result.stats.isDirectory ? 'directory' : 'file'; + this.$pathValidationResult.html( + `
    ✓ Valid ${type} (${result.stats.size} bytes, modified ${new Date(result.stats.modified).toLocaleString()})
    ` + ); + } else { + this.$pathValidationResult.html('
    ⚠ Path does not exist
    '); + } + } catch (error) { + this.$pathValidationResult.html('
    ✗ Invalid path
    '); + } + } + + private async saveMapping() { + const noteId = this.$selectedNoteId.val() as string; + const filePath = this.$filePathInput.val() as string; + const syncDirection = this.$syncDirectionSelect.val() as string; + const contentFormat = this.$contentFormatSelect.val() as string; + const includeSubtree = this.$includeSubtreeCheckbox.prop("checked"); + const preserveHierarchy = this.$preserveHierarchyCheckbox.prop("checked"); + const excludePatternsText = this.$excludePatternsTextarea.val() as string; + + // Validation + if (!noteId) { + toastService.showError("Please select a note"); + return; + } + + if (!filePath) { + toastService.showError("Please enter a file path"); + return; + } + + const excludePatterns = excludePatternsText.trim() + ? excludePatternsText.split('\n').map(p => p.trim()).filter(p => p) + : null; + + const mappingData: CreateMappingRequest = { + noteId, + filePath, + syncDirection: syncDirection as 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium', + contentFormat: contentFormat as 'auto' | 'markdown' | 'html' | 'raw', + includeSubtree, + preserveHierarchy, + excludePatterns + }; + + try { + if (this.currentEditingMappingId) { + await server.put(`file-system-sync/mappings/${this.currentEditingMappingId}`, mappingData as UpdateMappingRequest); + toastService.showMessage("Mapping updated successfully"); + } else { + await server.post("file-system-sync/mappings", mappingData); + toastService.showMessage("Mapping created successfully"); + } + + this.hideMappingModal(); + await this.loadMappings(); + } catch (error) { + toastService.showError("Failed to save mapping"); + } + } + + private async syncMapping(mappingId: string) { + try { + const result = await server.post(`file-system-sync/mappings/${mappingId}/sync`); + if (result.success) { + toastService.showMessage("Sync completed successfully"); + } else { + toastService.showError(`Sync failed: ${result.message}`); + } + await this.loadMappings(); + } catch (error) { + toastService.showError("Failed to trigger sync"); + } + } + + private editMapping(mapping: FileSystemMapping) { + this.showMappingModal(mapping); + } + + private async deleteMapping(mappingId: string) { + if (!confirm("Are you sure you want to delete this mapping?")) { + return; + } + + try { + await server.delete(`file-system-sync/mappings/${mappingId}`); + toastService.showMessage("Mapping deleted successfully"); + await this.loadMappings(); + } catch (error) { + toastService.showError("Failed to delete mapping"); + } + } + + async optionsLoaded(options: OptionMap) { + const isEnabled = options.fileSystemSyncEnabled === "true"; + this.$fileSyncEnabledCheckbox.prop("checked", isEnabled); + this.toggleControls(isEnabled); + + if (isEnabled) { + await this.refreshStatus(); + } + } +} diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index a7bc00b42..13434e1c8 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -20,6 +20,7 @@ import log from "./services/log.js"; import "./services/handlers.js"; import "./becca/becca_loader.js"; import { RESOURCE_DIR } from "./services/resource_dir.js"; +import fileSystemSyncInit from "./services/file_system_sync_init.js"; export default async function buildApp() { const app = express(); @@ -32,6 +33,9 @@ export default async function buildApp() { try { log.info("Database initialized, LLM features available"); log.info("LLM features ready"); + + // Initialize file system sync after database is ready + await fileSystemSyncInit.init(); } catch (error) { console.error("Error initializing LLM features:", error); } @@ -41,6 +45,9 @@ export default async function buildApp() { if (sql_init.isDbInitialized()) { try { log.info("LLM features ready"); + + // Initialize file system sync if database is already ready + await fileSystemSyncInit.init(); } catch (error) { console.error("Error initializing LLM features:", error); } diff --git a/apps/server/src/assets/db/schema.sql b/apps/server/src/assets/db/schema.sql index 07d924a91..962fc8ebe 100644 --- a/apps/server/src/assets/db/schema.sql +++ b/apps/server/src/assets/db/schema.sql @@ -152,3 +152,56 @@ CREATE TABLE IF NOT EXISTS sessions ( data TEXT, expires INTEGER ); + +-- Table to store file system mappings for notes and subtrees +CREATE TABLE IF NOT EXISTS "file_system_mappings" ( + "mappingId" TEXT NOT NULL PRIMARY KEY, + "noteId" TEXT NOT NULL, + "filePath" TEXT NOT NULL, + "syncDirection" TEXT NOT NULL DEFAULT 'bidirectional', -- 'bidirectional', 'trilium_to_disk', 'disk_to_trilium' + "isActive" INTEGER NOT NULL DEFAULT 1, + "includeSubtree" INTEGER NOT NULL DEFAULT 0, + "preserveHierarchy" INTEGER NOT NULL DEFAULT 1, + "contentFormat" TEXT NOT NULL DEFAULT 'auto', -- 'auto', 'markdown', 'html', 'raw' + "excludePatterns" TEXT DEFAULT NULL, -- JSON array of glob patterns to exclude + "lastSyncTime" TEXT DEFAULT NULL, + "syncErrors" TEXT DEFAULT NULL, -- JSON array of recent sync errors + "dateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL +); + +-- Table to track file to note mappings for efficient lookups +CREATE TABLE IF NOT EXISTS "file_note_mappings" ( + "fileNoteId" TEXT NOT NULL PRIMARY KEY, + "mappingId" TEXT NOT NULL, + "noteId" TEXT NOT NULL, + "filePath" TEXT NOT NULL, + "fileHash" TEXT DEFAULT NULL, + "fileModifiedTime" TEXT DEFAULT NULL, + "lastSyncTime" TEXT DEFAULT NULL, + "syncStatus" TEXT NOT NULL DEFAULT 'synced', -- 'synced', 'pending', 'conflict', 'error' + "dateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL, + FOREIGN KEY ("mappingId") REFERENCES "file_system_mappings" ("mappingId") ON DELETE CASCADE, + FOREIGN KEY ("noteId") REFERENCES "notes" ("noteId") ON DELETE CASCADE +); + +-- Index for quick lookup by noteId +CREATE INDEX "IDX_file_system_mappings_noteId" ON "file_system_mappings" ("noteId"); +-- Index for finding active mappings +CREATE INDEX "IDX_file_system_mappings_active" ON "file_system_mappings" ("isActive", "noteId"); +-- Unique constraint to prevent duplicate mappings for same note +CREATE UNIQUE INDEX "IDX_file_system_mappings_note_unique" ON "file_system_mappings" ("noteId"); + +-- Index for quick lookup by file path +CREATE INDEX "IDX_file_note_mappings_filePath" ON "file_note_mappings" ("filePath"); +-- Index for finding notes by mapping +CREATE INDEX "IDX_file_note_mappings_mapping" ON "file_note_mappings" ("mappingId", "noteId"); +-- Index for finding pending syncs +CREATE INDEX "IDX_file_note_mappings_sync_status" ON "file_note_mappings" ("syncStatus", "mappingId"); +-- Unique constraint for file path per mapping +CREATE UNIQUE INDEX "IDX_file_note_mappings_file_unique" ON "file_note_mappings" ("mappingId", "filePath"); diff --git a/apps/server/src/becca/becca-interface.ts b/apps/server/src/becca/becca-interface.ts index 005a5cc52..79302acf3 100644 --- a/apps/server/src/becca/becca-interface.ts +++ b/apps/server/src/becca/becca-interface.ts @@ -12,6 +12,8 @@ import type { AttachmentRow, BlobRow, RevisionRow } from "@triliumnext/commons"; import BBlob from "./entities/bblob.js"; import BRecentNote from "./entities/brecent_note.js"; import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js"; +import type BFileSystemMapping from "./entities/bfile_system_mapping.js"; +import type BFileNoteMapping from "./entities/bfile_note_mapping.js"; interface AttachmentOpts { includeContentLength?: boolean; @@ -32,6 +34,8 @@ export default class Becca { attributeIndex!: Record; options!: Record; etapiTokens!: Record; + fileSystemMappings!: Record; + fileNoteMappings!: Record; allNoteSetCache: NoteSet | null; @@ -48,6 +52,8 @@ export default class Becca { this.attributeIndex = {}; this.options = {}; this.etapiTokens = {}; + this.fileSystemMappings = {}; + this.fileNoteMappings = {}; this.dirtyNoteSetCache(); @@ -213,6 +219,39 @@ export default class Becca { return this.etapiTokens[etapiTokenId]; } + getFileSystemMapping(mappingId: string): BFileSystemMapping | null { + return this.fileSystemMappings[mappingId]; + } + + getFileSystemMappingOrThrow(mappingId: string): BFileSystemMapping { + const mapping = this.getFileSystemMapping(mappingId); + if (!mapping) { + throw new NotFoundError(`File system mapping '${mappingId}' has not been found.`); + } + return mapping; + } + + getFileNoteMapping(fileNoteId: string): BFileNoteMapping | null { + return this.fileNoteMappings[fileNoteId]; + } + + getFileNoteMappingOrThrow(fileNoteId: string): BFileNoteMapping { + const mapping = this.getFileNoteMapping(fileNoteId); + if (!mapping) { + throw new NotFoundError(`File note mapping '${fileNoteId}' has not been found.`); + } + return mapping; + } + + getFileSystemMappingByNoteId(noteId: string): BFileSystemMapping | null { + for (const mapping of Object.values(this.fileSystemMappings)) { + if (mapping.noteId === noteId) { + return mapping; + } + } + return null; + } + getEntity>(entityName: string, entityId: string): AbstractBeccaEntity | null { if (!entityName || !entityId) { return null; @@ -222,6 +261,10 @@ export default class Becca { return this.getRevision(entityId); } else if (entityName === "attachments") { return this.getAttachment(entityId); + } else if (entityName === "file_system_mappings") { + return this.getFileSystemMapping(entityId); + } else if (entityName === "file_note_mappings") { + return this.getFileNoteMapping(entityId); } const camelCaseEntityName = entityName.toLowerCase().replace(/(_[a-z])/g, (group) => group.toUpperCase().replace("_", "")); diff --git a/apps/server/src/becca/becca_loader.ts b/apps/server/src/becca/becca_loader.ts index cef89b43f..09d11a7f6 100644 --- a/apps/server/src/becca/becca_loader.ts +++ b/apps/server/src/becca/becca_loader.ts @@ -9,9 +9,13 @@ import BBranch from "./entities/bbranch.js"; import BAttribute from "./entities/battribute.js"; import BOption from "./entities/boption.js"; import BEtapiToken from "./entities/betapi_token.js"; +import BFileSystemMapping from "./entities/bfile_system_mapping.js"; +import BFileNoteMapping from "./entities/bfile_note_mapping.js"; import cls from "../services/cls.js"; import entityConstructor from "../becca/entity_constructor.js"; import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons"; +import type { FileSystemMappingRow } from "./entities/bfile_system_mapping.js"; +import type { FileNoteMappingRow } from "./entities/bfile_note_mapping.js"; import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js"; import ws from "../services/ws.js"; @@ -64,6 +68,14 @@ function load() { new BEtapiToken(row); } + for (const row of sql.getRows(/*sql*/`SELECT mappingId, noteId, filePath, syncDirection, isActive, includeSubtree, preserveHierarchy, contentFormat, excludePatterns, lastSyncTime, syncErrors, dateCreated, dateModified, utcDateCreated, utcDateModified FROM file_system_mappings`)) { + new BFileSystemMapping(row); + } + + for (const row of sql.getRows(/*sql*/`SELECT fileNoteId, mappingId, noteId, filePath, fileHash, fileModifiedTime, lastSyncTime, syncStatus, dateCreated, dateModified, utcDateCreated, utcDateModified FROM file_note_mappings`)) { + new BFileNoteMapping(row); + } + }); for (const noteId in becca.notes) { @@ -86,7 +98,7 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_CHANGE_SYNCED], ({ entity return; } - if (["notes", "branches", "attributes", "etapi_tokens", "options"].includes(entityName)) { + if (["notes", "branches", "attributes", "etapi_tokens", "options", "file_system_mappings", "file_note_mappings"].includes(entityName)) { const EntityClass = entityConstructor.getEntityFromEntityName(entityName); const primaryKeyName = EntityClass.primaryKeyName; @@ -144,6 +156,10 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_DELETED, eventService.ENT attributeDeleted(entityId); } else if (entityName === "etapi_tokens") { etapiTokenDeleted(entityId); + } else if (entityName === "file_system_mappings") { + fileSystemMappingDeleted(entityId); + } else if (entityName === "file_note_mappings") { + fileNoteMappingDeleted(entityId); } }); @@ -279,6 +295,14 @@ function etapiTokenDeleted(etapiTokenId: string) { delete becca.etapiTokens[etapiTokenId]; } +function fileSystemMappingDeleted(mappingId: string) { + delete becca.fileSystemMappings[mappingId]; +} + +function fileNoteMappingDeleted(fileNoteId: string) { + delete becca.fileNoteMappings[fileNoteId]; +} + eventService.subscribeBeccaLoader(eventService.ENTER_PROTECTED_SESSION, () => { try { diff --git a/apps/server/src/becca/entities/bfile_note_mapping.ts b/apps/server/src/becca/entities/bfile_note_mapping.ts new file mode 100644 index 000000000..114442426 --- /dev/null +++ b/apps/server/src/becca/entities/bfile_note_mapping.ts @@ -0,0 +1,233 @@ +"use strict"; + +import AbstractBeccaEntity from "./abstract_becca_entity.js"; +import dateUtils from "../../services/date_utils.js"; +import { newEntityId } from "../../services/utils.js"; + +export interface FileNoteMappingRow { + fileNoteId?: string; + mappingId: string; + noteId: string; + filePath: string; + fileHash?: string | null; + fileModifiedTime?: string | null; + lastSyncTime?: string | null; + syncStatus?: 'synced' | 'pending' | 'conflict' | 'error'; + dateCreated?: string; + dateModified?: string; + utcDateCreated?: string; + utcDateModified?: string; +} + +/** + * FileNoteMapping represents the mapping between a specific file and a specific note + * This is used for tracking sync status and file metadata + */ +class BFileNoteMapping extends AbstractBeccaEntity { + static get entityName() { + return "file_note_mappings"; + } + static get primaryKeyName() { + return "fileNoteId"; + } + static get hashedProperties() { + return ["fileNoteId", "mappingId", "noteId", "filePath", "fileHash", "syncStatus"]; + } + + fileNoteId!: string; + mappingId!: string; + noteId!: string; + filePath!: string; + fileHash?: string | null; + fileModifiedTime?: string | null; + lastSyncTime?: string | null; + syncStatus!: 'synced' | 'pending' | 'conflict' | 'error'; + + constructor(row?: FileNoteMappingRow) { + super(); + + if (!row) { + return; + } + + this.updateFromRow(row); + this.init(); + } + + updateFromRow(row: FileNoteMappingRow) { + this.update([ + row.fileNoteId, + row.mappingId, + row.noteId, + row.filePath, + row.fileHash, + row.fileModifiedTime, + row.lastSyncTime, + row.syncStatus || 'synced', + row.dateCreated, + row.dateModified, + row.utcDateCreated, + row.utcDateModified + ]); + } + + update([ + fileNoteId, + mappingId, + noteId, + filePath, + fileHash, + fileModifiedTime, + lastSyncTime, + syncStatus, + dateCreated, + dateModified, + utcDateCreated, + utcDateModified + ]: any) { + this.fileNoteId = fileNoteId; + this.mappingId = mappingId; + this.noteId = noteId; + this.filePath = filePath; + this.fileHash = fileHash; + this.fileModifiedTime = fileModifiedTime; + this.lastSyncTime = lastSyncTime; + this.syncStatus = syncStatus || 'synced'; + this.dateCreated = dateCreated; + this.dateModified = dateModified; + this.utcDateCreated = utcDateCreated; + this.utcDateModified = utcDateModified; + + return this; + } + + override init() { + if (this.fileNoteId) { + this.becca.fileNoteMappings = this.becca.fileNoteMappings || {}; + this.becca.fileNoteMappings[this.fileNoteId] = this; + } + } + + get note() { + return this.becca.notes[this.noteId]; + } + + get mapping() { + return this.becca.fileSystemMappings?.[this.mappingId]; + } + + getNote() { + const note = this.becca.getNote(this.noteId); + if (!note) { + throw new Error(`Note '${this.noteId}' for file note mapping '${this.fileNoteId}' does not exist.`); + } + return note; + } + + getMapping() { + const mapping = this.mapping; + if (!mapping) { + throw new Error(`File system mapping '${this.mappingId}' for file note mapping '${this.fileNoteId}' does not exist.`); + } + return mapping; + } + + /** + * Mark this mapping as needing sync + */ + markPending() { + this.syncStatus = 'pending'; + this.save(); + } + + /** + * Mark this mapping as having a conflict + */ + markConflict() { + this.syncStatus = 'conflict'; + this.save(); + } + + /** + * Mark this mapping as having an error + */ + markError() { + this.syncStatus = 'error'; + this.save(); + } + + /** + * Mark this mapping as synced and update sync time + */ + markSynced(fileHash?: string, fileModifiedTime?: string) { + this.syncStatus = 'synced'; + this.lastSyncTime = dateUtils.utcNowDateTime(); + + if (fileHash !== undefined) { + this.fileHash = fileHash; + } + + if (fileModifiedTime !== undefined) { + this.fileModifiedTime = fileModifiedTime; + } + + this.save(); + } + + /** + * Check if the file has been modified since last sync + */ + hasFileChanged(currentFileHash: string, currentModifiedTime: string): boolean { + return this.fileHash !== currentFileHash || this.fileModifiedTime !== currentModifiedTime; + } + + /** + * Check if the note has been modified since last sync + */ + hasNoteChanged(): boolean { + const note = this.note; + if (!note) return false; + + if (!this.lastSyncTime) return true; + + return (note.utcDateModified ?? note.dateModified ?? note.utcDateCreated) > this.lastSyncTime; + } + + override beforeSaving() { + super.beforeSaving(); + + if (!this.fileNoteId) { + this.fileNoteId = newEntityId(); + } + + if (!this.dateCreated) { + this.dateCreated = dateUtils.localNowDateTime(); + } + + if (!this.utcDateCreated) { + this.utcDateCreated = dateUtils.utcNowDateTime(); + } + + this.dateModified = dateUtils.localNowDateTime(); + this.utcDateModified = dateUtils.utcNowDateTime(); + } + + getPojo(): FileNoteMappingRow { + return { + fileNoteId: this.fileNoteId, + mappingId: this.mappingId, + noteId: this.noteId, + filePath: this.filePath, + fileHash: this.fileHash, + fileModifiedTime: this.fileModifiedTime, + lastSyncTime: this.lastSyncTime, + syncStatus: this.syncStatus, + dateCreated: this.dateCreated, + dateModified: this.dateModified, + utcDateCreated: this.utcDateCreated, + utcDateModified: this.utcDateModified + }; + } +} + +export default BFileNoteMapping; diff --git a/apps/server/src/becca/entities/bfile_system_mapping.ts b/apps/server/src/becca/entities/bfile_system_mapping.ts new file mode 100644 index 000000000..701f4a818 --- /dev/null +++ b/apps/server/src/becca/entities/bfile_system_mapping.ts @@ -0,0 +1,236 @@ +"use strict"; + +import AbstractBeccaEntity from "./abstract_becca_entity.js"; +import dateUtils from "../../services/date_utils.js"; +import { newEntityId } from "../../services/utils.js"; + +export interface FileSystemMappingRow { + mappingId?: string; + noteId: string; + filePath: string; + syncDirection?: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium'; + isActive?: number; + includeSubtree?: number; + preserveHierarchy?: number; + contentFormat?: 'auto' | 'markdown' | 'html' | 'raw'; + excludePatterns?: string | null; + lastSyncTime?: string | null; + syncErrors?: string | null; + dateCreated?: string; + dateModified?: string; + utcDateCreated?: string; + utcDateModified?: string; +} + +/** + * FileSystemMapping represents a mapping between a note/subtree and a file system path + */ +class BFileSystemMapping extends AbstractBeccaEntity { + static get entityName() { + return "file_system_mappings"; + } + static get primaryKeyName() { + return "mappingId"; + } + static get hashedProperties() { + return ["mappingId", "noteId", "filePath", "syncDirection", "isActive", "includeSubtree", "preserveHierarchy", "contentFormat"]; + } + + mappingId!: string; + noteId!: string; + filePath!: string; + syncDirection!: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium'; + isActive!: boolean; + includeSubtree!: boolean; + preserveHierarchy!: boolean; + contentFormat!: 'auto' | 'markdown' | 'html' | 'raw'; + excludePatterns?: (string | RegExp)[] | null; + lastSyncTime?: string | null; + syncErrors?: string[] | null; + + constructor(row?: FileSystemMappingRow) { + super(); + + if (!row) { + return; + } + + this.updateFromRow(row); + this.init(); + } + + updateFromRow(row: FileSystemMappingRow) { + this.update([ + row.mappingId, + row.noteId, + row.filePath, + row.syncDirection || 'bidirectional', + row.isActive !== undefined ? row.isActive : 1, + row.includeSubtree !== undefined ? row.includeSubtree : 0, + row.preserveHierarchy !== undefined ? row.preserveHierarchy : 1, + row.contentFormat || 'auto', + row.excludePatterns, + row.lastSyncTime, + row.syncErrors, + row.dateCreated, + row.dateModified, + row.utcDateCreated, + row.utcDateModified + ]); + } + + update([ + mappingId, + noteId, + filePath, + syncDirection, + isActive, + includeSubtree, + preserveHierarchy, + contentFormat, + excludePatterns, + lastSyncTime, + syncErrors, + dateCreated, + dateModified, + utcDateCreated, + utcDateModified + ]: any) { + this.mappingId = mappingId; + this.noteId = noteId; + this.filePath = filePath; + this.syncDirection = syncDirection || 'bidirectional'; + this.isActive = !!isActive; + this.includeSubtree = !!includeSubtree; + this.preserveHierarchy = !!preserveHierarchy; + this.contentFormat = contentFormat || 'auto'; + + // Parse JSON strings for arrays + try { + this.excludePatterns = excludePatterns ? JSON.parse(excludePatterns) : null; + } catch { + this.excludePatterns = null; + } + + try { + this.syncErrors = syncErrors ? JSON.parse(syncErrors) : null; + } catch { + this.syncErrors = null; + } + + this.lastSyncTime = lastSyncTime; + this.dateCreated = dateCreated; + this.dateModified = dateModified; + this.utcDateCreated = utcDateCreated; + this.utcDateModified = utcDateModified; + + return this; + } + + override init() { + if (this.mappingId) { + this.becca.fileSystemMappings = this.becca.fileSystemMappings || {}; + this.becca.fileSystemMappings[this.mappingId] = this; + } + } + + get note() { + return this.becca.notes[this.noteId]; + } + + getNote() { + const note = this.becca.getNote(this.noteId); + if (!note) { + throw new Error(`Note '${this.noteId}' for file system mapping '${this.mappingId}' does not exist.`); + } + return note; + } + + /** + * Check if the mapping allows syncing from Trilium to disk + */ + get canSyncToDisk(): boolean { + return this.isActive && (this.syncDirection === 'bidirectional' || this.syncDirection === 'trilium_to_disk'); + } + + /** + * Check if the mapping allows syncing from disk to Trilium + */ + get canSyncFromDisk(): boolean { + return this.isActive && (this.syncDirection === 'bidirectional' || this.syncDirection === 'disk_to_trilium'); + } + + /** + * Add a sync error to the errors list + */ + addSyncError(error: string) { + if (!this.syncErrors) { + this.syncErrors = []; + } + this.syncErrors.push(error); + + // Keep only the last 10 errors + if (this.syncErrors.length > 10) { + this.syncErrors = this.syncErrors.slice(-10); + } + + this.save(); + } + + /** + * Clear all sync errors + */ + clearSyncErrors() { + this.syncErrors = null; + this.save(); + } + + /** + * Update the last sync time + */ + updateLastSyncTime() { + this.lastSyncTime = dateUtils.utcNowDateTime(); + this.save(); + } + + override beforeSaving() { + super.beforeSaving(); + + if (!this.mappingId) { + this.mappingId = newEntityId(); + } + + if (!this.dateCreated) { + this.dateCreated = dateUtils.localNowDateTime(); + } + + if (!this.utcDateCreated) { + this.utcDateCreated = dateUtils.utcNowDateTime(); + } + + this.dateModified = dateUtils.localNowDateTime(); + this.utcDateModified = dateUtils.utcNowDateTime(); + } + + getPojo(): FileSystemMappingRow { + return { + mappingId: this.mappingId, + noteId: this.noteId, + filePath: this.filePath, + syncDirection: this.syncDirection, + isActive: this.isActive ? 1 : 0, + includeSubtree: this.includeSubtree ? 1 : 0, + preserveHierarchy: this.preserveHierarchy ? 1 : 0, + contentFormat: this.contentFormat, + excludePatterns: this.excludePatterns ? JSON.stringify(this.excludePatterns) : null, + lastSyncTime: this.lastSyncTime, + syncErrors: this.syncErrors ? JSON.stringify(this.syncErrors) : null, + dateCreated: this.dateCreated, + dateModified: this.dateModified, + utcDateCreated: this.utcDateCreated, + utcDateModified: this.utcDateModified + }; + } +} + +export default BFileSystemMapping; diff --git a/apps/server/src/becca/entity_constructor.ts b/apps/server/src/becca/entity_constructor.ts index 18f7a14c7..05c04777b 100644 --- a/apps/server/src/becca/entity_constructor.ts +++ b/apps/server/src/becca/entity_constructor.ts @@ -9,6 +9,8 @@ import BNote from "./entities/bnote.js"; import BOption from "./entities/boption.js"; import BRecentNote from "./entities/brecent_note.js"; import BRevision from "./entities/brevision.js"; +import BFileSystemMapping from "./entities/bfile_system_mapping.js"; +import BFileNoteMapping from "./entities/bfile_note_mapping.js"; type EntityClass = new (row?: any) => AbstractBeccaEntity; @@ -21,7 +23,9 @@ const ENTITY_NAME_TO_ENTITY: Record & EntityClass> notes: BNote, options: BOption, recent_notes: BRecentNote, - revisions: BRevision + revisions: BRevision, + file_system_mappings: BFileSystemMapping, + file_note_mappings: BFileNoteMapping }; function getEntityFromEntityName(entityName: keyof typeof ENTITY_NAME_TO_ENTITY) { diff --git a/apps/server/src/migrations/migrations.ts b/apps/server/src/migrations/migrations.ts index 2757b4c25..edf506a09 100644 --- a/apps/server/src/migrations/migrations.ts +++ b/apps/server/src/migrations/migrations.ts @@ -6,6 +6,64 @@ // Migrations should be kept in descending order, so the latest migration is first. const MIGRATIONS: (SqlMigration | JsMigration)[] = [ + // Add file system mapping support + { + version: 234, + sql: /*sql*/` + -- Table to store file system mappings for notes and subtrees + CREATE TABLE IF NOT EXISTS "file_system_mappings" ( + "mappingId" TEXT NOT NULL PRIMARY KEY, + "noteId" TEXT NOT NULL, + "filePath" TEXT NOT NULL, + "syncDirection" TEXT NOT NULL DEFAULT 'bidirectional', -- 'bidirectional', 'trilium_to_disk', 'disk_to_trilium' + "isActive" INTEGER NOT NULL DEFAULT 1, + "includeSubtree" INTEGER NOT NULL DEFAULT 0, + "preserveHierarchy" INTEGER NOT NULL DEFAULT 1, + "contentFormat" TEXT NOT NULL DEFAULT 'auto', -- 'auto', 'markdown', 'html', 'raw' + "excludePatterns" TEXT DEFAULT NULL, -- JSON array of glob patterns to exclude + "lastSyncTime" TEXT DEFAULT NULL, + "syncErrors" TEXT DEFAULT NULL, -- JSON array of recent sync errors + "dateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL + ); + + -- Index for quick lookup by noteId + CREATE INDEX "IDX_file_system_mappings_noteId" ON "file_system_mappings" ("noteId"); + -- Index for finding active mappings + CREATE INDEX "IDX_file_system_mappings_active" ON "file_system_mappings" ("isActive", "noteId"); + -- Unique constraint to prevent duplicate mappings for same note + CREATE UNIQUE INDEX "IDX_file_system_mappings_note_unique" ON "file_system_mappings" ("noteId"); + + -- Table to track file to note mappings for efficient lookups + CREATE TABLE IF NOT EXISTS "file_note_mappings" ( + "fileNoteId" TEXT NOT NULL PRIMARY KEY, + "mappingId" TEXT NOT NULL, + "noteId" TEXT NOT NULL, + "filePath" TEXT NOT NULL, + "fileHash" TEXT DEFAULT NULL, + "fileModifiedTime" TEXT DEFAULT NULL, + "lastSyncTime" TEXT DEFAULT NULL, + "syncStatus" TEXT NOT NULL DEFAULT 'synced', -- 'synced', 'pending', 'conflict', 'error' + "dateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL, + FOREIGN KEY ("mappingId") REFERENCES "file_system_mappings" ("mappingId") ON DELETE CASCADE, + FOREIGN KEY ("noteId") REFERENCES "notes" ("noteId") ON DELETE CASCADE + ); + + -- Index for quick lookup by file path + CREATE INDEX "IDX_file_note_mappings_filePath" ON "file_note_mappings" ("filePath"); + -- Index for finding notes by mapping + CREATE INDEX "IDX_file_note_mappings_mapping" ON "file_note_mappings" ("mappingId", "noteId"); + -- Index for finding pending syncs + CREATE INDEX "IDX_file_note_mappings_sync_status" ON "file_note_mappings" ("syncStatus", "mappingId"); + -- Unique constraint for file path per mapping + CREATE UNIQUE INDEX "IDX_file_note_mappings_file_unique" ON "file_note_mappings" ("mappingId", "filePath"); + ` + }, // Migrate geo map to collection { version: 233, diff --git a/apps/server/src/routes/api/file_system_sync.ts b/apps/server/src/routes/api/file_system_sync.ts new file mode 100644 index 000000000..cbc7ce4ba --- /dev/null +++ b/apps/server/src/routes/api/file_system_sync.ts @@ -0,0 +1,366 @@ +"use strict"; + +import express from "express"; +import becca from "../../becca/becca.js"; +import BFileSystemMapping from "../../becca/entities/bfile_system_mapping.js"; +import fileSystemSyncInit from "../../services/file_system_sync_init.js"; +import log from "../../services/log.js"; +import ValidationError from "../../errors/validation_error.js"; +import fs from "fs-extra"; +import path from "path"; + +const router = express.Router(); + +interface FileStat { + isFile: boolean; + isDirectory: boolean; + size: number; + modified: string; +} + +// Get all file system mappings +router.get("/mappings", (req, res) => { + try { + const mappings = Object.values(becca.fileSystemMappings || {}).map(mapping => ({ + mappingId: mapping.mappingId, + noteId: mapping.noteId, + filePath: mapping.filePath, + syncDirection: mapping.syncDirection, + isActive: mapping.isActive, + includeSubtree: mapping.includeSubtree, + preserveHierarchy: mapping.preserveHierarchy, + contentFormat: mapping.contentFormat, + excludePatterns: mapping.excludePatterns, + lastSyncTime: mapping.lastSyncTime, + syncErrors: mapping.syncErrors, + dateCreated: mapping.dateCreated, + dateModified: mapping.dateModified + })); + + res.json(mappings); + } catch (error) { + log.error(`Error getting file system mappings: ${error}`); + res.status(500).json({ error: "Failed to get file system mappings" }); + } +}); + +// Get a specific file system mapping +router.get("/mappings/:mappingId", (req, res) => { + try { + const { mappingId } = req.params; + const mapping = becca.fileSystemMappings[mappingId]; + + if (!mapping) { + return res.status(404).json({ error: "Mapping not found" }); + } + + res.json({ + mappingId: mapping.mappingId, + noteId: mapping.noteId, + filePath: mapping.filePath, + syncDirection: mapping.syncDirection, + isActive: mapping.isActive, + includeSubtree: mapping.includeSubtree, + preserveHierarchy: mapping.preserveHierarchy, + contentFormat: mapping.contentFormat, + excludePatterns: mapping.excludePatterns, + lastSyncTime: mapping.lastSyncTime, + syncErrors: mapping.syncErrors, + dateCreated: mapping.dateCreated, + dateModified: mapping.dateModified + }); + } catch (error) { + log.error(`Error getting file system mapping: ${error}`); + res.status(500).json({ error: "Failed to get file system mapping" }); + } +}); + +// Create a new file system mapping +router.post("/mappings", async (req, res) => { + try { + const { + noteId, + filePath, + syncDirection = 'bidirectional', + isActive = true, + includeSubtree = false, + preserveHierarchy = true, + contentFormat = 'auto', + excludePatterns = null + } = req.body; + + // Validate required fields + if (!noteId || !filePath) { + throw new ValidationError("noteId and filePath are required"); + } + + // Validate note exists + const note = becca.notes[noteId]; + if (!note) { + throw new ValidationError(`Note ${noteId} not found`); + } + + // Check if mapping already exists for this note + const existingMapping = becca.getFileSystemMappingByNoteId(noteId); + if (existingMapping) { + throw new ValidationError(`File system mapping already exists for note ${noteId}`); + } + + // Validate file path exists + const normalizedPath = path.resolve(filePath); + if (!await fs.pathExists(normalizedPath)) { + throw new ValidationError(`File path does not exist: ${normalizedPath}`); + } + + // Validate sync direction + const validDirections = ['bidirectional', 'trilium_to_disk', 'disk_to_trilium']; + if (!validDirections.includes(syncDirection)) { + throw new ValidationError(`Invalid sync direction. Must be one of: ${validDirections.join(', ')}`); + } + + // Validate content format + const validFormats = ['auto', 'markdown', 'html', 'raw']; + if (!validFormats.includes(contentFormat)) { + throw new ValidationError(`Invalid content format. Must be one of: ${validFormats.join(', ')}`); + } + + // Create the mapping + const mapping = new BFileSystemMapping({ + noteId, + filePath: normalizedPath, + syncDirection, + isActive: isActive ? 1 : 0, + includeSubtree: includeSubtree ? 1 : 0, + preserveHierarchy: preserveHierarchy ? 1 : 0, + contentFormat, + excludePatterns: Array.isArray(excludePatterns) ? JSON.stringify(excludePatterns) : excludePatterns + }).save(); + + log.info(`Created file system mapping ${mapping.mappingId} for note ${noteId} -> ${normalizedPath}`); + + res.status(201).json({ + mappingId: mapping.mappingId, + noteId: mapping.noteId, + filePath: mapping.filePath, + syncDirection: mapping.syncDirection, + isActive: mapping.isActive, + includeSubtree: mapping.includeSubtree, + preserveHierarchy: mapping.preserveHierarchy, + contentFormat: mapping.contentFormat, + excludePatterns: mapping.excludePatterns + }); + + } catch (error) { + if (error instanceof ValidationError) { + res.status(400).json({ error: error.message }); + } else { + log.error(`Error creating file system mapping: ${error}`); + res.status(500).json({ error: "Failed to create file system mapping" }); + } + } +}); + +// Update a file system mapping +router.put("/mappings/:mappingId", async (req, res) => { + try { + const { mappingId } = req.params; + const mapping = becca.fileSystemMappings[mappingId]; + + if (!mapping) { + return res.status(404).json({ error: "Mapping not found" }); + } + + const { + filePath, + syncDirection, + isActive, + includeSubtree, + preserveHierarchy, + contentFormat, + excludePatterns + } = req.body; + + // Update fields if provided + if (filePath !== undefined) { + const normalizedPath = path.resolve(filePath); + if (!await fs.pathExists(normalizedPath)) { + throw new ValidationError(`File path does not exist: ${normalizedPath}`); + } + mapping.filePath = normalizedPath; + } + + if (syncDirection !== undefined) { + const validDirections = ['bidirectional', 'trilium_to_disk', 'disk_to_trilium']; + if (!validDirections.includes(syncDirection)) { + throw new ValidationError(`Invalid sync direction. Must be one of: ${validDirections.join(', ')}`); + } + mapping.syncDirection = syncDirection; + } + + if (isActive !== undefined) { + mapping.isActive = !!isActive; + } + + if (includeSubtree !== undefined) { + mapping.includeSubtree = !!includeSubtree; + } + + if (preserveHierarchy !== undefined) { + mapping.preserveHierarchy = !!preserveHierarchy; + } + + if (contentFormat !== undefined) { + const validFormats = ['auto', 'markdown', 'html', 'raw']; + if (!validFormats.includes(contentFormat)) { + throw new ValidationError(`Invalid content format. Must be one of: ${validFormats.join(', ')}`); + } + mapping.contentFormat = contentFormat; + } + + if (excludePatterns !== undefined) { + mapping.excludePatterns = Array.isArray(excludePatterns) ? excludePatterns : null; + } + + mapping.save(); + + log.info(`Updated file system mapping ${mappingId}`); + + res.json({ + mappingId: mapping.mappingId, + noteId: mapping.noteId, + filePath: mapping.filePath, + syncDirection: mapping.syncDirection, + isActive: mapping.isActive, + includeSubtree: mapping.includeSubtree, + preserveHierarchy: mapping.preserveHierarchy, + contentFormat: mapping.contentFormat, + excludePatterns: mapping.excludePatterns + }); + + } catch (error) { + if (error instanceof ValidationError) { + res.status(400).json({ error: error.message }); + } else { + log.error(`Error updating file system mapping: ${error}`); + res.status(500).json({ error: "Failed to update file system mapping" }); + } + } +}); + +// Delete a file system mapping +router.delete("/mappings/:mappingId", (req, res) => { + try { + const { mappingId } = req.params; + const mapping = becca.fileSystemMappings[mappingId]; + + if (!mapping) { + return res.status(404).json({ error: "Mapping not found" }); + } + + mapping.markAsDeleted(); + + log.info(`Deleted file system mapping ${mappingId}`); + + res.json({ success: true }); + + } catch (error) { + log.error(`Error deleting file system mapping: ${error}`); + res.status(500).json({ error: "Failed to delete file system mapping" }); + } +}); + +// Trigger full sync for a mapping +router.post("/mappings/:mappingId/sync", async (req, res) => { + try { + const { mappingId } = req.params; + + if (!fileSystemSyncInit.isInitialized()) { + return res.status(503).json({ error: "File system sync is not initialized" }); + } + + const result = await fileSystemSyncInit.fullSync(mappingId); + + if (result.success) { + res.json(result); + } else { + res.status(400).json(result); + } + + } catch (error) { + log.error(`Error triggering sync: ${error}`); + res.status(500).json({ error: "Failed to trigger sync" }); + } +}); + +// Get sync status for all mappings +router.get("/status", (req, res) => { + try { + const status = fileSystemSyncInit.getStatus(); + res.json(status); + } catch (error) { + log.error(`Error getting sync status: ${error}`); + res.status(500).json({ error: "Failed to get sync status" }); + } +}); + +// Enable/disable file system sync +router.post("/enable", async (req, res) => { + try { + await fileSystemSyncInit.enable(); + res.json({ success: true, message: "File system sync enabled" }); + } catch (error) { + log.error(`Error enabling file system sync: ${error}`); + res.status(500).json({ error: "Failed to enable file system sync" }); + } +}); + +router.post("/disable", async (req, res) => { + try { + await fileSystemSyncInit.disable(); + res.json({ success: true, message: "File system sync disabled" }); + } catch (error) { + log.error(`Error disabling file system sync: ${error}`); + res.status(500).json({ error: "Failed to disable file system sync" }); + } +}); + +// Validate file path +router.post("/validate-path", async (req, res) => { + try { + const { filePath } = req.body; + + if (!filePath) { + throw new ValidationError("filePath is required"); + } + + const normalizedPath = path.resolve(filePath); + const exists = await fs.pathExists(normalizedPath); + + let stats: FileStat | null = null; + if (exists) { + const fileStats = await fs.stat(normalizedPath); + stats = { + isFile: fileStats.isFile(), + isDirectory: fileStats.isDirectory(), + size: fileStats.size, + modified: fileStats.mtime.toISOString() + }; + } + + res.json({ + path: normalizedPath, + exists, + stats + }); + + } catch (error) { + if (error instanceof ValidationError) { + res.status(400).json({ error: error.message }); + } else { + log.error(`Error validating file path: ${error}`); + res.status(500).json({ error: "Failed to validate file path" }); + } + } +}); + +export default router; diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index f1aeb9209..aafada0d9 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -59,6 +59,7 @@ import openaiRoute from "./api/openai.js"; import anthropicRoute from "./api/anthropic.js"; import llmRoute from "./api/llm.js"; import systemInfoRoute from "./api/system_info.js"; +import fileSystemSyncRoute from "./api/file_system_sync.js"; import etapiAuthRoutes from "../etapi/auth.js"; import etapiAppInfoRoutes from "../etapi/app_info.js"; @@ -385,6 +386,9 @@ function register(app: express.Application) { asyncApiRoute(GET, "/api/llm/providers/openai/models", openaiRoute.listModels); asyncApiRoute(GET, "/api/llm/providers/anthropic/models", anthropicRoute.listModels); + // File system sync API + app.use("/api/file-system-sync", [auth.checkApiAuthOrElectron, csrfMiddleware], fileSystemSyncRoute); + // API Documentation apiDocsRoute(app); diff --git a/apps/server/src/services/app_info.ts b/apps/server/src/services/app_info.ts index 0def56253..f3e7d6d77 100644 --- a/apps/server/src/services/app_info.ts +++ b/apps/server/src/services/app_info.ts @@ -3,7 +3,7 @@ import build from "./build.js"; import packageJson from "../../package.json" with { type: "json" }; import dataDir from "./data_dir.js"; -const APP_DB_VERSION = 233; +const APP_DB_VERSION = 234; const SYNC_VERSION = 36; const CLIPPER_PROTOCOL_VERSION = "1.0"; diff --git a/apps/server/src/services/file_system_content_converter.ts b/apps/server/src/services/file_system_content_converter.ts new file mode 100644 index 000000000..e2da0719a --- /dev/null +++ b/apps/server/src/services/file_system_content_converter.ts @@ -0,0 +1,464 @@ +"use strict"; + +import path from "path"; +import log from "./log.js"; +import markdownExportService from "./export/markdown.js"; +import markdownImportService from "./import/markdown.js"; +import BNote from "../becca/entities/bnote.js"; +import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js"; +import utils from "./utils.js"; +import { type NoteType } from "@triliumnext/commons"; + +export interface ConversionResult { + content: string | Buffer; + attributes?: Array<{ type: 'label' | 'relation'; name: string; value: string; isInheritable?: boolean }>; + mime?: string; + type?: NoteType; +} + +export interface ConversionOptions { + preserveAttributes?: boolean; + includeFrontmatter?: boolean; + relativeImagePaths?: boolean; +} + +/** + * Content converter for file system sync operations + * Handles conversion between Trilium note formats and file system formats + */ +class FileSystemContentConverter { + + /** + * Convert note content to file format based on mapping configuration + */ + async noteToFile(note: BNote, mapping: BFileSystemMapping, filePath: string, options: ConversionOptions = {}): Promise { + const fileExt = path.extname(filePath).toLowerCase(); + const contentFormat = mapping.contentFormat === 'auto' ? this.detectFormatFromExtension(fileExt) : mapping.contentFormat; + + switch (contentFormat) { + case 'markdown': + return this.noteToMarkdown(note, options); + case 'html': + return this.noteToHtml(note, options); + case 'raw': + default: + return this.noteToRaw(note, options); + } + } + + /** + * Convert file content to note format based on mapping configuration + */ + async fileToNote(fileContent: string | Buffer, mapping: BFileSystemMapping, filePath: string, options: ConversionOptions = {}): Promise { + const fileExt = path.extname(filePath).toLowerCase(); + const contentFormat = mapping.contentFormat === 'auto' ? this.detectFormatFromExtension(fileExt) : mapping.contentFormat; + + // Convert Buffer to string for text formats + const content = Buffer.isBuffer(fileContent) ? fileContent.toString('utf8') : fileContent; + + switch (contentFormat) { + case 'markdown': + // Extract title from note for proper H1 deduplication + const note = mapping.note; + const title = note ? note.title : path.basename(filePath, path.extname(filePath)); + return this.markdownToNote(content, options, title); + case 'html': + return this.htmlToNote(content, options); + case 'raw': + default: + return this.rawToNote(fileContent, fileExt, options); + } + } + + /** + * Detect content format from file extension + */ + private detectFormatFromExtension(extension: string): 'markdown' | 'html' | 'raw' { + const markdownExts = ['.md', '.markdown', '.mdown', '.mkd']; + const htmlExts = ['.html', '.htm']; + + if (markdownExts.includes(extension)) { + return 'markdown'; + } else if (htmlExts.includes(extension)) { + return 'html'; + } else { + return 'raw'; + } + } + + /** + * Convert note to Markdown format + */ + private async noteToMarkdown(note: BNote, options: ConversionOptions): Promise { + try { + let content = note.getContent() as string; + + // Convert HTML content to Markdown + if (note.type === 'text' && note.mime === 'text/html') { + content = markdownExportService.toMarkdown(content); + } + + // Add frontmatter with note attributes if requested + if (options.includeFrontmatter && options.preserveAttributes) { + const frontmatter = this.createFrontmatter(note); + if (frontmatter) { + content = `---\n${frontmatter}\n---\n\n${content}`; + } + } + + return { + content, + mime: 'text/markdown', + type: 'text' + }; + } catch (error) { + log.error(`Error converting note ${note.noteId} to Markdown: ${error}`); + throw error; + } + } + + /** + * Convert note to HTML format + */ + private async noteToHtml(note: BNote, options: ConversionOptions): Promise { + let content = note.getContent() as string; + + // If note is already HTML, just clean it up + if (note.type === 'text' && note.mime === 'text/html') { + // Could add HTML processing here if needed + } else if (note.type === 'code') { + // Wrap code content in pre/code tags + const language = this.getLanguageFromMime(note.mime); + content = `
    ${utils.escapeHtml(content)}
    `; + } + + // Add HTML frontmatter as comments if requested + if (options.includeFrontmatter && options.preserveAttributes) { + const frontmatter = this.createFrontmatter(note); + if (frontmatter) { + content = `\n\n${content}`; + } + } + + return { + content, + mime: 'text/html', + type: 'text' + }; + } + + /** + * Convert note to raw format (preserve original content) + */ + private async noteToRaw(note: BNote, options: ConversionOptions): Promise { + const content = note.getContent(); + + return { + content, + mime: note.mime, + type: note.type + }; + } + + /** + * Convert Markdown content to note format + */ + private async markdownToNote(content: string, options: ConversionOptions, title: string = ''): Promise { + try { + let processedContent = content; + let attributes: ConversionResult['attributes'] = []; + + // Extract frontmatter if present + if (options.preserveAttributes) { + const frontmatterResult = this.extractFrontmatter(content); + processedContent = frontmatterResult.content; + attributes = frontmatterResult.attributes; + } + + // Convert Markdown to HTML using the correct method + // The title helps deduplicate

    tags with the note title + const htmlContent = markdownImportService.renderToHtml(processedContent, title); + + return { + content: htmlContent, + attributes, + mime: 'text/html', + type: 'text' + }; + } catch (error) { + log.error(`Error converting Markdown to note: ${error}`); + throw error; + } + } + + /** + * Convert HTML content to note format + */ + private async htmlToNote(content: string, options: ConversionOptions): Promise { + let processedContent = content; + let attributes: ConversionResult['attributes'] = []; + + // Extract HTML comment frontmatter if present + if (options.preserveAttributes) { + const frontmatterResult = this.extractHtmlFrontmatter(content); + processedContent = frontmatterResult.content; + attributes = frontmatterResult.attributes; + } + + return { + content: processedContent, + attributes, + mime: 'text/html', + type: 'text' + }; + } + + /** + * Convert raw content to note format + */ + private async rawToNote(content: string | Buffer, extension: string, options: ConversionOptions): Promise { + // Determine note type and mime based on file extension + const { type, mime } = this.getTypeAndMimeFromExtension(extension); + + return { + content, + mime, + type + }; + } + + /** + * Create YAML frontmatter from note attributes + */ + private createFrontmatter(note: BNote): string | null { + const attributes = note.getOwnedAttributes(); + if (attributes.length === 0) { + return null; + } + + const yamlLines: string[] = []; + yamlLines.push(`title: "${note.title.replace(/"/g, '\\"')}"`); + yamlLines.push(`noteId: "${note.noteId}"`); + yamlLines.push(`type: "${note.type}"`); + yamlLines.push(`mime: "${note.mime}"`); + + const labels = attributes.filter(attr => attr.type === 'label'); + const relations = attributes.filter(attr => attr.type === 'relation'); + + if (labels.length > 0) { + yamlLines.push('labels:'); + for (const label of labels) { + const inheritable = label.isInheritable ? ' (inheritable)' : ''; + yamlLines.push(` - name: "${label.name}"`); + yamlLines.push(` value: "${label.value.replace(/"/g, '\\"')}"`); + if (label.isInheritable) { + yamlLines.push(` inheritable: true`); + } + } + } + + if (relations.length > 0) { + yamlLines.push('relations:'); + for (const relation of relations) { + yamlLines.push(` - name: "${relation.name}"`); + yamlLines.push(` target: "${relation.value}"`); + if (relation.isInheritable) { + yamlLines.push(` inheritable: true`); + } + } + } + + return yamlLines.join('\n'); + } + + /** + * Extract YAML frontmatter from Markdown content + */ + private extractFrontmatter(content: string): { content: string; attributes: ConversionResult['attributes'] } { + const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/; + const match = content.match(frontmatterRegex); + + if (!match) { + return { content, attributes: [] }; + } + + const frontmatterYaml = match[1]; + const mainContent = match[2]; + + try { + const attributes = this.parseFrontmatterYaml(frontmatterYaml); + return { content: mainContent, attributes }; + } catch (error) { + log.info(`Error parsing frontmatter YAML: ${error}`); + return { content, attributes: [] }; + } + } + + /** + * Extract frontmatter from HTML comments + */ + private extractHtmlFrontmatter(content: string): { content: string; attributes: ConversionResult['attributes'] } { + const frontmatterRegex = /^\s*\n([\s\S]*)$/; + const match = content.match(frontmatterRegex); + + if (!match) { + return { content, attributes: [] }; + } + + const frontmatterYaml = match[1]; + const mainContent = match[2]; + + try { + const attributes = this.parseFrontmatterYaml(frontmatterYaml); + return { content: mainContent, attributes }; + } catch (error) { + log.info(`Error parsing HTML frontmatter YAML: ${error}`); + return { content, attributes: [] }; + } + } + + /** + * Parse YAML frontmatter into attributes (simplified YAML parser) + */ + private parseFrontmatterYaml(yaml: string): ConversionResult['attributes'] { + const attributes: ConversionResult['attributes'] = []; + const lines = yaml.split('\n'); + + let currentSection: 'labels' | 'relations' | null = null; + let currentItem: any = {}; + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed === 'labels:') { + currentSection = 'labels'; + continue; + } else if (trimmed === 'relations:') { + currentSection = 'relations'; + continue; + } else if (trimmed.startsWith('- name:')) { + // Save previous item if exists + if (currentItem.name && currentSection) { + attributes.push({ + type: currentSection === 'labels' ? 'label' : 'relation', + name: currentItem.name, + value: currentItem.value || currentItem.target || '', + isInheritable: currentItem.inheritable || false + }); + } + + currentItem = { name: this.extractQuotedValue(trimmed) }; + } else if (trimmed.startsWith('name:')) { + currentItem.name = this.extractQuotedValue(trimmed); + } else if (trimmed.startsWith('value:')) { + currentItem.value = this.extractQuotedValue(trimmed); + } else if (trimmed.startsWith('target:')) { + currentItem.target = this.extractQuotedValue(trimmed); + } else if (trimmed.startsWith('inheritable:')) { + currentItem.inheritable = trimmed.includes('true'); + } + } + + // Save last item + if (currentItem.name && currentSection) { + attributes.push({ + type: currentSection === 'labels' ? 'label' : 'relation', + name: currentItem.name, + value: currentItem.value || currentItem.target || '', + isInheritable: currentItem.inheritable || false + }); + } + + return attributes; + } + + /** + * Extract quoted value from YAML line + */ + private extractQuotedValue(line: string): string { + const match = line.match(/:\s*"([^"]+)"/); + return match ? match[1].replace(/\\"/g, '"') : ''; + } + + /** + * Get language identifier from MIME type + */ + private getLanguageFromMime(mime: string): string { + const mimeToLang: Record = { + 'application/javascript': 'javascript', + 'text/javascript': 'javascript', + 'application/typescript': 'typescript', + 'text/typescript': 'typescript', + 'application/json': 'json', + 'text/css': 'css', + 'text/html': 'html', + 'application/xml': 'xml', + 'text/xml': 'xml', + 'text/x-python': 'python', + 'text/x-java': 'java', + 'text/x-csharp': 'csharp', + 'text/x-sql': 'sql', + 'text/x-sh': 'bash', + 'text/x-yaml': 'yaml' + }; + + return mimeToLang[mime] || 'text'; + } + + /** + * Get note type and MIME type from file extension + */ + private getTypeAndMimeFromExtension(extension: string): { type: NoteType; mime: string } { + const extToType: Record = { + '.txt': { type: 'text', mime: 'text/plain' }, + '.md': { type: 'text', mime: 'text/markdown' }, + '.html': { type: 'text', mime: 'text/html' }, + '.htm': { type: 'text', mime: 'text/html' }, + '.js': { type: 'code', mime: 'application/javascript' }, + '.ts': { type: 'code', mime: 'application/typescript' }, + '.json': { type: 'code', mime: 'application/json' }, + '.css': { type: 'code', mime: 'text/css' }, + '.xml': { type: 'code', mime: 'application/xml' }, + '.py': { type: 'code', mime: 'text/x-python' }, + '.java': { type: 'code', mime: 'text/x-java' }, + '.cs': { type: 'code', mime: 'text/x-csharp' }, + '.sql': { type: 'code', mime: 'text/x-sql' }, + '.sh': { type: 'code', mime: 'text/x-sh' }, + '.yaml': { type: 'code', mime: 'text/x-yaml' }, + '.yml': { type: 'code', mime: 'text/x-yaml' }, + '.png': { type: 'image', mime: 'image/png' }, + '.jpg': { type: 'image', mime: 'image/jpeg' }, + '.jpeg': { type: 'image', mime: 'image/jpeg' }, + '.gif': { type: 'image', mime: 'image/gif' }, + '.svg': { type: 'image', mime: 'image/svg+xml' } + }; + + return extToType[extension] || { type: 'file', mime: 'application/octet-stream' }; + } + + /** + * Validate if a file type is supported for sync + */ + isSupportedFileType(filePath: string): boolean { + const extension = path.extname(filePath).toLowerCase(); + const textExtensions = ['.txt', '.md', '.html', '.htm', '.js', '.ts', '.json', '.css', '.xml', '.py', '.java', '.cs', '.sql', '.sh', '.yaml', '.yml']; + const binaryExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.pdf']; + + return textExtensions.includes(extension) || binaryExtensions.includes(extension); + } + + /** + * Check if file should be treated as binary + */ + isBinaryFile(filePath: string): boolean { + const extension = path.extname(filePath).toLowerCase(); + const binaryExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.pdf', '.doc', '.docx', '.zip', '.tar', '.gz']; + + return binaryExtensions.includes(extension); + } +} + +// Create singleton instance +const fileSystemContentConverter = new FileSystemContentConverter(); + +export default fileSystemContentConverter; diff --git a/apps/server/src/services/file_system_sync.ts b/apps/server/src/services/file_system_sync.ts new file mode 100644 index 000000000..fb29a5583 --- /dev/null +++ b/apps/server/src/services/file_system_sync.ts @@ -0,0 +1,749 @@ +"use strict"; + +import fs from "fs-extra"; +import path from "path"; +import crypto from "crypto"; +import log from "./log.js"; +import becca from "../becca/becca.js"; +import BNote from "../becca/entities/bnote.js"; +import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js"; +import BFileNoteMapping from "../becca/entities/bfile_note_mapping.js"; +import BAttribute from "../becca/entities/battribute.js"; +import BBranch from "../becca/entities/bbranch.js"; +import fileSystemContentConverter from "./file_system_content_converter.js"; +import fileSystemWatcher from "./file_system_watcher.js"; +import eventService from "./events.js"; +import noteService from "./notes.js"; + +export interface SyncResult { + success: boolean; + message?: string; + conflicts?: ConflictInfo[]; +} + +export interface ConflictInfo { + type: 'content' | 'structure' | 'metadata'; + filePath: string; + noteId: string; + fileModified: string; + noteModified: string; + description: string; +} + +export interface SyncStats { + filesProcessed: number; + notesCreated: number; + notesUpdated: number; + filesCreated: number; + filesUpdated: number; + conflicts: number; + errors: number; +} + +/** + * Bidirectional sync engine between Trilium notes and file system + */ +class FileSystemSync { + private isInitialized = false; + private syncInProgress = new Set(); // Track ongoing syncs by mapping ID + + constructor() { + this.setupEventHandlers(); + } + + /** + * Initialize the sync engine + */ + async init() { + if (this.isInitialized) { + return; + } + + log.info('Initializing file system sync engine...'); + + // Initialize file system watcher + await fileSystemWatcher.init(); + + this.isInitialized = true; + log.info('File system sync engine initialized'); + } + + /** + * Shutdown the sync engine + */ + async shutdown() { + if (!this.isInitialized) { + return; + } + + log.info('Shutting down file system sync engine...'); + + await fileSystemWatcher.shutdown(); + + this.isInitialized = false; + log.info('File system sync engine shutdown complete'); + } + + /** + * Setup event handlers for file changes and note changes + */ + private setupEventHandlers() { + // Handle file changes from watcher + eventService.subscribe('FILE_CHANGED', async ({ fileNoteMapping, mapping, fileContent, isNew }) => { + await this.handleFileChanged(fileNoteMapping, mapping, fileContent, isNew); + }); + + eventService.subscribe('FILE_DELETED', async ({ fileNoteMapping, mapping }) => { + await this.handleFileDeleted(fileNoteMapping, mapping); + }); + + // Handle note changes + eventService.subscribe(eventService.NOTE_CONTENT_CHANGE, async ({ entity: note }) => { + await this.handleNoteChanged(note as BNote); + }); + + eventService.subscribe(eventService.ENTITY_CHANGED, async ({ entityName, entity }) => { + if (entityName === 'notes') { + await this.handleNoteChanged(entity as BNote); + } + }); + + eventService.subscribe(eventService.ENTITY_DELETED, async ({ entityName, entityId }) => { + if (entityName === 'notes') { + await this.handleNoteDeleted(entityId); + } + }); + } + + /** + * Perform full sync for a specific mapping + */ + async fullSync(mappingId: string): Promise { + const mapping = becca.fileSystemMappings[mappingId]; + if (!mapping) { + return { success: false, message: `Mapping ${mappingId} not found` }; + } + + if (this.syncInProgress.has(mappingId)) { + return { success: false, message: 'Sync already in progress for this mapping' }; + } + + this.syncInProgress.add(mappingId); + const stats: SyncStats = { + filesProcessed: 0, + notesCreated: 0, + notesUpdated: 0, + filesCreated: 0, + filesUpdated: 0, + conflicts: 0, + errors: 0 + }; + + try { + log.info(`Starting full sync for mapping ${mappingId}: ${mapping.filePath}`); + + if (!await fs.pathExists(mapping.filePath)) { + throw new Error(`Path does not exist: ${mapping.filePath}`); + } + + const pathStats = await fs.stat(mapping.filePath); + + if (pathStats.isFile()) { + await this.syncSingleFile(mapping, mapping.filePath, stats); + } else if (pathStats.isDirectory()) { + await this.syncDirectory(mapping, mapping.filePath, stats); + } + + mapping.updateLastSyncTime(); + mapping.clearSyncErrors(); + + log.info(`Full sync completed for mapping ${mappingId}. Stats: ${JSON.stringify(stats)}`); + return { success: true, message: `Sync completed successfully. ${stats.filesProcessed} files processed.` }; + + } catch (error) { + const errorMsg = `Full sync failed for mapping ${mappingId}: ${(error as Error).message}`; + log.error(errorMsg); + mapping.addSyncError(errorMsg); + stats.errors++; + return { success: false, message: errorMsg }; + } finally { + this.syncInProgress.delete(mappingId); + } + } + + /** + * Sync a single file + */ + private async syncSingleFile(mapping: BFileSystemMapping, filePath: string, stats: SyncStats) { + if (!fileSystemContentConverter.isSupportedFileType(filePath)) { + log.info(`DEBUG: Skipping unsupported file type: ${filePath}`); + return; + } + + stats.filesProcessed++; + + // Check if file note mapping exists + let fileNoteMapping = this.findFileNoteMappingByPath(mapping.mappingId, filePath); + + if (fileNoteMapping) { + await this.syncExistingFile(mapping, fileNoteMapping, stats); + } else { + await this.syncNewFile(mapping, filePath, stats); + } + } + + /** + * Sync a directory recursively + */ + private async syncDirectory(mapping: BFileSystemMapping, dirPath: string, stats: SyncStats) { + const entries = await fs.readdir(dirPath, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dirPath, entry.name); + + // Skip excluded patterns + if (this.isPathExcluded(fullPath, mapping)) { + continue; + } + + if (entry.isFile()) { + await this.syncSingleFile(mapping, fullPath, stats); + } else if (entry.isDirectory() && mapping.includeSubtree) { + await this.syncDirectory(mapping, fullPath, stats); + } + } + } + + /** + * Sync an existing file that has a note mapping + */ + private async syncExistingFile(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, stats: SyncStats) { + const filePath = fileNoteMapping.filePath; + + if (!await fs.pathExists(filePath)) { + // File was deleted + if (mapping.canSyncFromDisk) { + await this.deleteNoteFromFileMapping(fileNoteMapping, stats); + } + return; + } + + const fileStats = await fs.stat(filePath); + const fileHash = await this.calculateFileHash(filePath); + const fileModifiedTime = fileStats.mtime.toISOString(); + + const note = fileNoteMapping.note; + if (!note) { + log.info(`Note not found for file mapping: ${fileNoteMapping.noteId}`); + return; + } + + const fileChanged = fileNoteMapping.hasFileChanged(fileHash, fileModifiedTime); + const noteChanged = fileNoteMapping.hasNoteChanged(); + + if (!fileChanged && !noteChanged) { + // No changes + return; + } + + if (fileChanged && noteChanged) { + // Conflict - both changed + fileNoteMapping.markConflict(); + stats.conflicts++; + log.info(`Conflict detected for ${filePath} - both file and note modified`); + return; + } + + if (fileChanged && mapping.canSyncFromDisk) { + // Update note from file + await this.updateNoteFromFile(mapping, fileNoteMapping, fileHash, fileModifiedTime, stats); + } else if (noteChanged && mapping.canSyncToDisk) { + // Update file from note + await this.updateFileFromNote(mapping, fileNoteMapping, fileHash, fileModifiedTime, stats); + } + } + + /** + * Sync a new file that doesn't have a note mapping + */ + private async syncNewFile(mapping: BFileSystemMapping, filePath: string, stats: SyncStats) { + if (!mapping.canSyncFromDisk) { + return; + } + + try { + const fileStats = await fs.stat(filePath); + const fileHash = await this.calculateFileHash(filePath); + const fileModifiedTime = fileStats.mtime.toISOString(); + + // Create note from file + const note = await this.createNoteFromFile(mapping, filePath); + + // Create file note mapping + const fileNoteMapping = new BFileNoteMapping({ + mappingId: mapping.mappingId, + noteId: note.noteId, + filePath, + fileHash, + fileModifiedTime, + syncStatus: 'synced' + }).save(); + + stats.notesCreated++; + log.info(`Created note ${note.noteId} from file ${filePath}`); + + } catch (error) { + log.error(`Error creating note from file ${filePath}: ${error}`); + mapping.addSyncError(`Error creating note from file ${filePath}: ${(error as Error).message}`); + stats.errors++; + } + } + + /** + * Create a new note from a file + */ + private async createNoteFromFile(mapping: BFileSystemMapping, filePath: string): Promise { + const fileContent = await fs.readFile(filePath); + const fileName = path.basename(filePath, path.extname(filePath)); + + // Convert file content to note format + const conversion = await fileSystemContentConverter.fileToNote(fileContent, mapping, filePath, { + preserveAttributes: true, + includeFrontmatter: true + }); + + // Determine parent note + const parentNote = this.getParentNoteForFile(mapping, filePath); + + // Create the note + const note = new BNote({ + title: fileName, + type: conversion.type || 'text', + mime: conversion.mime || 'text/html' + }).save(); + + // Set content + note.setContent(conversion.content); + + // Create branch + new BBranch({ + noteId: note.noteId, + parentNoteId: parentNote.noteId + }).save(); + + // Add attributes from conversion + if (conversion.attributes) { + for (const attr of conversion.attributes) { + new BAttribute({ + noteId: note.noteId, + type: attr.type, + name: attr.name, + value: attr.value, + isInheritable: attr.isInheritable || false + }).save(); + } + } + + return note; + } + + /** + * Update note content from file + */ + private async updateNoteFromFile(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, fileHash: string, fileModifiedTime: string, stats: SyncStats) { + try { + const note = fileNoteMapping.getNote(); + const fileContent = await fs.readFile(fileNoteMapping.filePath); + + // Convert file content to note format + const conversion = await fileSystemContentConverter.fileToNote(fileContent, mapping, fileNoteMapping.filePath, { + preserveAttributes: true, + includeFrontmatter: true + }); + + // Update note content + note.setContent(conversion.content); + + // Update note type/mime if they changed + if (conversion.type && conversion.type !== note.type) { + note.type = conversion.type as any; + note.save(); + } + if (conversion.mime && conversion.mime !== note.mime) { + note.mime = conversion.mime; + note.save(); + } + + // Update attributes if needed + if (conversion.attributes) { + // Remove existing attributes that came from file + const existingAttrs = note.getOwnedAttributes(); + for (const attr of existingAttrs) { + if (attr.name.startsWith('_fileSync_')) { + attr.markAsDeleted(); + } + } + + // Add new attributes + for (const attr of conversion.attributes) { + new BAttribute({ + noteId: note.noteId, + type: attr.type, + name: attr.name, + value: attr.value, + isInheritable: attr.isInheritable || false + }).save(); + } + } + + fileNoteMapping.markSynced(fileHash, fileModifiedTime); + stats.notesUpdated++; + + log.info(`DEBUG: Updated note ${note.noteId} from file ${fileNoteMapping.filePath}`); + + } catch (error) { + log.error(`Error updating note from file ${fileNoteMapping.filePath}: ${error}`); + fileNoteMapping.markError(); + mapping.addSyncError(`Error updating note from file: ${(error as Error).message}`); + stats.errors++; + } + } + + /** + * Update file content from note + */ + private async updateFileFromNote(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, currentFileHash: string, currentModifiedTime: string, stats: SyncStats) { + try { + const note = fileNoteMapping.getNote(); + + // Convert note content to file format + const conversion = await fileSystemContentConverter.noteToFile(note, mapping, fileNoteMapping.filePath, { + preserveAttributes: true, + includeFrontmatter: true + }); + + // Ensure directory exists + await fs.ensureDir(path.dirname(fileNoteMapping.filePath)); + + // Write file + await fs.writeFile(fileNoteMapping.filePath, conversion.content); + + // Update file note mapping with new file info + const newStats = await fs.stat(fileNoteMapping.filePath); + const newFileHash = await this.calculateFileHash(fileNoteMapping.filePath); + + fileNoteMapping.markSynced(newFileHash, newStats.mtime.toISOString()); + stats.filesUpdated++; + + log.info(`DEBUG: Updated file ${fileNoteMapping.filePath} from note ${note.noteId}`); + + } catch (error) { + log.error(`Error updating file from note ${fileNoteMapping.noteId}: ${error}`); + fileNoteMapping.markError(); + mapping.addSyncError(`Error updating file from note: ${(error as Error).message}`); + stats.errors++; + } + } + + /** + * Handle file change event from watcher + */ + private async handleFileChanged(fileNoteMapping: BFileNoteMapping, mapping: BFileSystemMapping, fileContent: Buffer, isNew: boolean) { + if (this.syncInProgress.has(mapping.mappingId)) { + return; // Skip if full sync in progress + } + + const stats: SyncStats = { + filesProcessed: 1, + notesCreated: 0, + notesUpdated: 0, + filesCreated: 0, + filesUpdated: 0, + conflicts: 0, + errors: 0 + }; + + if (isNew) { + await this.syncNewFile(mapping, fileNoteMapping.filePath, stats); + } else { + const fileHash = crypto.createHash('sha256').update(fileContent).digest('hex'); + const fileStats = await fs.stat(fileNoteMapping.filePath); + const fileModifiedTime = fileStats.mtime.toISOString(); + + await this.syncExistingFile(mapping, fileNoteMapping, stats); + } + } + + /** + * Handle file deletion event from watcher + */ + private async handleFileDeleted(fileNoteMapping: BFileNoteMapping, mapping: BFileSystemMapping) { + if (this.syncInProgress.has(mapping.mappingId)) { + return; // Skip if full sync in progress + } + + const stats: SyncStats = { + filesProcessed: 0, + notesCreated: 0, + notesUpdated: 0, + filesCreated: 0, + filesUpdated: 0, + conflicts: 0, + errors: 0 + }; + + await this.deleteNoteFromFileMapping(fileNoteMapping, stats); + } + + /** + * Handle note change event + */ + private async handleNoteChanged(note: BNote) { + // Find all file mappings for this note + const fileMappings = this.findFileNoteMappingsByNote(note.noteId); + + for (const fileMapping of fileMappings) { + const mapping = fileMapping.mapping; + if (!mapping || !mapping.canSyncToDisk || this.syncInProgress.has(mapping.mappingId)) { + continue; + } + + // Check if note was actually modified since last sync + if (!fileMapping.hasNoteChanged()) { + continue; + } + + const stats: SyncStats = { + filesProcessed: 0, + notesCreated: 0, + notesUpdated: 0, + filesCreated: 0, + filesUpdated: 0, + conflicts: 0, + errors: 0 + }; + + // Check for conflicts + if (await fs.pathExists(fileMapping.filePath)) { + const fileStats = await fs.stat(fileMapping.filePath); + const fileHash = await this.calculateFileHash(fileMapping.filePath); + const fileModifiedTime = fileStats.mtime.toISOString(); + + if (fileMapping.hasFileChanged(fileHash, fileModifiedTime)) { + // Conflict + fileMapping.markConflict(); + log.info(`Conflict detected for note ${note.noteId} - both file and note modified`); + continue; + } + } + + // Update file from note + const currentFileHash = await this.calculateFileHash(fileMapping.filePath); + const currentModifiedTime = (await fs.stat(fileMapping.filePath)).mtime.toISOString(); + + await this.updateFileFromNote(mapping, fileMapping, currentFileHash, currentModifiedTime, stats); + } + } + + /** + * Handle note deletion event + */ + private async handleNoteDeleted(noteId: string) { + // Find all file mappings for this note + const fileMappings = this.findFileNoteMappingsByNote(noteId); + + for (const fileMapping of fileMappings) { + const mapping = fileMapping.mapping; + if (!mapping || !mapping.canSyncToDisk || this.syncInProgress.has(mapping.mappingId)) { + continue; + } + + try { + // Delete the file + if (await fs.pathExists(fileMapping.filePath)) { + await fs.remove(fileMapping.filePath); + log.info(`Deleted file ${fileMapping.filePath} for deleted note ${noteId}`); + } + + // Delete the mapping + fileMapping.markAsDeleted(); + + } catch (error) { + log.error(`Error deleting file for note ${noteId}: ${error}`); + mapping.addSyncError(`Error deleting file: ${(error as Error).message}`); + } + } + } + + /** + * Delete note when file is deleted + */ + private async deleteNoteFromFileMapping(fileNoteMapping: BFileNoteMapping, stats: SyncStats) { + try { + const note = fileNoteMapping.note; + if (note) { + note.deleteNote(); + log.info(`Deleted note ${note.noteId} for deleted file ${fileNoteMapping.filePath}`); + } + + // Delete the mapping + fileNoteMapping.markAsDeleted(); + + } catch (error) { + log.error(`Error deleting note for file ${fileNoteMapping.filePath}: ${error}`); + stats.errors++; + } + } + + /** + * Get parent note for a file based on mapping configuration + */ + private getParentNoteForFile(mapping: BFileSystemMapping, filePath: string): BNote { + const mappedNote = mapping.getNote(); + + if (!mapping.preserveHierarchy || !mapping.includeSubtree) { + return mappedNote; + } + + // Calculate relative path from mapping root + const relativePath = path.relative(mapping.filePath, path.dirname(filePath)); + + if (!relativePath || relativePath === '.') { + return mappedNote; + } + + // Create directory structure as notes + const pathParts = relativePath.split(path.sep); + let currentParent = mappedNote; + + for (const part of pathParts) { + if (!part) continue; + + // Look for existing child note with this name + let childNote = currentParent.children.find(child => child.title === part); + + if (!childNote) { + // Create new note for this directory + childNote = new BNote({ + title: part, + type: 'text', + mime: 'text/html' + }).save(); + + childNote.setContent('

    Directory note

    '); + + // Create branch (notePosition will be auto-calculated) + new BBranch({ + noteId: childNote.noteId, + parentNoteId: currentParent.noteId + }).save(); + } + + currentParent = childNote; + } + + return currentParent; + } + + /** + * Calculate SHA256 hash of a file + */ + private async calculateFileHash(filePath: string): Promise { + const content = await fs.readFile(filePath); + return crypto.createHash('sha256').update(content).digest('hex'); + } + + /** + * Check if a path should be excluded based on mapping patterns + */ + private isPathExcluded(filePath: string, mapping: BFileSystemMapping): boolean { + if (!mapping.excludePatterns) { + return false; + } + + const normalizedPath = path.normalize(filePath); + const basename = path.basename(normalizedPath); + + for (const pattern of mapping.excludePatterns) { + if (typeof pattern === 'string') { + // Simple string matching + if (normalizedPath.includes(pattern) || basename.includes(pattern)) { + return true; + } + } else if (pattern instanceof RegExp) { + // Regex pattern + if (pattern.test(normalizedPath) || pattern.test(basename)) { + return true; + } + } + } + + return false; + } + + /** + * Find file note mapping by file path + */ + private findFileNoteMappingByPath(mappingId: string, filePath: string): BFileNoteMapping | null { + const normalizedPath = path.normalize(filePath); + + for (const mapping of Object.values(becca.fileNoteMappings || {})) { + if (mapping.mappingId === mappingId && path.normalize(mapping.filePath) === normalizedPath) { + return mapping; + } + } + + return null; + } + + /** + * Find all file note mappings for a note + */ + private findFileNoteMappingsByNote(noteId: string): BFileNoteMapping[] { + const mappings: BFileNoteMapping[] = []; + + for (const mapping of Object.values(becca.fileNoteMappings || {})) { + if (mapping.noteId === noteId) { + mappings.push(mapping); + } + } + + return mappings; + } + + /** + * Get sync status for all mappings + */ + getSyncStatus() { + const status: Record = {}; + + for (const mapping of Object.values(becca.fileSystemMappings || {})) { + const fileMappings = Object.values(becca.fileNoteMappings || {}) + .filter(fm => fm.mappingId === mapping.mappingId); + + const conflicts = fileMappings.filter(fm => fm.syncStatus === 'conflict').length; + const pending = fileMappings.filter(fm => fm.syncStatus === 'pending').length; + const errors = fileMappings.filter(fm => fm.syncStatus === 'error').length; + + status[mapping.mappingId] = { + filePath: mapping.filePath, + isActive: mapping.isActive, + syncDirection: mapping.syncDirection, + fileCount: fileMappings.length, + conflicts, + pending, + errors, + lastSyncTime: mapping.lastSyncTime, + syncErrors: mapping.syncErrors, + isRunning: this.syncInProgress.has(mapping.mappingId) + }; + } + + return status; + } +} + +// Create singleton instance +const fileSystemSync = new FileSystemSync(); + +export default fileSystemSync; diff --git a/apps/server/src/services/file_system_sync_init.ts b/apps/server/src/services/file_system_sync_init.ts new file mode 100644 index 000000000..57902a335 --- /dev/null +++ b/apps/server/src/services/file_system_sync_init.ts @@ -0,0 +1,129 @@ +"use strict"; + +import log from "./log.js"; +import fileSystemSync from "./file_system_sync.js"; +import eventService from "./events.js"; +import optionService from "./options.js"; + +/** + * Initialization service for file system sync functionality + */ +class FileSystemSyncInit { + private initialized = false; + + /** + * Initialize file system sync if enabled + */ + async init() { + if (this.initialized) { + return; + } + + try { + // Check if file system sync is enabled + const isEnabled = optionService.getOption('fileSystemSyncEnabled') === 'true'; + + if (!isEnabled) { + log.info('File system sync is disabled'); + return; + } + + log.info('Initializing file system sync...'); + + // Initialize the sync engine + await fileSystemSync.init(); + + this.initialized = true; + log.info('File system sync initialized successfully'); + + } catch (error) { + log.error(`Failed to initialize file system sync: ${error}`); + throw error; + } + } + + /** + * Shutdown file system sync + */ + async shutdown() { + if (!this.initialized) { + return; + } + + try { + log.info('Shutting down file system sync...'); + + await fileSystemSync.shutdown(); + + this.initialized = false; + log.info('File system sync shutdown complete'); + + } catch (error) { + log.error(`Error shutting down file system sync: ${error}`); + } + } + + /** + * Check if file system sync is initialized + */ + isInitialized(): boolean { + return this.initialized; + } + + /** + * Get sync status + */ + getStatus() { + if (!this.initialized) { + return { enabled: false, initialized: false }; + } + + return { + enabled: true, + initialized: true, + status: fileSystemSync.getSyncStatus() + }; + } + + /** + * Enable file system sync + */ + async enable() { + optionService.setOption('fileSystemSyncEnabled', 'true'); + + if (!this.initialized) { + await this.init(); + } + + log.info('File system sync enabled'); + } + + /** + * Disable file system sync + */ + async disable() { + optionService.setOption('fileSystemSyncEnabled', 'false'); + + if (this.initialized) { + await this.shutdown(); + } + + log.info('File system sync disabled'); + } + + /** + * Perform full sync for a specific mapping + */ + async fullSync(mappingId: string) { + if (!this.initialized) { + throw new Error('File system sync is not initialized'); + } + + return await fileSystemSync.fullSync(mappingId); + } +} + +// Create singleton instance +const fileSystemSyncInit = new FileSystemSyncInit(); + +export default fileSystemSyncInit; \ No newline at end of file diff --git a/apps/server/src/services/file_system_watcher.ts b/apps/server/src/services/file_system_watcher.ts new file mode 100644 index 000000000..877cd5379 --- /dev/null +++ b/apps/server/src/services/file_system_watcher.ts @@ -0,0 +1,431 @@ +"use strict"; + +import chokidar from "chokidar"; +import path from "path"; +import fs from "fs-extra"; +import crypto from "crypto"; +import debounce from "debounce"; +import log from "./log.js"; +import becca from "../becca/becca.js"; +import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js"; +import BFileNoteMapping from "../becca/entities/bfile_note_mapping.js"; +import eventService from "./events.js"; +import { newEntityId } from "./utils.js"; +import type { FSWatcher } from "chokidar"; + +interface WatchedMapping { + mapping: BFileSystemMapping; + watcher: FSWatcher; +} + +interface FileChangeEvent { + type: 'add' | 'change' | 'unlink'; + filePath: string; + mappingId: string; + stats?: fs.Stats; +} + +class FileSystemWatcher { + private watchers: Map = new Map(); + private syncQueue: FileChangeEvent[] = []; + private isProcessing = false; + + // Debounced sync to batch multiple file changes + private processSyncQueue = debounce(this._processSyncQueue.bind(this), 500); + + constructor() { + // Subscribe to entity changes to watch for new/updated/deleted mappings + eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) => { + if (entityName === 'file_system_mappings') { + this.addWatcher(entity as BFileSystemMapping); + } + }); + + eventService.subscribe(eventService.ENTITY_CHANGED, ({ entityName, entity }) => { + if (entityName === 'file_system_mappings') { + this.updateWatcher(entity as BFileSystemMapping); + } + }); + + eventService.subscribe(eventService.ENTITY_DELETED, ({ entityName, entityId }) => { + if (entityName === 'file_system_mappings') { + this.removeWatcher(entityId); + } + }); + } + + /** + * Initialize the file system watcher by setting up watchers for all active mappings + */ + async init() { + log.info('Initializing file system watcher...'); + + try { + const mappings = Object.values(becca.fileSystemMappings || {}); + for (const mapping of mappings) { + if (mapping.isActive && mapping.canSyncFromDisk) { + await this.addWatcher(mapping); + } + } + + log.info(`File system watcher initialized with ${this.watchers.size} active mappings`); + } catch (error) { + log.error(`Failed to initialize file system watcher: ${error}`); + } + } + + /** + * Shutdown all watchers + */ + async shutdown() { + log.info('Shutting down file system watcher...'); + + for (const [mappingId, { watcher }] of this.watchers) { + await watcher.close(); + } + + this.watchers.clear(); + log.info('File system watcher shutdown complete'); + } + + /** + * Add a new file system watcher for a mapping + */ + private async addWatcher(mapping: BFileSystemMapping) { + if (this.watchers.has(mapping.mappingId)) { + await this.removeWatcher(mapping.mappingId); + } + + if (!mapping.isActive || !mapping.canSyncFromDisk) { + return; + } + + try { + // Check if the file path exists + if (!await fs.pathExists(mapping.filePath)) { + log.info(`File path does not exist for mapping ${mapping.mappingId}: ${mapping.filePath}`); + mapping.addSyncError(`File path does not exist: ${mapping.filePath}`); + return; + } + + const stats = await fs.stat(mapping.filePath); + const watchPath = stats.isDirectory() ? mapping.filePath : path.dirname(mapping.filePath); + + const watcher = chokidar.watch(watchPath, { + persistent: true, + ignoreInitial: true, + followSymlinks: false, + depth: mapping.includeSubtree ? undefined : 0, + ignored: this.buildIgnorePatterns(mapping) + }); + + watcher.on('add', (filePath, stats) => { + this.queueFileChange('add', filePath, mapping.mappingId, stats); + }); + + watcher.on('change', (filePath, stats) => { + this.queueFileChange('change', filePath, mapping.mappingId, stats); + }); + + watcher.on('unlink', (filePath) => { + this.queueFileChange('unlink', filePath, mapping.mappingId); + }); + + watcher.on('error', (error) => { + log.error(`File watcher error for mapping ${mapping.mappingId}: ${error}`); + if (error && typeof error === "object" && "message" in error && typeof error.message === 'string') { + mapping.addSyncError(`Watcher error: ${error.message}`); + } + }); + + watcher.on('ready', () => { + log.info(`File watcher ready for mapping ${mapping.mappingId}: ${mapping.filePath}`); + }); + + this.watchers.set(mapping.mappingId, { mapping, watcher }); + + } catch (error) { + log.error(`Failed to create file watcher for mapping ${mapping.mappingId}: ${error}`); + mapping.addSyncError(`Failed to create watcher: ${(error as Error).message}`); + } + } + + /** + * Update an existing watcher (remove and re-add) + */ + private async updateWatcher(mapping: BFileSystemMapping) { + await this.addWatcher(mapping); + } + + /** + * Remove a file system watcher + */ + private async removeWatcher(mappingId: string) { + const watchedMapping = this.watchers.get(mappingId); + if (watchedMapping) { + await watchedMapping.watcher.close(); + this.watchers.delete(mappingId); + log.info(`Removed file watcher for mapping ${mappingId}`); + } + } + + /** + * Build ignore patterns for chokidar based on mapping configuration + */ + private buildIgnorePatterns(mapping: BFileSystemMapping): (string | RegExp)[] { + const patterns: (string | RegExp)[] = [ + // Always ignore common temp/system files + /^\./, // Hidden files + /\.tmp$/, + /\.temp$/, + /~$/, // Backup files + /\.swp$/, // Vim swap files + /\.DS_Store$/, // macOS + /Thumbs\.db$/ // Windows + ]; + + // Add user-defined exclude patterns + if (mapping.excludePatterns) { + patterns.push(...mapping.excludePatterns); + } + + return patterns; + } + + /** + * Queue a file change event for processing + */ + private queueFileChange(type: 'add' | 'change' | 'unlink', filePath: string, mappingId: string, stats?: fs.Stats) { + this.syncQueue.push({ + type, + filePath: path.normalize(filePath), + mappingId, + stats + }); + + // Trigger debounced processing + this.processSyncQueue(); + } + + /** + * Process the sync queue (called after debounce delay) + */ + private async _processSyncQueue() { + if (this.isProcessing || this.syncQueue.length === 0) { + return; + } + + this.isProcessing = true; + const eventsToProcess = [...this.syncQueue]; + this.syncQueue = []; + + try { + // Group events by file path to handle multiple events for the same file + const eventMap = new Map(); + + for (const event of eventsToProcess) { + const key = `${event.mappingId}:${event.filePath}`; + eventMap.set(key, event); // Latest event wins + } + + // Process each unique file change + for (const event of eventMap.values()) { + await this.processFileChange(event); + } + + } catch (error) { + log.error(`Error processing file change queue: ${error}`); + } finally { + this.isProcessing = false; + + // If more events were queued while processing, schedule another run + if (this.syncQueue.length > 0) { + this.processSyncQueue(); + } + } + } + + /** + * Process a single file change event + */ + private async processFileChange(event: FileChangeEvent) { + try { + const mapping = becca.fileSystemMappings[event.mappingId]; + if (!mapping || !mapping.isActive || !mapping.canSyncFromDisk) { + return; + } + + log.info(`DEBUG: Processing file ${event.type}: ${event.filePath} for mapping ${event.mappingId}`); + + switch (event.type) { + case 'add': + case 'change': + await this.handleFileAddOrChange(event, mapping); + break; + case 'unlink': + await this.handleFileDelete(event, mapping); + break; + } + + } catch (error) { + log.error(`Error processing file change for ${event.filePath}: ${error}`); + const mapping = becca.fileSystemMappings[event.mappingId]; + if (mapping) { + mapping.addSyncError(`Error processing ${event.filePath}: ${(error as Error).message}`); + } + } + } + + /** + * Handle file addition or modification + */ + private async handleFileAddOrChange(event: FileChangeEvent, mapping: BFileSystemMapping) { + if (!await fs.pathExists(event.filePath)) { + return; // File was deleted between queuing and processing + } + + const stats = event.stats || await fs.stat(event.filePath); + if (stats.isDirectory()) { + return; // We only sync files, not directories + } + + // Calculate file hash for change detection + const fileContent = await fs.readFile(event.filePath); + const fileHash = crypto.createHash('sha256').update(fileContent).digest('hex'); + const fileModifiedTime = stats.mtime.toISOString(); + + // Find existing file note mapping + let fileNoteMapping: BFileNoteMapping | null = null; + for (const mapping of Object.values(becca.fileNoteMappings || {})) { + if (mapping.mappingId === event.mappingId && mapping.filePath === event.filePath) { + fileNoteMapping = mapping; + break; + } + } + + // Check if file actually changed + if (fileNoteMapping && !fileNoteMapping.hasFileChanged(fileHash, fileModifiedTime)) { + return; // No actual change + } + + if (fileNoteMapping) { + // Update existing mapping + if (fileNoteMapping.hasNoteChanged()) { + // Both file and note changed - mark as conflict + fileNoteMapping.markConflict(); + log.info(`Conflict detected for ${event.filePath} - both file and note modified`); + return; + } + + fileNoteMapping.markPending(); + } else { + // Create new file note mapping + fileNoteMapping = new BFileNoteMapping({ + mappingId: event.mappingId, + noteId: '', // Will be determined by sync service + filePath: event.filePath, + fileHash, + fileModifiedTime, + syncStatus: 'pending' + }).save(); + } + + // Emit event for sync service to handle + eventService.emit('FILE_CHANGED', { + fileNoteMapping, + mapping, + fileContent, + isNew: event.type === 'add' + }); + } + + /** + * Handle file deletion + */ + private async handleFileDelete(event: FileChangeEvent, mapping: BFileSystemMapping) { + // Find existing file note mapping + let fileNoteMapping: BFileNoteMapping | null = null; + for (const mappingObj of Object.values(becca.fileNoteMappings || {})) { + if (mappingObj.mappingId === event.mappingId && mappingObj.filePath === event.filePath) { + fileNoteMapping = mappingObj; + break; + } + } + + if (fileNoteMapping) { + // Emit event for sync service to handle deletion + eventService.emit('FILE_DELETED', { + fileNoteMapping, + mapping + }); + } + } + + /** + * Get status of all watchers + */ + getWatcherStatus() { + const status: Record = {}; + + for (const [mappingId, { mapping, watcher }] of this.watchers) { + status[mappingId] = { + filePath: mapping.filePath, + isActive: mapping.isActive, + watchedPaths: watcher.getWatched(), + syncDirection: mapping.syncDirection + }; + } + + return status; + } + + /** + * Force a full sync for a specific mapping + */ + async forceSyncMapping(mappingId: string) { + const mapping = becca.fileSystemMappings[mappingId]; + if (!mapping) { + throw new Error(`Mapping ${mappingId} not found`); + } + + log.info(`Force syncing mapping ${mappingId}: ${mapping.filePath}`); + + if (await fs.pathExists(mapping.filePath)) { + const stats = await fs.stat(mapping.filePath); + if (stats.isFile()) { + await this.queueFileChange('change', mapping.filePath, mappingId, stats); + } else if (stats.isDirectory() && mapping.includeSubtree) { + // Scan directory for files + await this.scanDirectoryForFiles(mapping.filePath, mapping); + } + } + } + + /** + * Recursively scan directory for files and queue them for sync + */ + private async scanDirectoryForFiles(dirPath: string, mapping: BFileSystemMapping) { + try { + const entries = await fs.readdir(dirPath, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dirPath, entry.name); + + if (entry.isFile()) { + const stats = await fs.stat(fullPath); + this.queueFileChange('change', fullPath, mapping.mappingId, stats); + } else if (entry.isDirectory() && mapping.includeSubtree) { + await this.scanDirectoryForFiles(fullPath, mapping); + } + } + } catch (error) { + log.error(`Error scanning directory ${dirPath}: ${error}`); + mapping.addSyncError(`Error scanning directory: ${(error as Error).message}`); + } + } +} + +// Create singleton instance +const fileSystemWatcher = new FileSystemWatcher(); + +export default fileSystemWatcher; diff --git a/apps/server/src/services/options_init.ts b/apps/server/src/services/options_init.ts index 73aebce20..7268f0551 100644 --- a/apps/server/src/services/options_init.ts +++ b/apps/server/src/services/options_init.ts @@ -211,6 +211,9 @@ const defaultOptions: DefaultOption[] = [ { name: "aiTemperature", value: "0.7", isSynced: true }, { name: "aiSystemPrompt", value: "", isSynced: true }, { name: "aiSelectedProvider", value: "openai", isSynced: true }, + + // File system sync options + { name: "fileSystemSyncEnabled", value: "false", isSynced: false }, ]; /** diff --git a/packages/commons/src/lib/options_interface.ts b/packages/commons/src/lib/options_interface.ts index 1cc6b419f..9e0043cce 100644 --- a/packages/commons/src/lib/options_interface.ts +++ b/packages/commons/src/lib/options_interface.ts @@ -123,6 +123,7 @@ export interface OptionDefinitions extends KeyboardShortcutsOptions