From 0d898385f6910df6dfdcf2b307f0ceac07489871 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Fri, 8 Aug 2025 22:15:58 -0700 Subject: [PATCH] feat(llm): try to stop some of the horrible memory management --- .../src/widgets/llm_chat/communication.ts | 50 ++- .../src/widgets/llm_chat/tool_execution_ui.ts | 309 ++++++++++++++++ .../src/services/llm/ai_service_manager.ts | 190 ++++++++-- .../src/services/llm/base_ai_service.ts | 47 ++- .../llm/providers/anthropic_service.ts | 161 ++------- .../services/llm/tools/tool_format_adapter.ts | 341 ++++++++++++++++++ 6 files changed, 939 insertions(+), 159 deletions(-) create mode 100644 apps/client/src/widgets/llm_chat/tool_execution_ui.ts create mode 100644 apps/server/src/services/llm/tools/tool_format_adapter.ts diff --git a/apps/client/src/widgets/llm_chat/communication.ts b/apps/client/src/widgets/llm_chat/communication.ts index 6281791af..7d1dde6c9 100644 --- a/apps/client/src/widgets/llm_chat/communication.ts +++ b/apps/client/src/widgets/llm_chat/communication.ts @@ -72,9 +72,14 @@ export async function setupStreamingResponse( let timeoutId: number | null = null; let initialTimeoutId: number | null = null; let cleanupTimeoutId: number | null = null; + let heartbeatTimeoutId: number | null = null; let receivedAnyMessage = false; let eventListener: ((event: Event) => void) | null = null; let lastMessageTimestamp = 0; + + // Configuration for timeouts + const HEARTBEAT_TIMEOUT_MS = 30000; // 30 seconds between messages + const MAX_IDLE_TIME_MS = 60000; // 60 seconds max idle time // Create a unique identifier for this response process const responseId = `llm-stream-${Date.now()}-${Math.floor(Math.random() * 1000)}`; @@ -107,12 +112,43 @@ export async function setupStreamingResponse( } })(); + // Function to reset heartbeat timeout + const resetHeartbeatTimeout = () => { + if (heartbeatTimeoutId) { + window.clearTimeout(heartbeatTimeoutId); + } + + heartbeatTimeoutId = window.setTimeout(() => { + const idleTime = Date.now() - lastMessageTimestamp; + console.warn(`[${responseId}] No message received for ${idleTime}ms`); + + if (idleTime > MAX_IDLE_TIME_MS) { + console.error(`[${responseId}] Connection appears to be stalled (idle for ${idleTime}ms)`); + performCleanup(); + reject(new Error('Connection lost: The AI service stopped responding. Please try again.')); + } else { + // Send a warning but continue waiting + console.warn(`[${responseId}] Connection may be slow, continuing to wait...`); + resetHeartbeatTimeout(); // Reset for another check + } + }, HEARTBEAT_TIMEOUT_MS); + }; + // Function to safely perform cleanup const performCleanup = () => { + // Clear all timeouts if (cleanupTimeoutId) { window.clearTimeout(cleanupTimeoutId); cleanupTimeoutId = null; } + if (heartbeatTimeoutId) { + window.clearTimeout(heartbeatTimeoutId); + heartbeatTimeoutId = null; + } + if (initialTimeoutId) { + window.clearTimeout(initialTimeoutId); + initialTimeoutId = null; + } console.log(`[${responseId}] Performing final cleanup of event listener`); cleanupEventListener(eventListener); @@ -121,13 +157,15 @@ export async function setupStreamingResponse( }; // Set initial timeout to catch cases where no message is received at all + // Increased timeout and better error messaging + const INITIAL_TIMEOUT_MS = 15000; // 15 seconds for initial response initialTimeoutId = window.setTimeout(() => { if (!receivedAnyMessage) { - console.error(`[${responseId}] No initial message received within timeout`); + console.error(`[${responseId}] No initial message received within ${INITIAL_TIMEOUT_MS}ms timeout`); performCleanup(); - reject(new Error('No response received from server')); + reject(new Error('Connection timeout: The AI service is taking longer than expected to respond. Please check your connection and try again.')); } - }, 10000); + }, INITIAL_TIMEOUT_MS); // Create a message handler for CustomEvents eventListener = (event: Event) => { @@ -161,6 +199,12 @@ export async function setupStreamingResponse( window.clearTimeout(initialTimeoutId); initialTimeoutId = null; } + + // Start heartbeat monitoring + resetHeartbeatTimeout(); + } else { + // Reset heartbeat on each new message + resetHeartbeatTimeout(); } // Handle error diff --git a/apps/client/src/widgets/llm_chat/tool_execution_ui.ts b/apps/client/src/widgets/llm_chat/tool_execution_ui.ts new file mode 100644 index 000000000..1fbc28868 --- /dev/null +++ b/apps/client/src/widgets/llm_chat/tool_execution_ui.ts @@ -0,0 +1,309 @@ +/** + * Tool Execution UI Components + * + * This module provides enhanced UI components for displaying tool execution status, + * progress, and user-friendly error messages during LLM tool calls. + */ + +import { t } from "../../services/i18n.js"; + +/** + * Tool execution status types + */ +export type ToolExecutionStatus = 'pending' | 'running' | 'success' | 'error' | 'cancelled'; + +/** + * Tool execution display data + */ +export interface ToolExecutionDisplay { + toolName: string; + displayName: string; + status: ToolExecutionStatus; + description?: string; + progress?: { + current: number; + total: number; + message?: string; + }; + result?: string; + error?: string; + startTime?: number; + endTime?: number; +} + +/** + * Map of tool names to user-friendly display names + */ +const TOOL_DISPLAY_NAMES: Record = { + 'search_notes': 'Searching Notes', + 'get_note_content': 'Reading Note', + 'create_note': 'Creating Note', + 'update_note': 'Updating Note', + 'execute_code': 'Running Code', + 'web_search': 'Searching Web', + 'get_note_attributes': 'Reading Note Properties', + 'set_note_attribute': 'Setting Note Property', + 'navigate_notes': 'Navigating Notes', + 'query_decomposition': 'Analyzing Query', + 'contextual_thinking': 'Processing Context' +}; + +/** + * Map of tool names to descriptions + */ +const TOOL_DESCRIPTIONS: Record = { + 'search_notes': 'Finding relevant notes in your knowledge base', + 'get_note_content': 'Reading the content of a specific note', + 'create_note': 'Creating a new note with the provided content', + 'update_note': 'Updating an existing note', + 'execute_code': 'Running code in a safe environment', + 'web_search': 'Searching the web for current information', + 'get_note_attributes': 'Reading note metadata and properties', + 'set_note_attribute': 'Updating note metadata', + 'navigate_notes': 'Exploring the note hierarchy', + 'query_decomposition': 'Breaking down complex queries', + 'contextual_thinking': 'Analyzing context for better understanding' +}; + +/** + * Create a tool execution indicator element + */ +export function createToolExecutionIndicator(toolName: string): HTMLElement { + const container = document.createElement('div'); + container.className = 'tool-execution-indicator mb-2 p-2 border rounded bg-light'; + container.dataset.toolName = toolName; + + const displayName = TOOL_DISPLAY_NAMES[toolName] || toolName; + const description = TOOL_DESCRIPTIONS[toolName] || ''; + + container.innerHTML = ` +
+
+ +
+
+
${displayName}
+ ${description ? `
${description}
` : ''} + + + +
+ +
+ `; + + return container; +} + +/** + * Update tool execution status + */ +export function updateToolExecutionStatus( + container: HTMLElement, + status: ToolExecutionStatus, + data?: { + progress?: { current: number; total: number; message?: string }; + result?: string; + error?: string; + duration?: number; + } +): void { + const statusIcon = container.querySelector('.tool-status-icon'); + const progressDiv = container.querySelector('.tool-progress') as HTMLElement; + const progressBar = container.querySelector('.progress-bar') as HTMLElement; + const progressMessage = container.querySelector('.progress-message') as HTMLElement; + const resultDiv = container.querySelector('.tool-result') as HTMLElement; + const errorDiv = container.querySelector('.tool-error') as HTMLElement; + const durationDiv = container.querySelector('.tool-duration') as HTMLElement; + + if (!statusIcon) return; + + // Update status icon + switch (status) { + case 'pending': + statusIcon.innerHTML = ` + + `; + break; + + case 'running': + statusIcon.innerHTML = ` + + `; + break; + + case 'success': + statusIcon.innerHTML = ''; + container.classList.add('border-success', 'bg-success-subtle'); + break; + + case 'error': + statusIcon.innerHTML = ''; + container.classList.add('border-danger', 'bg-danger-subtle'); + break; + + case 'cancelled': + statusIcon.innerHTML = ''; + container.classList.add('border-warning', 'bg-warning-subtle'); + break; + } + + // Update progress if provided + if (data?.progress && progressDiv && progressBar && progressMessage) { + progressDiv.style.display = 'block'; + const percentage = (data.progress.current / data.progress.total) * 100; + progressBar.style.width = `${percentage}%`; + if (data.progress.message) { + progressMessage.textContent = data.progress.message; + } + } + + // Update result if provided + if (data?.result && resultDiv) { + resultDiv.style.display = 'block'; + resultDiv.textContent = data.result; + } + + // Update error if provided + if (data?.error && errorDiv) { + errorDiv.style.display = 'block'; + errorDiv.textContent = formatErrorMessage(data.error); + } + + // Update duration if provided + if (data?.duration && durationDiv) { + durationDiv.style.display = 'block'; + durationDiv.textContent = formatDuration(data.duration); + } +} + +/** + * Format error messages to be user-friendly + */ +function formatErrorMessage(error: string): string { + // Remove technical details and provide user-friendly messages + const errorMappings: Record = { + 'ECONNREFUSED': 'Connection refused. Please check if the service is running.', + 'ETIMEDOUT': 'Request timed out. Please try again.', + 'ENOTFOUND': 'Service not found. Please check your configuration.', + '401': 'Authentication failed. Please check your API credentials.', + '403': 'Access denied. Please check your permissions.', + '404': 'Resource not found.', + '429': 'Rate limit exceeded. Please wait a moment and try again.', + '500': 'Server error. Please try again later.', + '503': 'Service temporarily unavailable. Please try again later.' + }; + + for (const [key, message] of Object.entries(errorMappings)) { + if (error.includes(key)) { + return message; + } + } + + // Generic error formatting + if (error.length > 100) { + return error.substring(0, 100) + '...'; + } + + return error; +} + +/** + * Format duration in a human-readable way + */ +function formatDuration(milliseconds: number): string { + if (milliseconds < 1000) { + return `${milliseconds}ms`; + } else if (milliseconds < 60000) { + return `${(milliseconds / 1000).toFixed(1)}s`; + } else { + const minutes = Math.floor(milliseconds / 60000); + const seconds = Math.floor((milliseconds % 60000) / 1000); + return `${minutes}m ${seconds}s`; + } +} + +/** + * Create a tool execution summary + */ +export function createToolExecutionSummary(executions: ToolExecutionDisplay[]): HTMLElement { + const container = document.createElement('div'); + container.className = 'tool-execution-summary mt-2 p-2 border rounded bg-light small'; + + const successful = executions.filter(e => e.status === 'success').length; + const failed = executions.filter(e => e.status === 'error').length; + const total = executions.length; + + const totalDuration = executions.reduce((sum, e) => { + if (e.startTime && e.endTime) { + return sum + (e.endTime - e.startTime); + } + return sum; + }, 0); + + container.innerHTML = ` +
+
+ + Tools Executed: + ${successful} successful + ${failed > 0 ? `${failed} failed` : ''} + ${total} total +
+ ${totalDuration > 0 ? ` +
+ + ${formatDuration(totalDuration)} +
+ ` : ''} +
+ `; + + return container; +} + +/** + * Create a loading indicator with custom message + */ +export function createLoadingIndicator(message: string = 'Processing...'): HTMLElement { + const container = document.createElement('div'); + container.className = 'loading-indicator-enhanced d-flex align-items-center p-2'; + + container.innerHTML = ` + + ${message} + `; + + return container; +} + +/** + * Update loading indicator message + */ +export function updateLoadingMessage(container: HTMLElement, message: string): void { + const messageElement = container.querySelector('.loading-message'); + if (messageElement) { + messageElement.textContent = message; + } +} + +export default { + createToolExecutionIndicator, + updateToolExecutionStatus, + createToolExecutionSummary, + createLoadingIndicator, + updateLoadingMessage +}; \ No newline at end of file diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index bd47b4327..f761e2e6d 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -39,10 +39,26 @@ interface NoteContext { score?: number; } -export class AIServiceManager implements IAIServiceManager { - private currentService: AIService | null = null; - private currentProvider: ServiceProviders | null = null; +// Service cache entry with TTL +interface ServiceCacheEntry { + service: AIService; + provider: ServiceProviders; + createdAt: number; + lastUsed: number; +} + +// Disposable interface for proper resource cleanup +export interface Disposable { + dispose(): void | Promise; +} + +export class AIServiceManager implements IAIServiceManager, Disposable { + private serviceCache: Map = new Map(); + private readonly SERVICE_TTL_MS = 5 * 60 * 1000; // 5 minutes TTL + private readonly CLEANUP_INTERVAL_MS = 60 * 1000; // Cleanup check every minute + private cleanupTimer: NodeJS.Timeout | null = null; private initialized = false; + private disposed = false; constructor() { // Initialize tools immediately @@ -50,7 +66,8 @@ export class AIServiceManager implements IAIServiceManager { log.error(`Error initializing LLM tools during AIServiceManager construction: ${error.message || String(error)}`); }); - // Removed complex provider change listener - we'll read options fresh each time + // Start periodic cleanup of stale services + this.startCleanupTimer(); this.initialized = true; } @@ -372,34 +389,103 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Clear the current provider (forces recreation on next access) + * Start the cleanup timer for removing stale services */ - public clearCurrentProvider(): void { - this.currentService = null; - this.currentProvider = null; - log.info('Cleared current provider - will be recreated on next access'); + private startCleanupTimer(): void { + if (this.cleanupTimer) return; + + this.cleanupTimer = setInterval(() => { + this.cleanupStaleServices(); + }, this.CLEANUP_INTERVAL_MS); } /** - * Get or create the current provider instance - only one instance total + * Stop the cleanup timer + */ + private stopCleanupTimer(): void { + if (this.cleanupTimer) { + clearInterval(this.cleanupTimer); + this.cleanupTimer = null; + } + } + + /** + * Cleanup stale services that haven't been used recently + */ + private cleanupStaleServices(): void { + if (this.disposed) return; + + const now = Date.now(); + const staleProviders: ServiceProviders[] = []; + + for (const [provider, entry] of this.serviceCache.entries()) { + if (now - entry.lastUsed > this.SERVICE_TTL_MS) { + staleProviders.push(provider); + } + } + + for (const provider of staleProviders) { + this.disposeService(provider); + } + + if (staleProviders.length > 0) { + log.info(`Cleaned up ${staleProviders.length} stale service(s): ${staleProviders.join(', ')}`); + } + } + + /** + * Dispose a specific service + */ + private disposeService(provider: ServiceProviders): void { + const entry = this.serviceCache.get(provider); + if (entry) { + // If the service implements disposable, call dispose + if ('dispose' in entry.service && typeof (entry.service as any).dispose === 'function') { + try { + (entry.service as any).dispose(); + } catch (error) { + log.error(`Error disposing ${provider} service: ${error}`); + } + } + this.serviceCache.delete(provider); + log.info(`Disposed ${provider} service`); + } + } + + /** + * Clear all cached providers (forces recreation on next access) + */ + public clearCurrentProvider(): void { + // Clear all cached services + for (const provider of this.serviceCache.keys()) { + this.disposeService(provider); + } + log.info('Cleared all cached providers - will be recreated on next access'); + } + + /** + * Get or create a provider instance with proper caching and TTL */ private async getOrCreateChatProvider(providerName: ServiceProviders): Promise { - // If provider type changed, clear the old one - if (this.currentProvider && this.currentProvider !== providerName) { - log.info(`Provider changed from ${this.currentProvider} to ${providerName}, clearing old service`); - this.currentService = null; - this.currentProvider = null; + if (this.disposed) { + throw new Error('AIServiceManager has been disposed'); } - // Return existing service if it matches and is available - if (this.currentService && this.currentProvider === providerName && this.currentService.isAvailable()) { - return this.currentService; - } - - // Clear invalid service - if (this.currentService) { - this.currentService = null; - this.currentProvider = null; + // Check cache first + const cached = this.serviceCache.get(providerName); + if (cached && cached.service.isAvailable()) { + // Update last used time + cached.lastUsed = Date.now(); + + // Check if service is still within TTL + if (Date.now() - cached.createdAt <= this.SERVICE_TTL_MS) { + log.info(`Using cached ${providerName} service (age: ${Math.round((Date.now() - cached.createdAt) / 1000)}s)`); + return cached.service; + } else { + // Service is stale, dispose and recreate + log.info(`Cached ${providerName} service is stale, recreating`); + this.disposeService(providerName); + } } // Create new service for the requested provider @@ -443,9 +529,14 @@ export class AIServiceManager implements IAIServiceManager { } if (service) { - // Cache the new service - this.currentService = service; - this.currentProvider = providerName; + // Cache the new service with metadata + const now = Date.now(); + this.serviceCache.set(providerName, { + service, + provider: providerName, + createdAt: now, + lastUsed: now + }); log.info(`Created and cached new ${providerName} service`); return service; } @@ -456,6 +547,26 @@ export class AIServiceManager implements IAIServiceManager { return null; } + /** + * Dispose of all resources and cleanup + */ + async dispose(): Promise { + if (this.disposed) return; + + log.info('Disposing AIServiceManager...'); + this.disposed = true; + + // Stop cleanup timer + this.stopCleanupTimer(); + + // Dispose all cached services + for (const provider of this.serviceCache.keys()) { + this.disposeService(provider); + } + + log.info('AIServiceManager disposed successfully'); + } + /** * Initialize the AI Service using the new configuration system */ @@ -706,21 +817,40 @@ export class AIServiceManager implements IAIServiceManager { } -// Don't create singleton immediately, use a lazy-loading pattern +// Singleton instance (lazy-loaded) - can be disposed and recreated let instance: AIServiceManager | null = null; /** - * Get the AIServiceManager instance (creates it if not already created) + * Get the AIServiceManager instance (creates it if not already created or disposed) */ function getInstance(): AIServiceManager { - if (!instance) { + if (!instance || (instance as any).disposed) { instance = new AIServiceManager(); } return instance; } +/** + * Create a new AIServiceManager instance (for testing or isolated contexts) + */ +function createNewInstance(): AIServiceManager { + return new AIServiceManager(); +} + +/** + * Dispose the current singleton instance + */ +async function disposeInstance(): Promise { + if (instance) { + await instance.dispose(); + instance = null; + } +} + export default { getInstance, + createNewInstance, + disposeInstance, // Also export methods directly for convenience isAnyServiceAvailable(): boolean { return getInstance().isAnyServiceAvailable(); diff --git a/apps/server/src/services/llm/base_ai_service.ts b/apps/server/src/services/llm/base_ai_service.ts index 3c6e05bc7..16a27474a 100644 --- a/apps/server/src/services/llm/base_ai_service.ts +++ b/apps/server/src/services/llm/base_ai_service.ts @@ -1,9 +1,18 @@ import options from '../options.js'; import type { AIService, ChatCompletionOptions, ChatResponse, Message } from './ai_interface.js'; import { DEFAULT_SYSTEM_PROMPT } from './constants/llm_prompt_constants.js'; +import log from '../log.js'; -export abstract class BaseAIService implements AIService { +/** + * Disposable interface for proper resource cleanup + */ +export interface Disposable { + dispose(): void | Promise; +} + +export abstract class BaseAIService implements AIService, Disposable { protected name: string; + protected disposed: boolean = false; constructor(name: string) { this.name = name; @@ -12,6 +21,9 @@ export abstract class BaseAIService implements AIService { abstract generateChatCompletion(messages: Message[], options?: ChatCompletionOptions): Promise; isAvailable(): boolean { + if (this.disposed) { + return false; + } return options.getOptionBool('aiEnabled'); // Base check if AI is enabled globally } @@ -23,4 +35,37 @@ export abstract class BaseAIService implements AIService { // Use prompt from constants file if no custom prompt is provided return customPrompt || DEFAULT_SYSTEM_PROMPT; } + + /** + * Dispose of any resources held by this service + * Override in subclasses to clean up specific resources + */ + async dispose(): Promise { + if (this.disposed) { + return; + } + + log.info(`Disposing ${this.name} service`); + this.disposed = true; + + // Subclasses should override this to clean up their specific resources + await this.disposeResources(); + } + + /** + * Template method for subclasses to implement resource cleanup + */ + protected async disposeResources(): Promise { + // Default implementation does nothing + // Subclasses should override to clean up their resources + } + + /** + * Check if the service has been disposed + */ + protected checkDisposed(): void { + if (this.disposed) { + throw new Error(`${this.name} service has been disposed and cannot be used`); + } + } } diff --git a/apps/server/src/services/llm/providers/anthropic_service.ts b/apps/server/src/services/llm/providers/anthropic_service.ts index 2bb957766..f374e8358 100644 --- a/apps/server/src/services/llm/providers/anthropic_service.ts +++ b/apps/server/src/services/llm/providers/anthropic_service.ts @@ -7,7 +7,8 @@ import { getAnthropicOptions } from './providers.js'; import log from '../../log.js'; import Anthropic from '@anthropic-ai/sdk'; import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; -import type { ToolCall } from '../tools/tool_interfaces.js'; +import type { ToolCall, Tool } from '../tools/tool_interfaces.js'; +import { ToolFormatAdapter } from '../tools/tool_format_adapter.js'; interface AnthropicMessage extends Omit { content: MessageContent[] | string; @@ -34,6 +35,17 @@ export class AnthropicService extends BaseAIService { return super.isAvailable() && !!options.getOption('anthropicApiKey'); } + /** + * Clean up resources when disposing + */ + protected async disposeResources(): Promise { + if (this.client) { + // Clear the client reference + this.client = null; + log.info('Anthropic client disposed'); + } + } + private getClient(apiKey: string, baseUrl: string, apiVersion?: string, betaVersion?: string): any { if (!this.client) { this.client = new Anthropic({ @@ -49,6 +61,9 @@ export class AnthropicService extends BaseAIService { } async generateChatCompletion(messages: Message[], opts: ChatCompletionOptions = {}): Promise { + // Check if service has been disposed + this.checkDisposed(); + if (!this.isAvailable()) { throw new Error('Anthropic service is not available. Check API key and AI settings.'); } @@ -104,15 +119,18 @@ export class AnthropicService extends BaseAIService { if (opts.tools && opts.tools.length > 0) { log.info(`========== ANTHROPIC TOOL PROCESSING ==========`); log.info(`Input tools count: ${opts.tools.length}`); - log.info(`Input tool names: ${opts.tools.map(t => t.function?.name || 'unnamed').join(', ')}`); + log.info(`Input tool names: ${opts.tools.map((t: any) => t.function?.name || 'unnamed').join(', ')}`); - // Convert OpenAI-style function tools to Anthropic format - const anthropicTools = this.convertToolsToAnthropicFormat(opts.tools); + // Use the new ToolFormatAdapter for consistent conversion + const anthropicTools = ToolFormatAdapter.convertToProviderFormat( + opts.tools as Tool[], + 'anthropic' + ); if (anthropicTools.length > 0) { requestParams.tools = anthropicTools; log.info(`Successfully added ${anthropicTools.length} tools to Anthropic request`); - log.info(`Final tool names: ${anthropicTools.map(t => t.name).join(', ')}`); + log.info(`Final tool names: ${anthropicTools.map((t: any) => t.name).join(', ')}`); } else { log.error(`CRITICAL: Tool conversion failed - 0 tools converted from ${opts.tools.length} input tools`); } @@ -164,22 +182,11 @@ export class AnthropicService extends BaseAIService { if (toolBlocks.length > 0) { log.info(`[DEBUG] Found ${toolBlocks.length} tool-related blocks in response`); - toolCalls = toolBlocks.map((block: any) => { - if (block.type === 'tool_use') { - log.info(`[DEBUG] Processing tool_use block: ${JSON.stringify(block, null, 2)}`); - - // Convert Anthropic tool_use format to standard format expected by our app - return { - id: block.id, - type: 'function', // Convert back to function type for internal use - function: { - name: block.name, - arguments: JSON.stringify(block.input || {}) - } - }; - } - return null; - }).filter(Boolean); + // Use ToolFormatAdapter to convert from Anthropic format + toolCalls = ToolFormatAdapter.convertToolCallsFromProvider( + toolBlocks, + 'anthropic' + ); log.info(`Extracted ${toolCalls?.length} tool calls from Anthropic response`); } @@ -324,21 +331,12 @@ export class AnthropicService extends BaseAIService { block => block.type === 'tool_use' ); - // Convert tool use blocks to our expected format + // Use ToolFormatAdapter to convert tool calls if (toolUseBlocks.length > 0) { - toolCalls = toolUseBlocks.map(block => { - if (block.type === 'tool_use') { - return { - id: block.id, - type: 'function', - function: { - name: block.name, - arguments: JSON.stringify(block.input || {}) - } - }; - } - return null; - }).filter(Boolean); + toolCalls = ToolFormatAdapter.convertToolCallsFromProvider( + toolUseBlocks, + 'anthropic' + ); // For any active tool calls, mark them as complete for (const [toolId, toolCall] of activeToolCalls.entries()) { @@ -525,96 +523,9 @@ export class AnthropicService extends BaseAIService { return anthropicMessages; } - /** - * Convert OpenAI-style function tools to Anthropic format - * OpenAI uses: { type: "function", function: { name, description, parameters } } - * Anthropic uses: { name, description, input_schema } - */ - private convertToolsToAnthropicFormat(tools: any[]): any[] { - if (!tools || tools.length === 0) { - return []; - } - - log.info(`[TOOL DEBUG] Converting ${tools.length} tools to Anthropic format`); - - // Filter out invalid tools - const validTools = tools.filter(tool => { - if (!tool || typeof tool !== 'object') { - log.error(`Invalid tool format (not an object)`); - return false; - } - - // For function tools, validate required fields - if (tool.type === 'function') { - if (!tool.function || !tool.function.name) { - log.error(`Function tool missing required fields`); - return false; - } - } - - return true; - }); - - if (validTools.length < tools.length) { - log.info(`Filtered out ${tools.length - validTools.length} invalid tools`); - } - - // Convert tools to Anthropic format - const convertedTools = validTools.map((tool: any) => { - // Convert from OpenAI format to Anthropic format - if (tool.type === 'function' && tool.function) { - log.info(`[TOOL DEBUG] Converting function tool: ${tool.function.name}`); - - // Check the parameters structure - if (tool.function.parameters) { - log.info(`[TOOL DEBUG] Parameters for ${tool.function.name}:`); - log.info(`[TOOL DEBUG] - Type: ${tool.function.parameters.type}`); - log.info(`[TOOL DEBUG] - Properties: ${JSON.stringify(tool.function.parameters.properties || {})}`); - log.info(`[TOOL DEBUG] - Required: ${JSON.stringify(tool.function.parameters.required || [])}`); - - // Check if the required array is present and properly populated - if (!tool.function.parameters.required || !Array.isArray(tool.function.parameters.required)) { - log.error(`[TOOL DEBUG] WARNING: Tool ${tool.function.name} missing required array in parameters`); - } else if (tool.function.parameters.required.length === 0) { - log.error(`[TOOL DEBUG] WARNING: Tool ${tool.function.name} has empty required array - Anthropic may send empty inputs`); - } - } else { - log.error(`[TOOL DEBUG] WARNING: Tool ${tool.function.name} has no parameters defined`); - } - - return { - name: tool.function.name, - description: tool.function.description || '', - input_schema: tool.function.parameters || {} - }; - } - - // Handle already converted Anthropic format (from our temporary fix) - if (tool.type === 'custom' && tool.custom) { - log.info(`[TOOL DEBUG] Converting custom tool: ${tool.custom.name}`); - return { - name: tool.custom.name, - description: tool.custom.description || '', - input_schema: tool.custom.parameters || {} - }; - } - - // If the tool is already in the correct Anthropic format - if (tool.name && (tool.input_schema || tool.parameters)) { - log.info(`[TOOL DEBUG] Tool already in Anthropic format: ${tool.name}`); - return { - name: tool.name, - description: tool.description || '', - input_schema: tool.input_schema || tool.parameters - }; - } - - log.error(`Unhandled tool format encountered`); - return null; - }).filter(Boolean); // Filter out any null values - - return convertedTools; - } + // Tool conversion is now handled by ToolFormatAdapter + // The old convertToolsToAnthropicFormat method has been removed in favor of the centralized adapter + // This ensures consistent tool format conversion across all providers /** * Clear cached Anthropic client to force recreation with new settings diff --git a/apps/server/src/services/llm/tools/tool_format_adapter.ts b/apps/server/src/services/llm/tools/tool_format_adapter.ts new file mode 100644 index 000000000..e2fc0df59 --- /dev/null +++ b/apps/server/src/services/llm/tools/tool_format_adapter.ts @@ -0,0 +1,341 @@ +/** + * Tool Format Adapter + * + * This module provides standardized conversion between different LLM provider tool formats. + * It ensures consistent tool handling across OpenAI, Anthropic, Ollama, and other providers. + */ + +import log from '../../log.js'; +import type { Tool, ToolCall, ToolParameter } from './tool_interfaces.js'; + +/** + * Anthropic tool format + */ +export interface AnthropicTool { + name: string; + description: string; + input_schema: { + type: 'object'; + properties: Record; + required?: string[]; + }; +} + +/** + * OpenAI tool format (already matches our standard Tool interface) + */ +export type OpenAITool = Tool; + +/** + * Ollama tool format + */ +export interface OllamaTool { + type: 'function'; + function: { + name: string; + description: string; + parameters: Record; + }; +} + +/** + * Provider types + */ +export type ProviderType = 'openai' | 'anthropic' | 'ollama' | 'unknown'; + +/** + * Tool format adapter for converting between different provider formats + */ +export class ToolFormatAdapter { + /** + * Convert tools from standard format to provider-specific format + */ + static convertToProviderFormat(tools: Tool[], provider: ProviderType): unknown[] { + switch (provider) { + case 'anthropic': + return this.convertToAnthropicFormat(tools); + case 'ollama': + return this.convertToOllamaFormat(tools); + case 'openai': + // OpenAI format matches our standard format + return tools; + default: + log.warn(`Unknown provider ${provider}, returning tools in standard format`); + return tools; + } + } + + /** + * Convert tools to Anthropic format + */ + static convertToAnthropicFormat(tools: Tool[]): AnthropicTool[] { + const converted: AnthropicTool[] = []; + + for (const tool of tools) { + if (!this.validateTool(tool)) { + log.error(`Invalid tool skipped: ${JSON.stringify(tool)}`); + continue; + } + + try { + const anthropicTool: AnthropicTool = { + name: tool.function.name, + description: tool.function.description || '', + input_schema: { + type: 'object', + properties: tool.function.parameters.properties || {}, + required: tool.function.parameters.required || [] + } + }; + + // Validate the converted tool + if (this.validateAnthropicTool(anthropicTool)) { + converted.push(anthropicTool); + log.info(`Successfully converted tool ${tool.function.name} to Anthropic format`); + } else { + log.error(`Failed to validate converted Anthropic tool: ${tool.function.name}`); + } + } catch (error) { + log.error(`Error converting tool ${tool.function.name} to Anthropic format: ${error}`); + } + } + + return converted; + } + + /** + * Convert tools to Ollama format + */ + static convertToOllamaFormat(tools: Tool[]): OllamaTool[] { + const converted: OllamaTool[] = []; + + for (const tool of tools) { + if (!this.validateTool(tool)) { + log.error(`Invalid tool skipped: ${JSON.stringify(tool)}`); + continue; + } + + try { + const ollamaTool: OllamaTool = { + type: 'function', + function: { + name: tool.function.name, + description: tool.function.description || '', + parameters: tool.function.parameters || {} + } + }; + + converted.push(ollamaTool); + log.info(`Successfully converted tool ${tool.function.name} to Ollama format`); + } catch (error) { + log.error(`Error converting tool ${tool.function.name} to Ollama format: ${error}`); + } + } + + return converted; + } + + /** + * Convert tool calls from provider format to standard format + */ + static convertToolCallsFromProvider(toolCalls: unknown[], provider: ProviderType): ToolCall[] { + switch (provider) { + case 'anthropic': + return this.convertAnthropicToolCalls(toolCalls); + case 'ollama': + return this.convertOllamaToolCalls(toolCalls); + case 'openai': + // OpenAI format matches our standard format + return toolCalls as ToolCall[]; + default: + log.warn(`Unknown provider ${provider}, attempting standard conversion`); + return toolCalls as ToolCall[]; + } + } + + /** + * Convert Anthropic tool calls to standard format + */ + private static convertAnthropicToolCalls(toolCalls: unknown[]): ToolCall[] { + const converted: ToolCall[] = []; + + for (const call of toolCalls) { + if (typeof call === 'object' && call !== null) { + const anthropicCall = call as any; + + // Handle tool_use blocks from Anthropic + if (anthropicCall.type === 'tool_use') { + converted.push({ + id: anthropicCall.id, + type: 'function', + function: { + name: anthropicCall.name, + arguments: typeof anthropicCall.input === 'string' + ? anthropicCall.input + : JSON.stringify(anthropicCall.input || {}) + } + }); + } + // Handle already converted format + else if (anthropicCall.function) { + converted.push(anthropicCall as ToolCall); + } + } + } + + return converted; + } + + /** + * Convert Ollama tool calls to standard format + */ + private static convertOllamaToolCalls(toolCalls: unknown[]): ToolCall[] { + // Ollama typically uses a format similar to OpenAI + return toolCalls as ToolCall[]; + } + + /** + * Validate a standard tool definition + */ + static validateTool(tool: unknown): tool is Tool { + if (!tool || typeof tool !== 'object') { + return false; + } + + const t = tool as any; + + // Check required fields + if (t.type !== 'function') { + log.error(`Tool validation failed: type must be 'function', got '${t.type}'`); + return false; + } + + if (!t.function || typeof t.function !== 'object') { + log.error('Tool validation failed: missing or invalid function object'); + return false; + } + + if (!t.function.name || typeof t.function.name !== 'string') { + log.error('Tool validation failed: missing or invalid function name'); + return false; + } + + if (!t.function.parameters || typeof t.function.parameters !== 'object') { + log.error(`Tool validation failed for ${t.function.name}: missing or invalid parameters`); + return false; + } + + if (t.function.parameters.type !== 'object') { + log.error(`Tool validation failed for ${t.function.name}: parameters.type must be 'object'`); + return false; + } + + // Validate required array if present + if (t.function.parameters.required && !Array.isArray(t.function.parameters.required)) { + log.error(`Tool validation failed for ${t.function.name}: parameters.required must be an array`); + return false; + } + + return true; + } + + /** + * Validate an Anthropic tool definition + */ + private static validateAnthropicTool(tool: AnthropicTool): boolean { + if (!tool.name || typeof tool.name !== 'string') { + log.error('Anthropic tool validation failed: missing or invalid name'); + return false; + } + + if (!tool.input_schema || typeof tool.input_schema !== 'object') { + log.error(`Anthropic tool validation failed for ${tool.name}: missing or invalid input_schema`); + return false; + } + + if (tool.input_schema.type !== 'object') { + log.error(`Anthropic tool validation failed for ${tool.name}: input_schema.type must be 'object'`); + return false; + } + + if (!tool.input_schema.properties || typeof tool.input_schema.properties !== 'object') { + log.error(`Anthropic tool validation failed for ${tool.name}: missing or invalid properties`); + return false; + } + + // Warn if required array is missing or empty (Anthropic may send empty inputs) + if (!tool.input_schema.required || tool.input_schema.required.length === 0) { + log.warn(`Anthropic tool ${tool.name} has no required parameters - may receive empty inputs`); + } + + return true; + } + + /** + * Create a standardized error response for tool execution failures + */ + static createToolErrorResponse(toolName: string, error: unknown): string { + const errorMessage = error instanceof Error ? error.message : String(error); + return JSON.stringify({ + error: true, + tool: toolName, + message: `Tool execution failed: ${errorMessage}`, + timestamp: new Date().toISOString() + }); + } + + /** + * Create a standardized success response for tool execution + */ + static createToolSuccessResponse(toolName: string, result: unknown): string { + if (typeof result === 'string') { + return result; + } + return JSON.stringify({ + success: true, + tool: toolName, + result: result, + timestamp: new Date().toISOString() + }); + } + + /** + * Parse tool arguments safely + */ + static parseToolArguments(args: string | Record): Record { + if (typeof args === 'string') { + try { + return JSON.parse(args); + } catch (error) { + log.error(`Failed to parse tool arguments as JSON: ${error}`); + return {}; + } + } + return args || {}; + } + + /** + * Detect provider type from tool format + */ + static detectProviderFromToolFormat(tool: unknown): ProviderType { + if (!tool || typeof tool !== 'object') { + return 'unknown'; + } + + const t = tool as any; + + // Check for Anthropic format + if (t.name && t.input_schema) { + return 'anthropic'; + } + + // Check for OpenAI/standard format + if (t.type === 'function' && t.function) { + return 'openai'; + } + + return 'unknown'; + } +} + +export default ToolFormatAdapter; \ No newline at end of file