mirror of
https://github.com/zadam/trilium.git
synced 2025-12-05 15:04:24 +01:00
debug(llm): add some llm debug tools
This commit is contained in:
perf3ct
parent
755b1ed42f
commit
27847ab720
@ -4,6 +4,8 @@ import options from "../../services/options.js";
|
||||
|
||||
import restChatService from "../../services/llm/rest_chat_service.js";
|
||||
import chatStorageService from '../../services/llm/chat_storage_service.js';
|
||||
import toolRegistry from '../../services/llm/tools/tool_registry.js';
|
||||
import aiServiceManager from '../../services/llm/ai_service_manager.js';
|
||||
|
||||
// Define basic interfaces
|
||||
interface ChatMessage {
|
||||
@ -646,6 +648,96 @@ async function handleStreamingProcess(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug endpoint to check tool recognition and registry status
|
||||
*/
|
||||
async function debugTools(req: Request, res: Response): Promise<void> {
|
||||
try {
|
||||
log.info("========== DEBUG TOOLS ENDPOINT CALLED ==========");
|
||||
|
||||
// Get detailed tool registry info
|
||||
const registryDebugInfo = toolRegistry.getDebugInfo();
|
||||
|
||||
// Get AI service manager status
|
||||
const availableProviders = aiServiceManager.getAvailableProviders();
|
||||
const providerStatus: Record<string, any> = {};
|
||||
|
||||
for (const provider of availableProviders) {
|
||||
try {
|
||||
const service = await aiServiceManager.getService(provider);
|
||||
providerStatus[provider] = {
|
||||
available: true,
|
||||
type: service.constructor.name,
|
||||
supportsTools: 'generateChatCompletion' in service
|
||||
};
|
||||
} catch (error) {
|
||||
providerStatus[provider] = {
|
||||
available: false,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Get current tool definitions being sent to LLM
|
||||
const currentToolDefinitions = toolRegistry.getAllToolDefinitions();
|
||||
|
||||
// Format tool definitions for debugging
|
||||
const toolDefinitionSummary = currentToolDefinitions.map(def => ({
|
||||
name: def.function.name,
|
||||
description: def.function.description || 'No description',
|
||||
parameterCount: Object.keys(def.function.parameters?.properties || {}).length,
|
||||
requiredParams: def.function.parameters?.required || [],
|
||||
type: def.type || 'function'
|
||||
}));
|
||||
|
||||
const debugData = {
|
||||
timestamp: new Date().toISOString(),
|
||||
summary: {
|
||||
registrySize: registryDebugInfo.registrySize,
|
||||
validToolCount: registryDebugInfo.validToolCount,
|
||||
definitionsForLLM: currentToolDefinitions.length,
|
||||
availableProviders: availableProviders.length,
|
||||
initializationAttempted: registryDebugInfo.initializationAttempted
|
||||
},
|
||||
toolRegistry: {
|
||||
...registryDebugInfo,
|
||||
toolDefinitionSummary
|
||||
},
|
||||
aiServiceManager: {
|
||||
availableProviders,
|
||||
providerStatus
|
||||
},
|
||||
fullToolDefinitions: currentToolDefinitions,
|
||||
troubleshooting: {
|
||||
commonIssues: [
|
||||
"No tools in registry - check tool initialization in AIServiceManager",
|
||||
"Tools failing validation - check execute methods and definitions",
|
||||
"Provider not supporting function calling - verify model capabilities",
|
||||
"Tool definitions not being sent to LLM - check enableTools option"
|
||||
],
|
||||
checkpoints: [
|
||||
`Tools registered: ${registryDebugInfo.registrySize > 0 ? '✓' : '✗'}`,
|
||||
`Tools valid: ${registryDebugInfo.validToolCount > 0 ? '✓' : '✗'}`,
|
||||
`Definitions available: ${currentToolDefinitions.length > 0 ? '✓' : '✗'}`,
|
||||
`Providers available: ${availableProviders.length > 0 ? '✓' : '✗'}`
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
log.info(`Debug tools response: ${JSON.stringify(debugData.summary, null, 2)}`);
|
||||
|
||||
res.status(200).json(debugData);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
log.error(`Error in debug tools endpoint: ${errorMessage}`);
|
||||
res.status(500).json({
|
||||
error: 'Failed to retrieve debug information',
|
||||
message: errorMessage,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
// Chat session management
|
||||
createSession,
|
||||
@ -654,5 +746,8 @@ export default {
|
||||
listSessions,
|
||||
deleteSession,
|
||||
sendMessage,
|
||||
streamMessage
|
||||
streamMessage,
|
||||
|
||||
// Debug endpoints
|
||||
debugTools
|
||||
};
|
||||
|
||||
@ -11,7 +11,7 @@ import auth from "../services/auth.js";
|
||||
import { doubleCsrfProtection as csrfMiddleware } from "./csrf_protection.js";
|
||||
import { safeExtractMessageAndStackFromError } from "../services/utils.js";
|
||||
|
||||
const MAX_ALLOWED_FILE_SIZE_MB = 250;
|
||||
const MAX_ALLOWED_FILE_SIZE_MB = 2500;
|
||||
export const router = express.Router();
|
||||
|
||||
// TODO: Deduplicate with etapi_utils.ts afterwards.
|
||||
|
||||
@ -378,6 +378,9 @@ function register(app: express.Application) {
|
||||
asyncApiRoute(PST, "/api/llm/chat/:chatNoteId/messages", llmRoute.sendMessage);
|
||||
asyncApiRoute(PST, "/api/llm/chat/:chatNoteId/messages/stream", llmRoute.streamMessage);
|
||||
|
||||
// Debug endpoints
|
||||
asyncApiRoute(GET, "/api/llm/debug/tools", llmRoute.debugTools);
|
||||
|
||||
|
||||
|
||||
// LLM provider endpoints - moved under /api/llm/providers hierarchy
|
||||
|
||||
@ -89,8 +89,20 @@ export class LLMCompletionStage extends BasePipelineStage<LLMCompletionInput, {
|
||||
if (toolDefinitions.length > 0) {
|
||||
updatedOptions.enableTools = true;
|
||||
updatedOptions.tools = toolDefinitions;
|
||||
log.info(`Adding ${toolDefinitions.length} tools to LLM request`);
|
||||
log.info(`========== ADDING TOOLS TO LLM REQUEST ==========`);
|
||||
log.info(`Tool count: ${toolDefinitions.length}`);
|
||||
log.info(`Tool names: ${toolDefinitions.map(t => t.function.name).join(', ')}`);
|
||||
log.info(`enableTools option: ${updatedOptions.enableTools}`);
|
||||
log.info(`===============================================`);
|
||||
} else {
|
||||
log.error(`========== NO TOOLS AVAILABLE FOR LLM ==========`);
|
||||
log.error(`Tool registry returned 0 definitions`);
|
||||
log.error(`This means the LLM will NOT have access to tools`);
|
||||
log.error(`Check tool initialization and registration`);
|
||||
log.error(`==============================================`);
|
||||
}
|
||||
} else {
|
||||
log.info(`Tools explicitly disabled (enableTools: ${updatedOptions.enableTools})`);
|
||||
}
|
||||
|
||||
// Determine which provider to use
|
||||
|
||||
@ -102,12 +102,21 @@ export class AnthropicService extends BaseAIService {
|
||||
|
||||
// Add tools support if provided
|
||||
if (opts.tools && opts.tools.length > 0) {
|
||||
log.info(`Adding ${opts.tools.length} tools to Anthropic request`);
|
||||
log.info(`========== ANTHROPIC TOOL PROCESSING ==========`);
|
||||
log.info(`Input tools count: ${opts.tools.length}`);
|
||||
log.info(`Input tool names: ${opts.tools.map(t => t.function?.name || 'unnamed').join(', ')}`);
|
||||
|
||||
// Convert OpenAI-style function tools to Anthropic format
|
||||
const anthropicTools = this.convertToolsToAnthropicFormat(opts.tools);
|
||||
|
||||
if (anthropicTools.length > 0) {
|
||||
requestParams.tools = anthropicTools;
|
||||
log.info(`Successfully added ${anthropicTools.length} tools to Anthropic request`);
|
||||
log.info(`Final tool names: ${anthropicTools.map(t => t.name).join(', ')}`);
|
||||
} else {
|
||||
log.error(`CRITICAL: Tool conversion failed - 0 tools converted from ${opts.tools.length} input tools`);
|
||||
}
|
||||
log.info(`============================================`);
|
||||
|
||||
// Add tool_choice parameter if specified
|
||||
if (opts.tool_choice) {
|
||||
|
||||
@ -153,8 +153,87 @@ export class ToolRegistry {
|
||||
// Only get definitions from valid tools
|
||||
const validTools = this.getAllTools();
|
||||
const toolDefs = validTools.map(handler => handler.definition);
|
||||
|
||||
// Enhanced debugging for tool recognition issues
|
||||
log.info(`========== TOOL REGISTRY DEBUG INFO ==========`);
|
||||
log.info(`Total tools in registry: ${this.tools.size}`);
|
||||
log.info(`Valid tools after validation: ${validTools.length}`);
|
||||
log.info(`Tool definitions being sent to LLM: ${toolDefs.length}`);
|
||||
|
||||
// Log each tool for debugging
|
||||
toolDefs.forEach((def, idx) => {
|
||||
log.info(`Tool ${idx + 1}: ${def.function.name} - ${def.function.description?.substring(0, 100) || 'No description'}...`);
|
||||
log.info(` Parameters: ${Object.keys(def.function.parameters?.properties || {}).join(', ') || 'none'}`);
|
||||
log.info(` Required: ${def.function.parameters?.required?.join(', ') || 'none'}`);
|
||||
});
|
||||
|
||||
if (toolDefs.length === 0) {
|
||||
log.error(`CRITICAL: No tool definitions available for LLM! This will prevent tool calling.`);
|
||||
log.error(`Registry size: ${this.tools.size}, Initialization attempted: ${this.initializationAttempted}`);
|
||||
|
||||
// Try to provide debugging info about what's in the registry
|
||||
log.error(`Raw tools in registry:`);
|
||||
this.tools.forEach((handler, name) => {
|
||||
log.error(` - ${name}: ${handler ? 'exists' : 'null'}, definition: ${handler?.definition ? 'exists' : 'missing'}`);
|
||||
});
|
||||
}
|
||||
|
||||
log.info(`==============================================`);
|
||||
|
||||
return toolDefs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug method to get detailed registry status
|
||||
*/
|
||||
public getDebugInfo(): {
|
||||
registrySize: number;
|
||||
validToolCount: number;
|
||||
initializationAttempted: boolean;
|
||||
toolDetails: Array<{
|
||||
name: string;
|
||||
hasDefinition: boolean;
|
||||
hasExecute: boolean;
|
||||
isValid: boolean;
|
||||
error?: string;
|
||||
}>;
|
||||
} {
|
||||
const toolDetails: Array<{
|
||||
name: string;
|
||||
hasDefinition: boolean;
|
||||
hasExecute: boolean;
|
||||
isValid: boolean;
|
||||
error?: string;
|
||||
}> = [];
|
||||
|
||||
this.tools.forEach((handler, name) => {
|
||||
let isValid = false;
|
||||
let error: string | undefined;
|
||||
|
||||
try {
|
||||
isValid = this.validateToolHandler(handler);
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
}
|
||||
|
||||
toolDetails.push({
|
||||
name,
|
||||
hasDefinition: !!handler?.definition,
|
||||
hasExecute: typeof handler?.execute === 'function',
|
||||
isValid,
|
||||
error
|
||||
});
|
||||
});
|
||||
|
||||
const validTools = this.getAllTools();
|
||||
|
||||
return {
|
||||
registrySize: this.tools.size,
|
||||
validToolCount: validTools.length,
|
||||
initializationAttempted: this.initializationAttempted,
|
||||
toolDetails
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user