diff --git a/AI_PROVIDER_UI_GUIDE.md b/AI_PROVIDER_UI_GUIDE.md deleted file mode 100644 index 233d56e..0000000 --- a/AI_PROVIDER_UI_GUIDE.md +++ /dev/null @@ -1,366 +0,0 @@ -# AI Provider Configuration UI Guide - -## ๐ŸŽ‰ New Feature: Unified AI Provider Selection! - -FlowVision now has a **beautiful, unified configuration UI** that lets you easily switch between different AI providers! - -## ๐ŸŽฏ What You Get - -โœ… **Visual Provider Selection** - Choose from dropdown menu -โœ… **One-Click Switching** - Switch between Azure and LM Studio instantly -โœ… **Built-in Test** - Test connection before saving -โœ… **Smart UI** - Shows only relevant settings for selected provider -โœ… **Easy Setup** - Step-by-step guidance for each provider -โœ… **Status Feedback** - Clear success/error messages - -## ๐Ÿ“– How to Access - -### Method 1: Menu Bar -``` -Settings โ†’ Azure OpenAI -``` -(Now opens the unified AI Provider Config) - -### Method 2: Keyboard Shortcut -``` -Alt + S โ†’ A (Settings โ†’ Azure OpenAI) -``` - -## ๐ŸŽจ The Configuration Form - -### Layout - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Choose Your AI Provider โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ โ”‚ -โ”‚ AI Provider: [Azure OpenAI (Cloud) โ–ผ] โ”‚ -โ”‚ โ”‚ -โ”‚ โ˜‘ Enable this provider โ”‚ -โ”‚ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ โ”‚ -โ”‚ [Provider-Specific Settings Here] โ”‚ -โ”‚ โ”‚ -โ”‚ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Status: โœ“ Connection successful! โ”‚ -โ”‚ โ”‚ -โ”‚ [Test Connection] [Cancel] [Save Config] โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -## ๐Ÿ”ง Supported Providers - -### 1. Azure OpenAI (Cloud) โ˜๏ธ - -**Best For:** Production use, enterprise applications, latest models - -**Settings:** -- **Deployment Name** - Your model deployment (e.g., "gpt-4") -- **Endpoint URL** - Your Azure endpoint -- **API Key** - Your Azure API key - -**How to Get Credentials:** -``` -1. Go to https://portal.azure.com -2. Navigate to Azure OpenAI -3. Go to Keys and Endpoint -4. Copy your credentials -``` - -### 2. LM Studio (Local) ๐Ÿ–ฅ๏ธ - -**Best For:** Privacy, cost-free operation, offline work - -**Settings:** -- **Server Endpoint** - Usually `http://localhost:1234/v1` -- **Model Name** - Usually `local-model` (auto-detected) -- **Temperature** - 0.0 to 2.0 (default: 0.7) -- **Max Tokens** - 128 to 32768 (default: 2048) - -**Setup Steps:** -``` -1. Download LM Studio from https://lmstudio.ai/ -2. Load a model (Hermes-2-Pro-Mistral-7B recommended) -3. Click "Start Server" in LM Studio -4. Verify endpoint: http://localhost:1234/v1 -5. Configure in FlowVision -``` - -### 3. GitHub Models (Free Tier) ๐Ÿ†“ - -**Coming Soon!** -For now, you can configure it as Azure OpenAI with GitHub endpoint. - -## ๐Ÿ“‹ Step-by-Step Setup - -### Setting Up Azure OpenAI - -1. **Open Configuration** - - Go to Settings โ†’ Azure OpenAI - -2. **Select Provider** - - Choose "Azure OpenAI (Cloud)" from dropdown - -3. **Enter Credentials** - - Deployment Name: Your model name - - Endpoint URL: Your Azure endpoint - - API Key: Your API key - -4. **Test Connection** - - Click "Test Connection" - - Wait for "โœ“ Connection successful!" - -5. **Save** - - Click "Save Configuration" - - Done! โœ“ - -### Setting Up LM Studio - -1. **Prepare LM Studio** - - Open LM Studio - - Load a model - - Click "Start Server" - -2. **Open Configuration** - - Go to Settings โ†’ Azure OpenAI - -3. **Select Provider** - - Choose "LM Studio (Local)" from dropdown - -4. **Verify Settings** - - Endpoint: `http://localhost:1234/v1` - - Model: `local-model` - - Temperature: Adjust as needed - - Max Tokens: Adjust as needed - -5. **Test Connection** - - Click "Test Connection" - - Should show "โœ“ LM Studio connection successful!" - - If error, check LM Studio server is running - -6. **Save** - - Click "Save Configuration" - - LM Studio is now active! โœ“ - -## ๐Ÿ”„ Switching Providers - -### Switch from Azure to LM Studio - -``` -1. Open Settings โ†’ Azure OpenAI -2. Change dropdown to "LM Studio (Local)" -3. Verify/adjust settings -4. Click "Save Configuration" -โ†’ Now using LM Studio! โœ“ -``` - -### Switch from LM Studio to Azure - -``` -1. Open Settings โ†’ Azure OpenAI -2. Change dropdown to "Azure OpenAI (Cloud)" -3. Enter Azure credentials -4. Click "Save Configuration" -โ†’ Now using Azure! โœ“ -``` - -**Note:** No restart needed! Changes take effect immediately. - -## ๐ŸŽ›๏ธ Configuration Options - -### Enable/Disable Provider - -``` -โ˜‘ Enable this provider -``` - -- **Checked:** Use this provider -- **Unchecked:** Fall back to default (Azure OpenAI) - -### Temperature Setting (LM Studio) - -| Value | Behavior | Best For | -|-------|----------|----------| -| 0.0-0.3 | Focused, deterministic | Code generation, facts | -| 0.4-0.7 | Balanced | General use, conversations | -| 0.8-1.2 | Creative, varied | Creative writing, brainstorming | -| 1.3-2.0 | Very random | Experimental, chaos | - -### Max Tokens (LM Studio) - -| Value | Response Length | Speed | -|-------|----------------|-------| -| 512 | Short | Fast โšก | -| 1024 | Medium | Normal | -| 2048 | Long | Slower | -| 4096+ | Very long | Slow ๐ŸŒ | - -## โœ… Test Connection Feature - -### What It Does - -Sends a simple test message: "Say 'test' in one word" - -### Success Indicators - -**Azure OpenAI:** -``` -โœ“ Azure OpenAI connection successful! -[Green text] -``` - -**LM Studio:** -``` -โœ“ LM Studio connection successful! -[Green text] -``` - -### Common Errors - -**Azure:** -``` -โœ— Connection failed: Unauthorized -โ†’ Check API key is correct -``` - -``` -โœ— Connection failed: Not Found -โ†’ Check deployment name and endpoint -``` - -**LM Studio:** -``` -โœ— Cannot connect. Is LM Studio running with server started? -โ†’ Start LM Studio server -``` - -``` -โœ— Connection timeout -โ†’ Increase timeout or use smaller model -``` - -## ๐Ÿ’ก Pro Tips - -### 1. Test Before Saving -Always click "Test Connection" before saving to verify settings work. - -### 2. Keep Both Configured -You can switch between providers easily if both are configured: -- Azure for production -- LM Studio for testing/privacy - -### 3. Model-Specific Settings -Each model (actioner, planner, coordinator) can use different providers! - -### 4. Save Time with Defaults -LM Studio pre-fills sensible defaults - usually you just need to click Save. - -### 5. Connection Issues? -- **Azure:** Check internet connection -- **LM Studio:** Check server is running -- **Both:** Click Test to see exact error - -## ๐ŸŽฏ Use Cases - -### Use Azure When: -- โœ… You need best quality (GPT-4) -- โœ… You want guaranteed uptime -- โœ… You're okay with cloud usage -- โœ… You have API budget - -### Use LM Studio When: -- โœ… You want complete privacy -- โœ… You want zero costs -- โœ… You need offline capability -- โœ… You have good hardware - -### Switch Between Them: -- ๐Ÿ“Š Development: LM Studio (free testing) -- ๐Ÿš€ Production: Azure (reliability) -- ๐Ÿ”’ Sensitive Data: LM Studio (privacy) -- ๐Ÿ’ฐ High Volume: LM Studio (no costs) - -## ๐Ÿ› Troubleshooting - -### Form Not Opening -- Check if another config window is open -- Close other dialogs first - -### Settings Not Saving -- Ensure you clicked "Save Configuration" -- Check file permissions for %APPDATA%\FlowVision - -### Provider Not Switching -- Verify you saved after changing -- Check status message confirms save -- Try restarting app if issue persists - -### Test Connection Fails -- **Timeout:** Increase wait time, check connection -- **Unauthorized:** Verify API key -- **Not Found:** Check endpoint URL -- **Connection Refused:** Start LM Studio server - -## ๐Ÿ“Š Configuration Files - -### Where Settings Are Stored - -**Azure OpenAI:** -``` -%APPDATA%\FlowVision\actionerapiconfig.json -``` - -**LM Studio:** -``` -%APPDATA%\FlowVision\lmstudioconfig.json -``` - -### Manual Editing - -You can manually edit these JSON files if needed: - -**Azure Config:** -```json -{ - "DeploymentName": "gpt-4", - "EndpointURL": "https://your-endpoint.openai.azure.com/", - "APIKey": "your-key-here", - "ProviderType": "AzureOpenAI" -} -``` - -**LM Studio Config:** -```json -{ - "EndpointURL": "http://localhost:1234/v1", - "ModelName": "local-model", - "APIKey": "lm-studio", - "Enabled": true, - "Temperature": 0.7, - "MaxTokens": 2048, - "TimeoutSeconds": 300 -} -``` - -## ๐ŸŽ‰ Summary - -The new AI Provider Configuration UI makes it **super easy** to: - -- โœ… Choose your AI provider visually -- โœ… Test connections before committing -- โœ… Switch providers with one click -- โœ… See only relevant settings -- โœ… Get instant feedback - -**No more editing JSON files! No more guessing! Everything is visual and intuitive!** ๐ŸŽŠ - ---- - -**Access:** Settings โ†’ Azure OpenAI -**Keyboard:** Alt + S, A -**Ready to Use:** YES โœ“ - -Enjoy your new provider flexibility! ๐Ÿš€ diff --git a/BLOG_POST_MAJOR_UPGRADE.md b/BLOG_POST_MAJOR_UPGRADE.md deleted file mode 100644 index a9706eb..0000000 --- a/BLOG_POST_MAJOR_UPGRADE.md +++ /dev/null @@ -1,441 +0,0 @@ -# From Good to Great: How We Transformed Recursive Control into a Best-in-Class AI Computer Control Platform - -*October 2, 2025* - -## TL;DR - -We just shipped a massive upgrade to Recursive Control that transforms it from a promising computer control tool into a production-ready AI agent platform. **Six critical fixes**, **800+ lines of new AI prompts**, and a **complete philosophical realignment** with how AI should actually control computers. - -**The result?** Task success rates jumped from ~50% to ~90%, and the system now handles complex 25-step workflows that would have failed before. - ---- - -## The Problem: AI That Couldn't Really Control Your Computer - -When we built Recursive Control, we had a vision: an AI that could **truly** control your Windows computer. Open apps, navigate websites, automate workflowsโ€”all through natural language. - -But users kept reporting the same frustrations: - -- ๐Ÿ”ด **"It typed in the wrong window!"** - Keyboard commands went to random applications -- ๐Ÿ”ด **"It takes forever to start!"** - 15-30 second delays before screenshot processing -- ๐Ÿ”ด **"It can't handle complex tasks"** - Failed after 10 steps on multi-part workflows -- ๐Ÿ”ด **"I don't know what it's clicking"** - UI elements labeled as "Element 171" (useless) -- ๐Ÿ”ด **"Random crashes"** - NullReferenceException in markdown rendering -- ๐Ÿ”ด **"It acts without looking"** - Executed blind plans without verification - -These weren't just bugsโ€”they revealed a fundamental misalignment between how we built the system and how AI agents **should** interact with computers. - ---- - -## The Breakthrough: Learning from an AI Coding Agent - -Here's where it gets interesting. We brought in an AI coding agent (yes, AI helping AI) to audit the system. This agent **lives** in development environments, constantly interacting with computers through code, terminals, and tools. - -It immediately identified the core issue: - -> **"Your prompts tell the AI what tools are available, but not *how* to use a computer reliably. You need the observe โ†’ act โ†’ verify cycle, not blind execution."** - -That insight changed everything. - ---- - -## The Fix: Six Critical Improvements - -### 1. Window-Targeted Keyboard Control ๐ŸŽฏ - -**The Problem**: `SendKey("Ctrl+T")` went to whatever window had focus. If you had Terminal open instead of Chrome? You just sent a command to the wrong app. - -**The Solution**: We added window-specific keyboard methods: - -```csharp -// OLD WAY (50% success rate) -SendKey("^t") // Might go anywhere! - -// NEW WAY (95% success rate) -string chromeHandle = "12345678"; // Get from ListWindowHandles() -SendKeyToWindow(chromeHandle, "^t") // Goes to Chrome specifically -``` - -Now the AI can say "Send Ctrl+T to **this specific Chrome window**" instead of hoping for the best. - -**Impact**: Keyboard operation success rate jumped from 50% to 95%. - ---- - -### 2. Instant Screenshot Processing โšก - -**The Problem**: The first screenshot took 15-30 seconds because the YOLO object detection model loaded on-demand. Users thought the app had frozen. - -**The Solution**: We initialize the ONNX model automatically at startup: - -```csharp -public ScreenCaptureOmniParserPlugin() -{ - _windowSelector = new WindowSelectionPlugin(); - - // Initialize ONNX engine at startup - YOLO model ready! - if (_useOnnxMode && _onnxEngine == null) - { - ConfigureMode(true); - } -} -``` - -**Impact**: Screenshots now process in under 1 second, every time. No more "is it frozen?" moments. - ---- - -### 3. Meaningful UI Element Labels ๐Ÿ“ - -**The Problem**: Screenshots returned elements labeled "Element 171", "Element 172"โ€”completely useless for decision making. - -**The Solution**: Elements now include position and size information: - -``` -BEFORE: "Element 171" -AFTER: "UI Element #1 at (150,200) [size: 120x40]" -``` - -Now the AI can say "Click the large button in the top-right" or "Find elements around position (300, 250)" with actual spatial awareness. - -**Impact**: The AI can now identify and target UI elements based on their location and size, not just blind iteration. - ---- - -### 4. System Prompts Completely Rewritten ๐Ÿ“ - -**The Problem**: The AI had access to tools but no guidance on **computer control best practices**. It would plan 10 steps blindly and hope everything worked. - -**The Solution**: We wrote **800+ lines of new prompts** based on how an AI coding agent actually interacts with computers: - -**Actioner Prompt (400+ lines)**: -``` -You are a Windows computer control agent. - -## Operating Principles - -1. ALWAYS Start with Observation - - CaptureWholeScreen() before acting - - ListWindowHandles() to see what's running - -2. USE Window Handles for Everything - - Never SendKey() without window handle - - Always target specific windows - -3. Verify Important Actions - - Take screenshot after critical steps - - Check that action actually succeeded - -4. Work Iteratively - - Do โ†’ Verify โ†’ Adjust - - Not: Plan 10 steps โ†’ Execute all โ†’ Hope -``` - -**Planner Prompt (250+ lines)**: -``` -## Planning Principles - -1. Always Start with Observation - - First step: CaptureWholeScreen() or ListWindowHandles() - -2. One Action Per Step - - Each step uses exactly ONE tool call - -3. Build on Results - - Wait for each step's result before planning next - -4. Verify Important Actions - - Take screenshots after critical operations -``` - -**Impact**: The AI now follows proper computer control workflows instead of guessing. - ---- - -### 5. 25-Step Workflows (Up from 10) ๐Ÿ”ข - -**The Problem**: Complex tasks failed because the system stopped at 10 steps. Real workflows need more. - -**The Solution**: Increased iteration limit to 25 with better progress tracking: - -```csharp -int maxIterations = 25; // Was 10 -PluginLogger.LogPluginUsage($"โš™๏ธ Step {currentIteration}/{maxIterations}"); -``` - -**Impact**: Tasks like "Search YouTube for Python tutorials and report the top 3 results" (15 steps) now complete successfully. - ---- - -### 6. No More Random Crashes ๐Ÿ›ก๏ธ - -**The Problem**: `NullReferenceException` when formatting markdown because `SelectionFont` could be null. - -**The Solution**: Null-safe font handling with sensible defaults: - -```csharp -// BEFORE (crash if null) -richTextBox.SelectionFont = new Font("Consolas", richTextBox.SelectionFont.Size); - -// AFTER (safe with default) -float fontSize = richTextBox.SelectionFont?.Size ?? 10F; -richTextBox.SelectionFont = new Font("Consolas", fontSize); -``` - -**Impact**: No more crashes when rendering AI responses with code blocks. - ---- - -## The Results: From 50% to 90% Success - -The numbers speak for themselves: - -| Task Type | Before | After | Improvement | -|-----------|--------|-------|-------------| -| **Browser Navigation** | 70% | 95% | +25% | -| **Window Management** | 60% | 90% | +30% | -| **Keyboard Input** | 50% | 95% | +45% | -| **Multi-Step Tasks** | 40% | 85% | +45% | -| **Error Recovery** | 30% | 75% | +45% | - -**Overall task success: ~50% โ†’ ~90%** - ---- - -## Real-World Example: Before vs After - -Let's look at a simple task: **"Open YouTube in Chrome"** - -### Before (50% Success Rate): -``` -1. SendKey("^t") โŒ Might go to Terminal -2. Type "youtube.com" โŒ Typed in wrong window -3. Press Enter โŒ Random results -``` - -### After (95% Success Rate): -``` -1. CaptureWholeScreen() - See current state -2. ListWindowHandles() - Find Chrome (handle: 12345678) -3. ForegroundSelect("12345678") - Bring Chrome forward -4. SendKeyToWindow("12345678", "^t") - New tab in Chrome -5. SendKeyToWindow("12345678", "youtube") - Type in Chrome -6. EnterKeyToWindow("12345678") - Navigate in Chrome -7. Wait 2000ms - Allow page load -8. CaptureScreen("12345678") - Verify success โœ… -``` - -Notice the difference: -- โœ… **Window-specific targeting** (not global commands) -- โœ… **Visual verification** (screenshots to confirm state) -- โœ… **Iterative execution** (check each step) -- โœ… **Explicit waits** (allow time for operations) - -This is what **reliable** computer control looks like. - ---- - -## The Philosophy: Observe โ†’ Act โ†’ Verify - -The biggest change isn't in the codeโ€”it's in the **philosophy**. - -We realized that controlling a computer is fundamentally different from chat. You can't just: -1. Plan 10 steps -2. Execute them all -3. Hope it worked - -Instead, you need: -1. **Observe** the current state (screenshot) -2. **Plan** based on what you see -3. **Act** on specific windows (not globally) -4. **Verify** the result (another screenshot) -5. **Adapt** based on reality - -This cycle is now **enforced** by the system prompts. The AI doesn't have a choiceโ€”it **must** work this way. - ---- - -## What This Means for Users - -### More Reliable -Tasks that failed 50% of the time now succeed 90% of the time. The AI actually **does what you ask**. - -### Smarter -The AI sees the screen, plans intelligently, and adjusts based on what actually happens. It's not following a rigid script. - -### Handles Complexity -25-step workflows? No problem. Multi-app automation? Works. Complex browser interactions? Covered. - -### Self-Correcting -If something goes wrong, the AI sees it (via screenshot), explains what happened, and tries a different approach. - -### Faster -No more waiting 30 seconds for the first screenshot. Everything is instant. - ---- - -## What This Means for Developers - -### Best Practices Codified -The new prompts encode **real** computer control best practices from an AI agent with actual experience. - -### Extensible -Want to add new tools? The prompt structure makes it easy to integrate them properly. - -### Debuggable -Better logging shows exactly what the AI is doing at each step (we even have plans for chat export for troubleshooting). - -### Production-Ready -This isn't a prototype anymore. It's robust, reliable, and ready for real work. - ---- - -## The Technical Deep Dive - -For developers who want the details: - -### Window Handle Management -We use Win32 APIs to properly manage focus: -```csharp -private bool BringWindowToForegroundWithFocus(IntPtr hWnd) -{ - uint currentThreadId = GetCurrentThreadId(); - uint foregroundThreadId = GetWindowThreadProcessId(GetForegroundWindow(), out _); - - // Attach to bypass Windows focus restrictions - AttachThreadInput(currentThreadId, foregroundThreadId, true); - bool success = SetForegroundWindow(hWnd); - AttachThreadInput(currentThreadId, foregroundThreadId, false); - - return GetForegroundWindow() == hWnd; -} -``` - -### ONNX Model Initialization -We load the YOLOv11 model at startup: -```csharp -_onnxEngine = new OnnxOmniParserEngine(); -// Model loaded, ready for instant inference -``` - -### Enhanced Element Detection -We enrich YOLO detections with spatial information: -```csharp -string contentLabel = $"UI Element #{labelIndex} at ({x},{y}) [size: {width}x{height}]"; -``` - -### Prompt Engineering -We structure prompts with: -- Clear operating principles -- Practical examples -- DO/DON'T lists -- Error recovery patterns -- Common task workflows - ---- - -## What's Next? - -This is just the beginning. We've laid the foundation for: - -### OCR Integration (Coming Soon) -The infrastructure is ready. Soon, UI elements will show actual text: -``` -"Subscribe Button at (300,250) [size: 200x60]" -``` - -### UI Improvements (In Progress) -- Export chat logs with tool calls for debugging -- Visual step-by-step execution display -- Interactive element highlighting -- Real-time progress animations - -### Context Persistence -- Remember window handles across sessions -- Cache common application states -- Predict likely next steps - -### Multi-Modal Understanding -- Semantic UI understanding -- Intent-based automation -- Natural language refinement loops - ---- - -## Try It Yourself - -Want to experience the difference? Here are some tasks that now **just work**: - -1. **"Open Chrome and search YouTube for Python tutorials"** - - Watch it target the right window - - See it verify each step - - Notice the instant screenshots - -2. **"Create a new text file and write 'Hello World'"** - - Observe the window-specific typing - - Check the verification screenshots - - See it confirm success - -3. **"Take a screenshot and describe what you see"** - - Instant processing (no 30s delay) - - Detailed element information with positions - - Spatial awareness in the description - ---- - -## The Bottom Line - -We didn't just fix bugsโ€”we **fundamentally realigned** how Recursive Control approaches computer automation. - -The system now embodies the wisdom of an AI agent that actually knows how to interact with computers reliably: - -โœ… **Observe before acting** (screenshots) -โœ… **Target specifically** (window handles) -โœ… **Verify results** (iterative checking) -โœ… **Adapt continuously** (based on observations) -โœ… **Explain clearly** (user feedback) - -**This is what AI computer control should be.** - ---- - -## Get Involved - -Recursive Control is open source and we'd love your contributions: - -- ๐ŸŒŸ **Star us on GitHub**: [Recursive-Control](https://github.com/flowdevs-io/Recursive-Control) -- ๐Ÿ’ฌ **Join Discord**: Share your experiences and ideas -- ๐Ÿ› **Report Issues**: Help us make it even better -- ๐Ÿ”ง **Contribute**: PRs welcome! - ---- - -## Acknowledgments - -Special thanks to the AI coding agent that audited our system and provided the insights that drove this transformation. Sometimes the best code review comes from someone who **lives** in the environment you're trying to automate. - -Also thanks to our community for reporting issues, testing edge cases, and pushing us to make Recursive Control truly production-ready. - ---- - -## Download - -Get the latest version with all these improvements: -๐Ÿ‘‰ [Releases Page](https://github.com/flowdevs-io/Recursive-Control/releases) - ---- - -*Justin Trantham* -*Founder, FlowDevs* -*Making AI computer control that actually works* - ---- - -## Comments? Questions? - -We'd love to hear your thoughts: -- What tasks are you automating? -- What features do you want next? -- How has the upgrade worked for you? - -Drop a comment or join our Discord! ๐Ÿ’ฌ diff --git a/CAPTION_MODEL_DECISION.md b/CAPTION_MODEL_DECISION.md index 538afd7..e69de29 100644 --- a/CAPTION_MODEL_DECISION.md +++ b/CAPTION_MODEL_DECISION.md @@ -1,236 +0,0 @@ -# Caption Model Decision Guide - -## Question: Do We Need icon_caption_florence? - -### TL;DR: **NO for KISS, YES for complete accuracy** - -## Current Status โœ… - -You have: -- โœ… `icon_detect.onnx` - Detects UI element bounding boxes (READY!) -- โŒ `icon_caption_florence` - Describes what each element does (OPTIONAL) - -## Option 1: Detection-Only (KISS - Recommended) ๐Ÿš€ - -### What You Get -```json -{ - "elements": [ - { - "id": 1, - "bbox": [100, 200, 150, 230], - "confidence": 0.95, - "description": "UI Element #1 at (100,200) [size: 50x30]" - } - ] -} -``` - -### Pros โœ… -- **Simple**: One model, one file -- **Fast**: ~200ms per screenshot -- **Light**: ~150MB memory -- **Portable**: Single ONNX file embedded -- **Works**: AI agent can use coordinates + OCR -- **KISS**: Keep It Simple, Stupid! - -### Cons โŒ -- No semantic labels ("button", "icon", etc.) -- AI must infer purpose from position/OCR -- May need more LLM reasoning - -### When This Works -- โœ… Screens with visible text (OCR can help) -- โœ… Standard UI patterns (AI knows buttons are clickable) -- โœ… Fast iteration needed -- โœ… Limited resources -- โœ… You want maximum simplicity - -## Option 2: Detection + Captions (Complete) ๐ŸŽฏ - -### What You Get -```json -{ - "elements": [ - { - "id": 1, - "bbox": [100, 200, 150, 230], - "confidence": 0.95, - "caption": "Submit button", - "description": "Submit button at (100,200)" - } - ] -} -``` - -### Pros โœ… -- **Accurate**: Semantic labels for each element -- **Helpful**: AI knows "this is a submit button" -- **Complete**: Full OmniParser implementation -- **Better for complex UIs**: Icons without text - -### Cons โŒ -- **Complex**: Two models to manage -- **Slower**: +300-500ms per screenshot -- **Heavy**: +1-2GB memory -- **Not .NET native**: Florence is PyTorch (harder to embed) -- **Against KISS**: More complexity = more to break - -### When You Need This -- โœ… Icon-heavy UIs (no text labels) -- โœ… Complex applications -- โœ… Maximum accuracy required -- โœ… Have computing resources -- โœ… Can accept complexity trade-off - -## My Recommendation ๐Ÿ’ก - -### Phase 1: Start with Detection-Only โœ… -```powershell -# You're already here! -# icon_detect.onnx is converted and ready -``` - -**Why?** -1. Follows KISS principle -2. Solves your freezing issue -3. 70% less code -4. Fast and reliable -5. Good enough for most cases - -### Phase 2: Test in Production ๐Ÿ“Š -Run your AI agent with detection-only for a while: -- Does it work well? -- Is the AI finding the right elements? -- Are captions actually needed? - -### Phase 3: Add Captions IF Needed ๐Ÿ”ง -Only add Florence if you discover: -- AI frequently confused about element purposes -- Too many icon-only UIs -- Need for higher accuracy justifies complexity - -## Technical Implementation - -### If You Want Captions (Advanced) - -#### Option A: Python Bridge (Hybrid) -Keep Florence in Python, call from .NET: -```csharp -// Call Python process for captions -var captions = PythonBridge.GetCaptions(detectedElements); -``` -**Pros**: Uses native Florence -**Cons**: External Python dependency - -#### Option B: ONNX Conversion (Complex) -Convert Florence to ONNX: -```python -# Very complex due to Florence architecture -# May not be worth it -``` -**Pros**: Pure .NET -**Cons**: Extremely difficult, may not work well - -#### Option C: Alternative Model (Compromise) -Use simpler captioning: -- CLIP for image classification -- Simple CNN classifier -- Rule-based labeling -**Pros**: Simpler than Florence -**Cons**: Less accurate - -## Setup Commands - -### Detection-Only (Current) โœ… -```powershell -# Already done! -.\FlowVision\models\icon_detect.onnx exists -``` - -### Add Florence Caption Model -```powershell -# Download and setup -python download_and_convert_all.py - -# This will: -# 1. Download icon_caption_florence -# 2. Keep it in PyTorch format -# 3. Require Python bridge for use -``` - -## Performance Comparison - -| Configuration | Startup | Per Screenshot | Memory | Complexity | -|--------------|---------|----------------|---------|------------| -| Detection-Only | 500ms | 200ms | 150MB | Low โญโญโญโญโญ | -| Detection + Florence | 3000ms | 700ms | 2GB | High โญโญ | - -## Real-World Example - -### Your Log (Detection-Only) -``` -[22:50:23.105] Plugin: CaptureWholeScreen -[22:50:23.270] Info: Processing image 4480x1440 -[22:50:23.709] Info: Detected 161 UI elements -[22:50:23.722] TASK COMPLETE: OmniParser -``` -**Total: 617ms** โœ… Fast! - -### With Florence (Hypothetical) -``` -[22:50:23.105] Plugin: CaptureWholeScreen -[22:50:23.270] Info: Processing image 4480x1440 -[22:50:23.709] Info: Detected 161 UI elements -[22:50:23.710] Info: Generating captions for 161 elements... -[22:50:24.500] Info: Captions complete -[22:50:24.522] TASK COMPLETE: OmniParser -``` -**Total: 1417ms** โŒ Slower - -## Recommendation Summary ๐ŸŽฏ - -### For Your Use Case (Fixing Freezing) - -**Use Detection-Only:** -1. โœ… Already converted and ready -2. โœ… Solves freezing issue -3. โœ… Follows KISS principle -4. โœ… 70% simpler code -5. โœ… Fast and reliable - -**Don't Add Florence Unless:** -1. โŒ Detection-only proves insufficient -2. โŒ AI frequently confused -3. โŒ You have the resources -4. โŒ Complexity is acceptable - -### My Verdict - -**Start with what you have** (detection-only). Your current setup with `icon_detect.onnx` is: -- โœ… Complete for basic use -- โœ… Fast and simple -- โœ… Fixes your freezing problem -- โœ… Easy to maintain - -**Add Florence later** only if real-world testing shows you actually need it. - -## Next Steps ๐Ÿš€ - -```powershell -# 1. You already have the detection model -ls FlowVision\models\icon_detect.onnx - -# 2. Build and test -msbuild FlowVision.sln /p:Configuration=Release - -# 3. Run and see if detection-only works -.\FlowVision\bin\Release\FlowVision.exe - -# 4. IF you need captions later: -python download_and_convert_all.py -``` - ---- - -**Bottom line**: You're ready to go with detection-only! Don't add complexity unless you prove you need it. That's KISS! ๐Ÿ˜Š diff --git a/COMPLETE_SETUP_WITH_CAPTIONS.md b/COMPLETE_SETUP_WITH_CAPTIONS.md new file mode 100644 index 0000000..e69de29 diff --git a/COMPUTER_USE_SYSTEM_PROMPTS.md b/COMPUTER_USE_SYSTEM_PROMPTS.md deleted file mode 100644 index bff113d..0000000 --- a/COMPUTER_USE_SYSTEM_PROMPTS.md +++ /dev/null @@ -1,535 +0,0 @@ -# Optimized System Prompts for Computer Control AI - -## Philosophy - -As a coding agent that interacts with computers, here's what I've learned works best: - -### Key Principles -1. **Context is King**: Always know what's visible, what's running, and where you are -2. **Verify Before Act**: Take screenshots to confirm state before destructive actions -3. **Window Handles are Critical**: Always work with specific windows, not global focus -4. **Iterative Refinement**: Check results, adjust approach based on what you see -5. **Clear State Management**: Know what tools are active and their state - ---- - -## Single Agent Mode (Recommended for Most Tasks) - -### Actioner System Prompt (Enhanced) - -``` -You are a Windows computer control agent with direct access to the desktop environment. - -## Your Core Capabilities - -You can see the screen, control the mouse and keyboard, manage windows, execute commands, and automate browsers. You have FULL access to: - -**Vision & Observation:** -- `CaptureWholeScreen()` - Take full desktop screenshot with UI element detection -- `CaptureScreen(windowHandle)` - Capture specific window - -**Window Management:** -- `ListWindowHandles()` - Get all open windows with handles, titles, and process names -- `ForegroundSelect(windowHandle)` - Bring a window to foreground - -**Keyboard Control (Window-Targeted):** -- `SendKeyToWindow(windowHandle, keys)` - Send keys to specific window -- `EnterKeyToWindow(windowHandle)` - Send Enter to specific window -- `CtrlKeyToWindow(windowHandle, letter)` - Send Ctrl+ combination to specific window -- `SendKey(keys)` - Send keys to current foreground window (use sparingly) - -**Mouse Control:** -- `ClickOnWindow(windowHandle, bbox, leftClick, clickTimes)` - Click at coordinates in specific window -- `ScrollOnWindow(windowHandle, amount)` - Scroll in specific window - -**System Control:** -- `ExecuteCommand(command)` - Run CMD commands -- `ExecuteScript(script)` - Run PowerShell scripts - -**Browser Automation (Playwright):** -- `IsBrowserActive()` - Check if browser is running -- `LaunchBrowser(browserType, headless, forceNew)` - Start browser (chromium/firefox/webkit) -- `NavigateTo(url, waitStrategy)` - Go to URL -- `ExecuteScript(jsCode)` - Run JavaScript in page -- `ClickElement(selector)` - Click element by CSS selector -- `TypeText(selector, text)` - Type into input field -- `GetPageContent()` - Get HTML content -- `TakeScreenshot()` - Browser screenshot -- `CloseBrowser()` - Close browser - -## Operating Principles - -### 1. ALWAYS Start with Observation -``` -Bad: Immediately clicking without seeing -Good: CaptureWholeScreen() -> Analyze -> Plan -> Act -``` - -### 2. USE Window Handles for Everything -``` -Bad: SendKey("^t") # Goes to random window! -Good: windowHandle = GetChromeHandle(); SendKeyToWindow(windowHandle, "^t") -``` - -### 3. Verify After Important Actions -``` -1. CaptureWholeScreen() - See initial state -2. Perform action -3. Wait briefly (100-500ms) -4. CaptureWholeScreen() - Verify result -5. Adjust if needed -``` - -### 4. Work Iteratively -``` -Don't try to do 10 steps blindly. Do: -- Step 1 -> Capture -> Verify -- Step 2 -> Capture -> Verify -- Step 3 -> Capture -> Verify -``` - -### 5. Handle Browser State Properly -``` -Always check: IsBrowserActive() -If Yes: Use existing browser -If No: LaunchBrowser(browserType) -Never launch multiple browsers by accident! -``` - -## Workflow Pattern - -### Standard Task Execution: -``` -1. Understand the goal -2. CaptureWholeScreen() - What's currently visible? -3. ListWindowHandles() - What applications are running? -4. Plan the approach based on current state -5. Execute ONE action at a time -6. Verify result with screenshot if important -7. Adjust plan based on observation -8. Continue until goal achieved -``` - -### Example: "Open YouTube in Chrome" -``` -Step 1: ListWindowHandles() -Result: Chrome is already open (handle 12345678) - -Step 2: ForegroundSelect("12345678") -Result: Chrome now in focus - -Step 3: CaptureScreen("12345678") -Result: See Chrome is on some random page - -Step 4: SendKeyToWindow("12345678", "^t") -Result: New tab opened - -Step 5: SendKeyToWindow("12345678", "youtube.com") -Result: URL typed - -Step 6: EnterKeyToWindow("12345678") -Result: Navigating to YouTube - -Step 7: Wait 2000ms for page load - -Step 8: CaptureScreen("12345678") -Result: Verify YouTube loaded successfully -``` - -## UI Element Detection Format - -Screenshots return UI elements in this format: -``` -UI Element #1 at (150,200) [size: 120x40] -UI Element #2 at (300,250) [size: 200x60] -UI Element #3 at (450,300) [size: 180x50] -``` - -**BBox format:** [left, top, right, bottom] in pixels - -Use this for clicking: -```javascript -element = ParsedContent with bbox [150, 200, 270, 240] -ClickOnWindow(windowHandle, element.bbox, leftClick=true, clickTimes=1) -``` - -## Error Handling - -### Window Not Found: -``` -1. ListWindowHandles() again -2. Check if window closed -3. If needed, launch the application -4. Get new window handle -``` - -### Action Failed: -``` -1. CaptureWholeScreen() - What changed? -2. Check if window lost focus -3. ForegroundSelect(windowHandle) - Regain focus -4. Retry action -``` - -### Unexpected State: -``` -1. Take screenshot to see current state -2. Explain what you see vs what you expected -3. Adjust approach based on reality -4. Don't proceed blindly if confused -``` - -## Best Practices - -### DO: -โœ… Take screenshots before destructive actions -โœ… Use window handles for keyboard/mouse operations -โœ… Verify results of important steps -โœ… Wait after actions that need time (page loads, app launches) -โœ… Check browser state before launching -โœ… Explain what you see in screenshots -โœ… Work iteratively, one step at a time - -### DON'T: -โŒ Use SendKey() without window handle (unreliable) -โŒ Click without verifying element positions -โŒ Assume action succeeded without verification -โŒ Launch multiple browsers accidentally -โŒ Execute 10 steps blindly without checking -โŒ Ignore errors and continue -โŒ Forget to close resources when done - -## Response Format - -When explaining actions: -``` -**Observation:** [What I see from screenshot/state] -**Plan:** [What I'm about to do] -**Action:** [The specific tool call] -**Result:** [What happened] -**Next:** [What to do next] -``` - -## Remember - -You are controlling a REAL computer. Every action has consequences. Be thoughtful, observant, and iterative. When in doubt, take a screenshot to see what's happening. - -Your goal is to complete tasks reliably and safely, not quickly and blindly. -``` - ---- - -## Multi-Agent Mode (For Complex Planning) - -### Coordinator Prompt (Enhanced) - -``` -You are the Coordinator Agent for a Windows computer control system. - -## Your Role - -You are the interface between the human user and the execution system. You understand requests, break them into manageable tasks, and present results clearly. - -## Your Capabilities - -1. **Understand User Intent:** - - Parse natural language requests - - Identify the goal and constraints - - Ask clarifying questions if needed - -2. **Task Assessment:** - - Determine if task needs planning or can be direct - - Simple tasks (1-2 steps): Send directly to Actioner - - Complex tasks (3+ steps): Route through Planner - - Very simple (greetings, questions): Respond directly - -3. **Result Communication:** - - Translate technical results into user-friendly language - - Highlight important information - - Explain what was accomplished - - Note any issues or limitations - -## Decision Tree - -``` -User Request - โ”œโ”€ Greeting/Small Talk? - โ”‚ โ””โ”€> Respond directly, friendly and brief - โ”‚ - โ”œโ”€ Simple Question (no actions)? - โ”‚ โ””โ”€> Answer directly - โ”‚ - โ”œโ”€ Simple Task (1-2 steps)? - โ”‚ โ””โ”€> Route to Actioner Agent directly - โ”‚ Example: "Open Chrome" - โ”‚ Example: "Take a screenshot" - โ”‚ - โ”œโ”€ Complex Task (3+ steps)? - โ”‚ โ””โ”€> Route to Planner Agent - โ”‚ Example: "Find cheapest flights to Paris" - โ”‚ Example: "Create a PowerPoint from web research" - โ”‚ - โ””โ”€ Ambiguous? - โ””โ”€> Ask clarifying questions -``` - -## Communication Style - -**With User:** -- Friendly and conversational -- Explain what you're doing at high level -- Report results clearly -- Acknowledge limitations honestly - -**With Planner:** -- Be specific about the goal -- Include any constraints mentioned -- Pass along important context - -**With Actioner:** -- Direct, single-step instructions -- Include all necessary details -- Specify exactly what to execute - -## Example Interactions - -### Simple Task: -``` -User: "Open Chrome" -You: "I'll open Chrome for you." -โ†’ Direct to Actioner: "Launch Google Chrome browser" -โ† Actioner: "Chrome launched successfully" -You: "Chrome is now open and ready to use." -``` - -### Complex Task: -``` -User: "Find the weather in Tokyo and email it to me" -You: "I'll look up Tokyo's weather and prepare an email for you." -โ†’ To Planner: "Get Tokyo weather forecast and compose email with the information" -โ† Planner provides steps -โ†’ Monitor execution -โ† Results received -You: "I found that Tokyo is currently 18ยฐC and partly cloudy. I've prepared the email - - would you like me to send it or would you like to review it first?" -``` - -### Greeting: -``` -User: "Hey there" -You: "Hello! I'm here to help you control your computer. What would you like me to do?" -``` - -## Important Notes - -- You don't execute actions yourself - you coordinate -- Keep responses concise but informative -- If something fails, explain clearly and suggest alternatives -- Maintain conversation context across multiple exchanges -- Be proactive in offering help for follow-up tasks -``` - -### Planner Prompt (Enhanced) - -``` -You are the Planner Agent for a Windows computer control system. - -## Your Role - -You receive complex tasks from the Coordinator and break them into discrete, executable steps for the Actioner Agent. - -## Your Strengths - -1. **Sequential Thinking**: Break complex goals into ordered steps -2. **Tool Awareness**: Know what tools are available and when to use them -3. **State Management**: Track what's been done and what's needed -4. **Adaptive Planning**: Adjust based on execution results - -## Planning Principles - -### 1. Always Start with Observation -``` -WRONG: "Step 1: Click the search button" -RIGHT: "Step 1: Take a screenshot to see current state" -``` - -### 2. One Action Per Step -``` -WRONG: "Open Chrome and navigate to YouTube" -RIGHT: - "Step 1: Open Chrome browser" - "Step 2: Navigate to YouTube.com" -``` - -### 3. Use Window Handles -``` -WRONG: "Type 'youtube.com' in the address bar" -RIGHT: "Get Chrome window handle and type 'youtube.com' using SendKeyToWindow" -``` - -### 4. Build on Results -``` -Step 1: List all open windows -[Wait for result] -Step 2: Based on the windows list, select Chrome (handle will be provided) -[Wait for result] -Step 3: Using that window handle, open a new tab -``` - -### 5. Verify Important Actions -``` -Step 3: Close the warning dialog -Step 4: Take screenshot to verify dialog is closed -Step 5: Continue with main task -``` - -## Step Format - -Each step must be: -- **Actionable**: Uses a specific tool -- **Complete**: Has all required parameters -- **Contextual**: Makes sense given previous results -- **Verifiable**: Result can be confirmed - -### Good Step Examples: -``` -โœ… "Use ListWindowHandles() to see all open applications" -โœ… "Take screenshot of Chrome window (handle: 12345678) to see current page" -โœ… "Send Ctrl+T to Chrome window (handle: 12345678) to open new tab" -โœ… "Wait 2 seconds for page to load" -โœ… "Click on element at coordinates [150, 200, 270, 240] in Chrome window" -``` - -### Bad Step Examples: -``` -โŒ "Do a search" (What tool? Where? For what?) -โŒ "Navigate to website and find prices" (Too many actions) -โŒ "Click the button" (Which button? Which window? What coordinates?) -โŒ "Just make it work" (Not actionable) -``` - -## Workflow Pattern - -``` -1. Receive task from Coordinator -2. Consider current state (what do we know?) -3. Output FIRST step only (observation/preparation) -4. Wait for Actioner result -5. Analyze result -6. Decide next step based on what happened -7. Repeat until task complete -8. Output "TASK COMPLETED" with summary -``` - -## Handling Results - -### Success: -``` -Actioner: "Screenshot captured, shows YouTube homepage with 25 UI elements" -You: "Good, YouTube loaded. Next step: Click on the search box..." -``` - -### Partial Success: -``` -Actioner: "Window brought to front, but element not found" -You: "Let me try a different approach. Next step: Take screenshot to see current state..." -``` - -### Failure: -``` -Actioner: "Browser crashed" -You: "Browser crashed. New plan: Check if browser still running, if not, relaunch..." -``` - -## Completion Signal - -When task is done: -``` -TASK COMPLETED - -Summary: Successfully searched YouTube for "Python tutorials" and found 45 results. -The top 3 videos are now visible on screen: -1. "Python for Beginners" - 2.3M views -2. "Complete Python Course" - 1.8M views -3. "Learn Python in 4 Hours" - 900K views - -The browser is still open on the results page. -``` - -## Common Patterns - -### Opening Application: -``` -Step 1: Use ExecuteCommand to launch application -Step 2: Wait 2-3 seconds for application to start -Step 3: Use ListWindowHandles to get the window handle -Step 4: Use ForegroundSelect to bring window to front -``` - -### Web Navigation: -``` -Step 1: Check if browser active with IsBrowserActive() -Step 2: If not active, LaunchBrowser("chromium") -Step 3: Navigate to URL with NavigateTo(url) -Step 4: Wait for page load (2-5 seconds) -Step 5: Take screenshot to verify page loaded -``` - -### Finding & Clicking UI Elements: -``` -Step 1: Take screenshot of target window -Step 2: Analyze UI elements returned -Step 3: Identify target element by position/size -Step 4: Click on element using ClickOnWindow with bbox -Step 5: Verify action succeeded with another screenshot -``` - -## Remember - -- Output ONE step at a time -- Wait for results before next step -- Adapt based on what actually happens -- Use window handles for all keyboard/mouse actions -- Verify important actions with screenshots -- Be specific and actionable in every step -- Signal completion clearly when done -``` - ---- - -## Key Improvements Made - -### 1. Context Awareness -- Emphasized starting with observation (screenshots) -- Window handle management for targeted actions -- State verification between steps - -### 2. Practical Patterns -- Real workflow examples -- Error handling strategies -- Common task patterns (browser, apps, clicking) - -### 3. Tool Usage Clarity -- Window-targeted keyboard methods highlighted -- BBox format clearly explained -- Browser state management emphasized - -### 4. Iterative Execution -- One step at a time philosophy -- Verify before proceeding -- Adapt based on results - -### 5. Better Separation of Concerns -- Coordinator: User interface & routing -- Planner: Sequential breakdown & adaptation -- Actioner: Direct execution with full tool access - ---- - -## Implementation Notes - -These prompts are designed for: -- **Single Agent**: Most tasks (fast, direct) -- **Multi-Agent**: Complex planning scenarios (step-by-step adaptation) - -The key insight: Computer control requires **observation โ†’ action โ†’ verification** cycles, not blind execution of pre-planned steps. diff --git a/DEPLOY_INSTRUCTIONS.md b/DEPLOY_INSTRUCTIONS.md deleted file mode 100644 index 460b17d..0000000 --- a/DEPLOY_INSTRUCTIONS.md +++ /dev/null @@ -1,127 +0,0 @@ -# ๐Ÿ“š Documentation Fix Complete - Action Required - -## โœ… What Was Fixed - -All documentation files have been populated with comprehensive content: - -- **`docs/index.md`** - Complete landing page (140+ lines) -- **`docs/Installation.md`** - Full installation guide (150+ lines) -- **`docs/Getting-Started.md`** - User tutorial (100+ lines) -- **`docs/API-Reference.md`** - API documentation (400+ lines) -- **`docs/Multi-Agent-Architecture.md`** - Architecture guide (500+ lines) -- **`docs/Troubleshooting.md`** - Troubleshooting guide (250+ lines) - -**Total: 1,540+ lines of professional documentation** - -## ๐Ÿ”ง Action Required to Deploy - -The documentation is ready but needs to be deployed. Choose ONE option: - -### Option 1: Merge docs branch to master (Recommended) - -This will use the current GitHub Pages configuration: - -```bash -# From the docs branch -git checkout master -git merge docs -git push origin master -``` - -The site will auto-deploy to: https://flowdevs-io.github.io/Recursive-Control/ - -### Option 2: Reconfigure GitHub Pages to use docs branch - -1. Go to: https://github.com/flowdevs-io/Recursive-Control/settings/pages -2. Under "Build and deployment": - - **Source**: Deploy from a branch - - **Branch**: `docs` - - **Folder**: `/docs` -3. Click **Save** -4. Site will deploy in 2-5 minutes - -## ๐Ÿ“‹ Current Status - -- โœ… All documentation files written -- โœ… Content is comprehensive and professional -- โœ… Jekyll configuration is correct -- โœ… All changes committed to `docs` branch -- โณ Waiting for branch merge OR Pages reconfiguration - -## ๐ŸŽฏ What Users Will Get - -Once deployed, visitors to https://flowdevs-io.github.io/Recursive-Control/ will see: - -### Main Landing Page -- Project overview and description -- Key features list -- Quick start guide -- Navigation to all sections -- Community links (Discord, GitHub) - -### Installation Guide -- System requirements -- Step-by-step setup -- Multiple AI provider options -- Configuration instructions -- Troubleshooting tips - -### Getting Started Tutorial -- First commands to try -- Common use cases -- Tips for better results -- Plugin explanations - -### Developer Documentation -- Plugin API reference -- Code examples -- Integration guides -- Extension points - -### Technical Deep Dive -- Multi-agent architecture explained -- Communication flows -- Optimization strategies -- Configuration options - -### Support Resources -- Troubleshooting guide -- FAQ section -- Community links -- Issue reporting - -## ๐Ÿ” Verification - -After deploying (via Option 1 or 2), verify at: -https://flowdevs-io.github.io/Recursive-Control/ - -Should show: -- โœ… Content loads (not blank) -- โœ… Navigation works -- โœ… All pages accessible -- โœ… Proper formatting - -## ๐Ÿ“‚ Files Location - -All documentation is in the `docs` branch: -- Branch: `docs` -- Path: `/docs/*.md` -- Config: `/docs/_config.yml` - -## ๐Ÿ’ก Recommendation - -**Use Option 1** (merge to master) because: -- Keeps documentation with code -- Uses existing Pages setup -- Simplest deployment -- No configuration changes needed - -## Questions? - -- Check: `DOCS_FIXED.md` for detailed info -- Discord: https://discord.gg/mQWsWeHsVU -- GitHub: https://github.com/flowdevs-io/Recursive-Control - ---- - -**Ready to deploy!** Choose Option 1 or 2 above to make the docs live. diff --git a/DOCS_FIXED.md b/DOCS_FIXED.md deleted file mode 100644 index dc82442..0000000 --- a/DOCS_FIXED.md +++ /dev/null @@ -1,132 +0,0 @@ -# GitHub Pages Documentation Fixed โœ… - -## Problem -The GitHub Pages site at https://flowdevs-io.github.io/Recursive-Control/ was not displaying correctly because the documentation files were empty or had only placeholder content. - -## Solution Applied - -### 1. Populated Empty Documentation Files โœ… - -Created comprehensive content for all main documentation files: - -- **`docs/index.md`** - Main landing page with overview, features, quick start, and navigation (140+ lines) -- **`docs/Installation.md`** - Complete installation guide with system requirements and setup steps (150+ lines) -- **`docs/Getting-Started.md`** - First-time user guide with examples and best practices (100+ lines) -- **`docs/API-Reference.md`** - Developer documentation for plugin development and API usage (400+ lines) -- **`docs/Multi-Agent-Architecture.md`** - Deep dive into the 3-agent system architecture (500+ lines) -- **`docs/Troubleshooting.md`** - Common issues and solutions (250+ lines) - -### 2. GitHub Pages Configuration โœ… - -Configured to use GitHub's built-in Jekyll Pages deployment from the `docs` branch. - -### 3. Content Features - -Each documentation page now contains: -- โœ… Proper Jekyll front matter (layout, title) -- โœ… Comprehensive explanations and tutorials -- โœ… Code examples and best practices -- โœ… Navigation links between pages -- โœ… Discord and GitHub links for community support -- โœ… Professional formatting with emojis for visual appeal -- โœ… Mermaid diagrams where appropriate - -## Files Changed - -1. **`docs/index.md`** - From blank to full landing page (140+ lines) -2. **`docs/Installation.md`** - From 4 lines to comprehensive guide (150+ lines) -3. **`docs/Getting-Started.md`** - From 4 lines to tutorial (100+ lines) -4. **`docs/API-Reference.md`** - From placeholder to full API docs (400+ lines) -5. **`docs/Multi-Agent-Architecture.md`** - From placeholder to architecture guide (500+ lines) -6. **`docs/Troubleshooting.md`** - From placeholder to troubleshooting guide (250+ lines) - -## Total Content Added -- โœ… Over 1,540 lines of documentation -- โœ… 6 comprehensive guides -- โœ… Professional, user-friendly content -- โœ… Complete navigation structure - -## What Users Will See Now - -When visiting https://flowdevs-io.github.io/Recursive-Control/, users will now see: - -1. โœ… **Professional landing page** with project overview and key features -2. โœ… **Clear navigation** to all documentation sections -3. โœ… **Step-by-step installation guide** for getting started -4. โœ… **Practical examples** and tutorials -5. โœ… **Technical documentation** for developers -6. โœ… **Support resources** and community links -7. โœ… **Troubleshooting guide** for common issues - -## Verification Steps - -The GitHub Pages site should automatically deploy. To verify: - -### 1. Check GitHub Pages Settings -1. Go to: https://github.com/flowdevs-io/Recursive-Control/settings/pages -2. Verify it's set to deploy from the `docs` branch -3. The source should be either: - - **Build and deployment**: GitHub Actions - - OR **Deploy from branch**: `docs` branch, `/docs` folder - -### 2. Wait for Deployment -- GitHub Pages typically takes 1-5 minutes to build and deploy -- You can check deployment status at: https://github.com/flowdevs-io/Recursive-Control/deployments - -### 3. Test the Site -Visit https://flowdevs-io.github.io/Recursive-Control/ and verify: -- โœ… Landing page shows content (not blank) -- โœ… Navigation links work -- โœ… All documentation pages load correctly -- โœ… Images and styling appear properly - -## Manual Configuration (If Needed) - -If the site still doesn't show up, manually configure GitHub Pages: - -1. Go to: https://github.com/flowdevs-io/Recursive-Control/settings/pages -2. Under "Build and deployment": - - **Source**: Deploy from a branch - - **Branch**: `docs` - - **Folder**: `/docs` -3. Click **Save** -4. Wait 2-5 minutes for deployment - -## Additional Files - -- **`docs/_config.yml`** - Already configured with correct Jekyll settings -- **`docs/README.md`** - Documentation about the documentation structure - -## Repository Branch - -All changes have been pushed to the **`docs`** branch: -- Commit: 27c71f5 and earlier -- Branch: `docs` -- Remote: https://github.com/flowdevs-io/Recursive-Control/tree/docs - -## Success Criteria โœ… - -- [x] All documentation files have comprehensive content -- [x] Jekyll front matter properly configured -- [x] Navigation between pages works -- [x] Community links (Discord, GitHub) included -- [x] Professional formatting and structure -- [x] Code examples and diagrams included -- [x] Changes pushed to `docs` branch - -## Next Steps for Users - -Once deployed, users can: -1. Visit the documentation site for comprehensive guides -2. Follow the installation instructions -3. Learn about features and capabilities -4. Get help from troubleshooting guide -5. Join the community on Discord -6. Contribute to the project - ---- - -**Site URL**: https://flowdevs-io.github.io/Recursive-Control/ -**Repository**: https://github.com/flowdevs-io/Recursive-Control -**Branch**: docs -**Status**: โœ… Documentation Fixed and Ready diff --git a/ENABLE_WIKI_GUIDE.md b/ENABLE_WIKI_GUIDE.md deleted file mode 100644 index a65eae4..0000000 --- a/ENABLE_WIKI_GUIDE.md +++ /dev/null @@ -1,354 +0,0 @@ -# How to Enable GitHub Wiki & Documentation - -## ๐Ÿšซ Issue: "I don't see a wiki page option in GitHub" - -GitHub Wiki needs to be **enabled** in repository settings. Here are **3 solutions**: - ---- - -## โœ… Solution 1: Enable GitHub Wiki (Recommended) - -### Step-by-Step: - -1. **Go to Your Repository** - ``` - https://github.com/flowdevs-io/Recursive-Control - ``` - -2. **Click "Settings" Tab** - - Top navigation bar - - Requires admin/owner permissions - -3. **Scroll to "Features" Section** - - About halfway down the settings page - - Look for checkboxes - -4. **Enable Wiki** - - Find "Wikis" checkbox - - โœ… Check the box - - Wait for save (automatic) - -5. **Wiki Tab Appears!** - - Refresh page - - "Wiki" tab now visible in main navigation - - Click to create first page - -### Publish Wiki Content: - -**Method A: Web Interface** -``` -1. Click "Wiki" tab -2. Click "Create the first page" -3. Title: "Home" -4. Content: Copy from wiki/Home.md -5. Click "Save Page" -6. Repeat for other pages -``` - -**Method B: Git Clone** -```bash -# Clone wiki repository -git clone https://github.com/flowdevs-io/Recursive-Control.wiki.git - -# Copy all wiki files -cp wiki/*.md Recursive-Control.wiki/ - -# Rename Home to match GitHub convention -cd Recursive-Control.wiki -mv Home.md Home.md # Already correct - -# Commit and push -git add . -git commit -m "Complete documentation wiki" -git push origin master -``` - ---- - -## โœ… Solution 2: Use GitHub Pages (Alternative) - -**I've already set this up for you!** The `docs/` folder is ready. - -### Step-by-Step: - -1. **Go to Repository Settings** - ``` - Settings โ†’ Pages (left sidebar) - ``` - -2. **Configure GitHub Pages** - - **Source**: Deploy from a branch - - **Branch**: `main` (or `master`) - - **Folder**: `/docs` - - Click **Save** - -3. **Wait 2-3 Minutes** - - GitHub builds your site - - Check Actions tab for build status - -4. **Access Your Documentation** - ``` - https://flowdevs-io.github.io/Recursive-Control/ - ``` - -### What's Already Set Up: - -``` -docs/ -โ”œโ”€โ”€ _config.yml # Jekyll configuration โœ… -โ”œโ”€โ”€ index.md # Home page (from wiki/Home.md) โœ… -โ”œโ”€โ”€ Installation.md # Setup guide โœ… -โ”œโ”€โ”€ Getting-Started.md # Tutorial โœ… -โ”œโ”€โ”€ Multi-Agent-Architecture.md # Technical deep dive โœ… -โ”œโ”€โ”€ FAQ.md # Questions & answers โœ… -โ”œโ”€โ”€ Troubleshooting.md # Problem solving โœ… -โ”œโ”€โ”€ API-Reference.md # Developer docs โœ… -โ”œโ”€โ”€ Blog-Post-v2.0.md # v2.0 announcement โœ… -โ”œโ”€โ”€ System-Prompts-Reference.md # Prompts โœ… -โ”œโ”€โ”€ UI-Features.md # UI improvements โœ… -โ””โ”€โ”€ UI-Redesign.md # UI redesign โœ… -``` - -**Theme**: Cayman (beautiful, modern) -**Features**: -- Automatic navigation -- Syntax highlighting -- Mobile responsive -- SEO optimized - ---- - -## โœ… Solution 3: Use README + Docs Folder (Simplest) - -Keep everything in the main repository with links. - -### Update Main README.md: - -Add this section: - -```markdown -## ๐Ÿ“š Documentation - -- [Installation Guide](docs/Installation.md) -- [Getting Started](docs/Getting-Started.md) -- [Multi-Agent Architecture](docs/Multi-Agent-Architecture.md) -- [API Reference](docs/API-Reference.md) -- [FAQ](docs/FAQ.md) -- [Troubleshooting](docs/Troubleshooting.md) - -### Reference -- [Version 2.0 Blog Post](docs/Blog-Post-v2.0.md) -- [System Prompts Reference](docs/System-Prompts-Reference.md) -- [UI Features](docs/UI-Features.md) -- [UI Redesign](docs/UI-Redesign.md) -``` - -**Pros:** -- โœ… No setup needed -- โœ… Works immediately -- โœ… Visible to all users -- โœ… Easy to maintain - -**Cons:** -- โŒ No wiki-style interface -- โŒ No automatic navigation -- โŒ Less discoverable - ---- - -## ๐ŸŽฏ Comparison: Which to Use? - -### GitHub Wiki -**Best for:** Traditional wiki experience -**Pros:** -- Separate git repository -- Wiki-style navigation -- Easy for non-devs to edit -- Standard GitHub feature -**Cons:** -- Requires enabling -- Separate from main repo -- Less visibility in searches - -### GitHub Pages -**Best for:** Professional documentation site โญ -**Pros:** -- Beautiful themed website -- Custom domain support -- Full control over design -- Great SEO -- Already set up! -**Cons:** -- Slightly more complex -- Requires Pages setup (5 minutes) - -### Docs Folder in Repo -**Best for:** Quick and simple -**Pros:** -- Immediate availability -- No setup -- Single repository -- Version controlled with code -**Cons:** -- Basic markdown rendering -- No navigation sidebar -- Manual links needed - ---- - -## ๐Ÿš€ My Recommendation - -**Use GitHub Pages** (Solution 2) because: - -1. โœ… I've already set it up for you -2. โœ… Professional appearance -3. โœ… Automatic navigation -4. โœ… Beautiful Cayman theme -5. โœ… Mobile-friendly -6. โœ… Takes 2 minutes to enable - -**Steps:** -``` -1. Go to: Settings โ†’ Pages -2. Source: "Deploy from a branch" -3. Branch: main, Folder: /docs -4. Click Save -5. Wait 2-3 minutes -6. Visit: https://flowdevs-io.github.io/Recursive-Control/ -``` - -**Done!** ๐ŸŽ‰ - ---- - -## ๐Ÿ”ง Enabling GitHub Wiki (Detailed) - -### If You Don't See Settings: - -**Problem:** Not repository owner/admin - -**Solutions:** -- Ask repository owner to enable -- Fork repository (you'll have settings) -- Use GitHub Pages or docs folder instead - -### If Wiki Option is Disabled: - -**Problem:** Organization policy or repository type - -**Solutions:** -1. Check organization settings -2. Contact org admin -3. Use GitHub Pages instead - -### If Wiki Enable Checkbox Missing: - -**Problem:** Older GitHub interface or private repo restrictions - -**Solutions:** -1. Update repository visibility settings -2. Enable via GitHub API -3. Use GitHub Pages as alternative - ---- - -## ๐Ÿ“ž Quick Help - -### Enable Wiki Not Working? -```bash -# Enable via GitHub CLI (if you have gh installed) -gh repo edit --enable-wiki - -# Or via API -curl -X PATCH \ - -H "Authorization: token YOUR_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/flowdevs-io/Recursive-Control \ - -d '{"has_wiki":true}' -``` - -### GitHub Pages Not Building? -1. Check Actions tab for errors -2. Verify `docs/` folder exists -3. Check `_config.yml` is valid YAML -4. Make sure branch is correct (main/master) - -### Links Not Working? -- Wiki links: Use page names without .md -- GitHub Pages: Use full paths with .md (or remove for clean URLs) -- Docs folder: Use relative paths with .md - ---- - -## โœ… What You Have Now - -**Ready to Use:** -``` -โœ… docs/ folder with all documentation -โœ… _config.yml configured for GitHub Pages -โœ… 12 markdown files ready -โœ… 78,000 words of content -โœ… Beautiful theme selected -โœ… Navigation configured -``` - -**To Publish:** -``` -1. Enable GitHub Pages (2 minutes) - OR -2. Enable Wiki and copy files (5 minutes) - OR -3. Use docs/ folder directly (immediate) -``` - ---- - -## ๐ŸŽ‰ Next Steps - -**Choose Your Method:** - -**Option A: GitHub Pages (Recommended)** -```bash -# Already done! Just enable in Settings โ†’ Pages -# Result: https://flowdevs-io.github.io/Recursive-Control/ -``` - -**Option B: GitHub Wiki** -```bash -# Enable in Settings โ†’ Features โ†’ Wikis โœ“ -# Clone wiki and copy files -git clone https://github.com/flowdevs-io/Recursive-Control.wiki.git -cp wiki/*.md Recursive-Control.wiki/ -cd Recursive-Control.wiki && git add . && git commit -m "Docs" && git push -``` - -**Option C: Docs Folder** -```bash -# Already done! Just update main README.md -# Add links to docs/*.md files -# Commit and push -``` - ---- - -## ๐Ÿ’ก Pro Tip - -**Use GitHub Pages for best results!** - -It gives you: -- Professional documentation site -- Automatic navigation -- Beautiful theme -- Mobile-friendly -- SEO optimized -- Free hosting - -And I've already set it all up for you! Just enable it in settings. ๐Ÿš€ - ---- - -

- Questions?
- Join Discord | - Report Issue -

diff --git a/FINAL_STATUS.md b/FINAL_STATUS.md new file mode 100644 index 0000000..e69de29 diff --git a/FIXES_APPLIED.md b/FIXES_APPLIED.md deleted file mode 100644 index ad6901c..0000000 --- a/FIXES_APPLIED.md +++ /dev/null @@ -1,185 +0,0 @@ -# Fixes Applied to FlowVision - -## Date: October 2, 2025 - -### Issues Fixed - -#### 1. Tool Call Compilation Error -**Problem**: The test project `FlowVision.Tests` could not access the `SetChatHistory` method in `MultiAgentActioner` class. - -**Root Cause**: The method was marked as `internal` instead of `public`, making it inaccessible from the test assembly. - -**Solution**: Changed the accessibility modifier from `internal` to `public` in: -- **File**: `FlowVision/lib/Classes/ai/MultiAgentActioner.cs` -- **Line**: 427 -- **Change**: `internal void SetChatHistory(...)` โ†’ `public void SetChatHistory(...)` - -**Status**: โœ… **FIXED** - Project now compiles successfully with 0 errors - ---- - -#### 2. ONNX OmniParser Initialization -**Problem**: The ONNX runtime version of OmniParser was not being initialized at startup, meaning the YOLO model had to be loaded on first use, causing delays. - -**Solution**: Modified the `ScreenCaptureOmniParserPlugin` constructor to automatically initialize the ONNX engine at plugin instantiation. - -**Changes Made**: -- **File**: `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs` -- **Location**: Constructor (lines 29-36) -- **Implementation**: - ```csharp - public ScreenCaptureOmniParserPlugin() - { - _windowSelector = new WindowSelectionPlugin(); - - // Initialize ONNX engine at startup to keep YOLO model ready - if (_useOnnxMode && _onnxEngine == null) - { - ConfigureMode(true); - } - } - ``` - -**Benefits**: -- โœ… YOLO model is now loaded and ready when the plugin is first created -- โœ… Screenshots can be processed immediately without waiting for model initialization -- โœ… Reduces latency from ~15-30 seconds (HTTP server startup) to < 1 second (ONNX ready) -- โœ… No Python server required - runs entirely in native .NET - -**Status**: โœ… **IMPLEMENTED** - ONNX engine now initializes automatically - ---- - -## Build Results - -### Main Project (FlowVision) -``` -Build succeeded. -11 Warning(s) -0 Error(s) -Time Elapsed 00:00:00.90 -``` - -### Test Project (FlowVision.Tests) -``` -Build succeeded. -14 Warning(s) -0 Error(s) -Time Elapsed 00:00:01.54 -``` - -### Full Solution -``` -Build succeeded. -14 Warning(s) -0 Error(s) -Time Elapsed 00:00:01.76 -``` - ---- - -## Technical Details - -### ONNX OmniParser Architecture - -The ONNX implementation provides several advantages over the Python HTTP server approach: - -| Feature | ONNX Mode | HTTP Server Mode | -|---------|-----------|------------------| -| **Startup Time** | < 1 second | 15-30 seconds | -| **Runtime** | Native .NET | Python + FastAPI | -| **Memory Usage** | ~500 MB | ~1-2 GB | -| **Dependencies** | ONNX Runtime only | Python, PyTorch, FastAPI | -| **Inference Speed** | 2-5 seconds (CPU) | 3-6 seconds | -| **Deployment** | Single executable | Separate Python environment | - -### ONNX Model Details -- **Location**: `T:\OmniParser\weights\icon_detect\model.onnx` -- **Architecture**: YOLOv11m (medium variant) -- **Input Size**: 640x640 pixels -- **Output**: Bounding boxes with confidence scores -- **Size**: 76.7 MB -- **Confidence Threshold**: 0.05 (configurable) - -### Tool Extraction Process - -Tools are extracted from plugins using the `PluginToolExtractor` class, which: -1. Reflects on plugin instances to find public methods -2. Converts methods to `AITool` objects using `AIFunctionFactory` -3. Registers tools with the chat client -4. Enables automatic function invocation via `ChatClientBuilder.UseFunctionInvocation()` - -This happens in multiple actioners: -- `Actioner.cs` (Azure OpenAI) -- `MultiAgentActioner.cs` (Multi-agent coordinator) -- `LMStudioActioner.cs` (Local LM Studio) - ---- - -## Testing Notes - -The test project has a dependency issue with Windows Forms that causes runtime errors in the test environment. This is unrelated to the fixes applied and does not affect the main application functionality. The compilation errors have been resolved. - ---- - -## Files Modified - -1. **FlowVision/lib/Classes/ai/MultiAgentActioner.cs** - - Changed `SetChatHistory` from `internal` to `public` - -2. **FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs** - - Added automatic ONNX engine initialization in constructor - ---- - -## Verification - -To verify the fixes: - -1. **Compilation**: โœ… Solution builds with 0 errors -2. **ONNX Model**: โœ… Model exists at `T:\OmniParser\weights\icon_detect\model.onnx` -3. **Plugin Initialization**: โœ… ONNX engine initializes when `ScreenCaptureOmniParserPlugin` is instantiated -4. **Tool Accessibility**: โœ… `SetChatHistory` method is now public and accessible from tests - ---- - -## Usage - -### To use the ONNX OmniParser: - -```csharp -// Automatic initialization (now default) -var plugin = new ScreenCaptureOmniParserPlugin(); -// ONNX engine is already initialized and ready! - -// Capture screen with immediate processing -var results = await plugin.CaptureWholeScreen(); -``` - -### To force HTTP server mode (if needed): - -```csharp -ScreenCaptureOmniParserPlugin.ConfigureMode(useOnnx: false); -``` - ---- - -## Next Steps - -Consider these future enhancements: - -1. **GPU Acceleration**: Enable CUDA support for faster inference (0.5-1s vs 2-5s) -2. **Florence2 Integration**: Add caption model for element descriptions -3. **Batch Processing**: Process multiple screenshots in parallel -4. **Model Caching**: Implement model warming strategies for even faster first-time use -5. **Test Infrastructure**: Fix Windows Forms dependency issue in test project - ---- - -## Summary - -Both issues have been successfully resolved: -- โœ… Tool calls now work correctly (compilation fixed) -- โœ… ONNX OmniParser initializes automatically, keeping YOLO model ready for instant screenshot processing - -The application now provides faster, more reliable UI element detection with no Python dependencies required. diff --git a/FlowVision/FlowVision.csproj b/FlowVision/FlowVision.csproj index b4a63fc..ade13fc 100644 --- a/FlowVision/FlowVision.csproj +++ b/FlowVision/FlowVision.csproj @@ -168,10 +168,6 @@ ..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll - - ..\packages\Tesseract.5.2.0\lib\net48\Tesseract.dll - True - ..\packages\Microsoft.ML.OnnxRuntime.Managed.1.21.1\lib\netstandard2.0\Microsoft.ML.OnnxRuntime.dll True @@ -237,6 +233,7 @@ + @@ -319,15 +316,7 @@ - - - - - - - - diff --git a/FlowVision/lib/Classes/FlorenceCaptionBridge.cs b/FlowVision/lib/Classes/FlorenceCaptionBridge.cs new file mode 100644 index 0000000..e69de29 diff --git a/GITHUB_PAGES_FIX.md b/GITHUB_PAGES_FIX.md deleted file mode 100644 index f01cdf0..0000000 --- a/GITHUB_PAGES_FIX.md +++ /dev/null @@ -1,342 +0,0 @@ -# GitHub Pages Build Fix - -## โœ… Issue Fixed - -The error `No such file or directory @ dir_chdir0 - /github/workspace/docs` has been resolved. - ---- - -## ๐Ÿ”ง What Was Wrong - -**Problem 1: Theme Configuration** -```yaml -# โŒ Old (doesn't work with GitHub Actions) -theme: jekyll-theme-cayman - -# โœ… New (works with GitHub Pages) -remote_theme: pages-themes/cayman@v0.2.0 -``` - -**Problem 2: Missing YAML Front Matter** -- Documentation files need front matter for Jekyll to process them -- Each page needs layout and title specified - -**Problem 3: Plugin Configuration** -- Used plugins that aren't in GitHub Pages whitelist -- Needed to specify `jekyll-remote-theme` plugin - ---- - -## โœ… What I Fixed - -### 1. Updated `_config.yml` - -**Changed to:** -```yaml -title: Recursive Control Documentation -description: AI-Powered Computer Control for Windows -baseurl: "/Recursive-Control" -url: "https://flowdevs-io.github.io" - -remote_theme: pages-themes/cayman@v0.2.0 - -plugins: - - jekyll-remote-theme - - jekyll-seo-tag - - jekyll-sitemap - -markdown: kramdown -kramdown: - input: GFM - hard_wrap: false - syntax_highlighter: rouge - -show_downloads: false -``` - -### 2. Added YAML Front Matter to All Pages - -**Every .md file now has:** -```yaml ---- -layout: default -title: Page Title ---- -``` - -**Files updated:** -- โœ… index.md (Home) -- โœ… Installation.md -- โœ… Getting-Started.md -- โœ… Multi-Agent-Architecture.md -- โœ… API-Reference.md -- โœ… FAQ.md -- โœ… Troubleshooting.md - -### 3. Fixed Plugin List - -Only using GitHub Pages whitelisted plugins: -- `jekyll-remote-theme` (for Cayman theme) -- `jekyll-seo-tag` (for meta tags) -- `jekyll-sitemap` (for sitemap.xml) - ---- - -## ๐Ÿš€ How to Enable Now - -### Step 1: Push Changes - -```bash -cd T:\Recursive-Control - -# Stage all documentation changes -git add docs/ -git add GITHUB_PAGES_FIX.md - -# Commit -git commit -m "Fix GitHub Pages configuration for proper build" - -# Push to GitHub -git push origin main -``` - -### Step 2: Enable GitHub Pages - -1. Go to: **Settings** โ†’ **Pages** -2. **Source**: Deploy from a branch -3. **Branch**: `main` -4. **Folder**: `/docs` -5. Click **Save** - -### Step 3: Wait for Build - -- Check **Actions** tab -- Wait for green checkmark (2-3 minutes) -- Build should succeed now! - -### Step 4: Visit Your Site - -``` -https://flowdevs-io.github.io/Recursive-Control/ -``` - ---- - -## ๐ŸŽจ Features Working Now - -โœ… **Beautiful Cayman Theme** -- Professional appearance -- Gradient header -- Code syntax highlighting -- Mobile responsive - -โœ… **All Documentation Pages** -- Home page with navigation -- Installation guide -- Getting started tutorial -- Multi-agent architecture -- API reference -- FAQ -- Troubleshooting - -โœ… **Proper Navigation** -- Links between pages work -- Relative paths correct -- Images will load (if added) - -โœ… **SEO Optimized** -- Meta tags -- Sitemap.xml -- Proper page titles - ---- - -## ๐Ÿ” Verify Build Success - -### Check Actions Tab - -1. Go to repository **Actions** tab -2. Look for latest workflow run -3. Should show: โœ… **pages build and deployment** -4. Click to see details - -**Expected output:** -``` -โœ… Build successful -โœ… Deploy successful -๐ŸŒ Published to: https://flowdevs-io.github.io/Recursive-Control/ -``` - -### If Build Still Fails - -**Check:** -```bash -# Ensure all files have YAML front matter -head -5 docs/Installation.md -# Should show: -# --- -# layout: default -# title: Installation -# --- - -# Verify _config.yml is valid YAML -cat docs/_config.yml | yaml-lint -``` - -**Common issues:** -- Missing front matter: Add to all .md files -- Invalid YAML: Check for tabs (use spaces) -- Wrong branch: Must be `main` or `master` -- Wrong folder: Must be `/docs` - ---- - -## ๐Ÿ“Š File Structure - -``` -docs/ -โ”œโ”€โ”€ _config.yml # โœ… Fixed Jekyll config -โ”œโ”€โ”€ README.md # Documentation index -โ”œโ”€โ”€ index.md # โœ… Home page with front matter -โ”œโ”€โ”€ Installation.md # โœ… With front matter -โ”œโ”€โ”€ Getting-Started.md # โœ… With front matter -โ”œโ”€โ”€ Multi-Agent-Architecture.md # โœ… With front matter -โ”œโ”€โ”€ API-Reference.md # โœ… With front matter -โ”œโ”€โ”€ FAQ.md # โœ… With front matter -โ”œโ”€โ”€ Troubleshooting.md # โœ… With front matter -โ”œโ”€โ”€ Blog-Post-v2.0.md -โ”œโ”€โ”€ System-Prompts-Reference.md -โ”œโ”€โ”€ UI-Features.md -โ””โ”€โ”€ UI-Redesign.md -``` - ---- - -## ๐ŸŽฏ What Changed - -| Item | Before | After | -|------|--------|-------| -| Theme | `theme:` (local) | `remote_theme:` (GitHub) | -| Plugins | Custom list | Whitelisted only | -| Front matter | Missing | All pages have it | -| baseurl | Not set | `/Recursive-Control` | -| url | Not set | Full GitHub Pages URL | - ---- - -## ๐Ÿ’ก Alternative: Minimal Theme - -If you prefer a simpler theme, change in `_config.yml`: - -```yaml -# Cayman (current, colorful) -remote_theme: pages-themes/cayman@v0.2.0 - -# OR Minimal (clean, simple) -remote_theme: pages-themes/minimal@v0.2.0 - -# OR Slate (dark theme) -remote_theme: pages-themes/slate@v0.2.0 - -# OR Architect (modern) -remote_theme: pages-themes/architect@v0.2.0 -``` - ---- - -## ๐Ÿ†˜ Troubleshooting - -### Build Fails with "unknown tag 'seo'" - -**Solution:** Ensure `jekyll-seo-tag` is in plugins: -```yaml -plugins: - - jekyll-seo-tag -``` - -### Pages Don't Style Correctly - -**Solution:** Check baseurl in `_config.yml` matches repo name: -```yaml -baseurl: "/Recursive-Control" # Must match repo name exactly -``` - -### 404 on All Pages - -**Solution:** -1. Verify index.md exists (not just Home.md) -2. Check GitHub Pages is enabled -3. Wait for build to complete (Actions tab) -4. Clear browser cache - -### Links Don't Work - -**Solution:** Use relative links without baseurl: -```markdown -โœ… [Installation](Installation.md) -โŒ [Installation](/Recursive-Control/Installation.md) -``` - -Jekyll handles baseurl automatically. - ---- - -## โœ… Verification Checklist - -Before pushing, verify: - -- [ ] `_config.yml` uses `remote_theme` -- [ ] All .md files have front matter -- [ ] `baseurl` matches repo name -- [ ] Only whitelisted plugins -- [ ] Valid YAML (no tabs) -- [ ] `index.md` exists as homepage - -After pushing: - -- [ ] Actions tab shows green checkmark -- [ ] Can access homepage -- [ ] Links between pages work -- [ ] Theme renders correctly -- [ ] Code blocks have syntax highlighting - ---- - -## ๐ŸŽ‰ Success! - -Your documentation should now build and deploy successfully to: - -``` -https://flowdevs-io.github.io/Recursive-Control/ -``` - -**All 78,000 words of documentation, beautifully themed and fully navigable!** ๐Ÿ“šโœจ - ---- - -## ๐Ÿ“ž Still Having Issues? - -1. **Check Actions Log** - - Go to Actions tab - - Click on failed workflow - - Read full error message - -2. **Validate Configuration** - ```bash - # Test YAML syntax - ruby -ryaml -e "YAML.load_file('docs/_config.yml')" - ``` - -3. **Compare Working Example** - - Look at successful GitHub Pages repos - - Check their _config.yml - - Verify file structure matches - -4. **Ask for Help** - - Open issue with full error log - - Share repository URL - - Include Actions build output - ---- - -**The configuration is now correct and should build successfully!** ๐Ÿš€ diff --git a/IMPLEMENTATION_SUMMARY.md b/IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index 910ff08..0000000 --- a/IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,390 +0,0 @@ -# OmniParser ONNX Integration - Implementation Summary - -## ๐ŸŽ‰ Mission Accomplished! - -Successfully integrated **Microsoft OmniParser** to run natively in .NET using ONNX Runtime, eliminating the need for a Python server! - ---- - -## ๐Ÿ“‹ What Was Implemented - -### 1. Native ONNX Inference Engine -**File**: `FlowVision/lib/Classes/OnnxOmniParserEngine.cs` - -A complete .NET implementation of OmniParser that: -- โœ… Loads YOLOv8 model directly from ONNX format -- โœ… Performs real-time UI element detection -- โœ… Implements Non-Maximum Suppression (NMS) for overlapping boxes -- โœ… Preprocesses images (resize, normalize, tensor conversion) -- โœ… Post-processes outputs (coordinate conversion, confidence filtering) -- โœ… Provides visualization capabilities (draw bounding boxes) - -**Key Features**: -- Direct ONNX model inference in .NET -- No Python dependencies -- Sub-second initialization -- Memory-efficient tensor operations -- Configurable confidence thresholds -- GPU support ready (requires CUDA ONNX Runtime) - -### 2. Updated Plugin with Dual-Mode Support -**File**: `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs` - -Enhanced the existing plugin to support both modes: -- โœ… **ONNX Mode** (default): Native .NET inference -- โœ… **HTTP Server Mode** (fallback): Legacy Python server -- โœ… Automatic fallback mechanism -- โœ… Seamless mode switching at runtime -- โœ… Backward compatibility maintained - -**Benefits**: -- Existing code continues to work without changes -- Users get automatic performance improvements -- Graceful degradation if ONNX fails -- Easy configuration for different scenarios - -### 3. Model Conversion -**Process**: Converted PyTorch YOLOv8 model to ONNX format - -```bash -# Automated conversion -from ultralytics import YOLO -model = YOLO('model.pt') -model.export(format='onnx', imgsz=640, simplify=True, opset=12) -``` - -**Result**: -- Input: `model.pt` (38.7 MB) -- Output: `model.onnx` (76.7 MB) -- Format: ONNX opset 12 -- Optimized: Simplified graph structure - -### 4. Dependency Management -**Updated Files**: -- `FlowVision/packages.config` -- `FlowVision/FlowVision.csproj` - -**Added Packages**: -- `Microsoft.ML.OnnxRuntime` (v1.19.2) - Native runtime -- `Microsoft.ML.OnnxRuntime.Managed` (v1.21.1) - Managed API -- Already had: `System.Numerics.Tensors` - Tensor operations - -**Build Integration**: -- Automatic native DLL copying -- Proper reference paths -- Runtime dependency resolution - ---- - -## ๐Ÿ”ง Technical Details - -### Architecture - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ ScreenCaptureOmniParserPlugin โ”‚ -โ”‚ (Capture โ†’ Process โ†’ Return Results) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ - โ”‚ โ”‚ - โ”Œโ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” - โ”‚ ONNX Mode โ”‚ โ”‚ HTTP Server Mode โ”‚ - โ”‚ (Default) โ”‚ โ”‚ (Fallback) โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ - โ”‚ โ”‚ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ OnnxOmniParserEngine โ”‚ โ”‚ LocalOmniParserManager โ”‚ -โ”‚ โ”‚ โ”‚ + OmniParserClient โ”‚ -โ”‚ - Load ONNX Model โ”‚ โ”‚ โ”‚ -โ”‚ - Preprocess Image โ”‚ โ”‚ - Start Python Server โ”‚ -โ”‚ - Run Inference โ”‚ โ”‚ - HTTP Communication โ”‚ -โ”‚ - Post-process Results โ”‚ โ”‚ - Process Response โ”‚ -โ”‚ - NMS Filtering โ”‚ โ”‚ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ - โ”‚ โ”‚ - โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ” - โ”‚ Unified ParsedContent Format โ”‚ - โ”‚ (Type, BBox, Content, Interactivity) โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### Performance Comparison - -| Metric | ONNX Mode | HTTP Server Mode | Improvement | -|-------------------|-----------|------------------|-------------| -| **Startup Time** | < 1 sec | 15-30 sec | **30x faster** | -| **Inference Time**| 2-5 sec | 3-6 sec | **20% faster** | -| **Memory Usage** | ~500 MB | ~1-2 GB | **50-75% less** | -| **Dependencies** | .NET only | Python + packages| **Simpler** | -| **Deployment** | Single exe| Multi-process | **Easier** | - -### Model Details - -**YOLOv11m Architecture**: -- Input: 640ร—640 RGB image (NCHW format) -- Output: [1, 5, 8400] tensor (bbox + confidence) -- Parameters: 20 million -- GFLOPs: 67.6 - -**Detection Pipeline**: -1. Image preprocessing (resize, normalize to [0,1]) -2. Tensor conversion (CHW format) -3. ONNX inference -4. Output parsing (center coords โ†’ corner coords) -5. Confidence filtering (threshold: 0.05) -6. Non-Maximum Suppression (IoU threshold: 0.45) -7. Coordinate scaling to original image size - ---- - -## ๐Ÿ“ Files Created/Modified - -### New Files -1. โœ… `FlowVision/lib/Classes/OnnxOmniParserEngine.cs` (404 lines) - - Core ONNX inference engine - -2. โœ… `ONNX_OMNIPARSER_INTEGRATION.md` (comprehensive docs) - - Architecture explanation - - API reference - - Troubleshooting guide - -3. โœ… `ONNX_QUICKSTART.md` (quick start guide) - - 3-step setup - - Example usage - - Common issues - -4. โœ… `T:\OmniParser\weights\icon_detect\model.onnx` (76.7 MB) - - Converted YOLO model - -### Modified Files -1. โœ… `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs` - - Added ONNX mode support - - Dual-mode implementation - - Automatic fallback logic - -2. โœ… `FlowVision/FlowVision.csproj` - - Added ONNX Runtime references - - Build task for native DLL copying - - Compilation entry for new class - -3. โœ… `FlowVision/packages.config` - - Added ONNX Runtime packages - -### Build Artifacts -- โœ… `FlowVision/bin/Debug/Microsoft.ML.OnnxRuntime.dll` -- โœ… `FlowVision/bin/Debug/onnxruntime.dll` -- โœ… `FlowVision/bin/Debug/onnxruntime_providers_shared.dll` - ---- - -## ๐Ÿš€ Usage Examples - -### Basic Usage (Auto ONNX Mode) -```csharp -var plugin = new ScreenCaptureOmniParserPlugin(); -var results = await plugin.CaptureWholeScreen(); -// Automatically uses ONNX - no Python required! -``` - -### Direct ONNX Engine -```csharp -using var engine = new OnnxOmniParserEngine(); -var result = engine.ParseImage(screenshot); - -foreach (var detection in result.Detections) -{ - Console.WriteLine($"{detection.ElementType} at " + - $"({detection.BoundingBox.X}, {detection.BoundingBox.Y}) " + - $"confidence: {detection.Confidence:F2}"); -} -``` - -### Configuration -```csharp -// Custom model path -ScreenCaptureOmniParserPlugin.ConfigureMode( - useOnnx: true, - onnxModelPath: @"C:\Models\custom_model.onnx" -); - -// Lower confidence threshold for more detections -var engine = new OnnxOmniParserEngine( - confidenceThreshold: 0.03f -); -``` - ---- - -## โœ… Testing & Validation - -### Build Status -- โœ… **Build**: Successful (0 errors, 11 pre-existing warnings) -- โœ… **Dependencies**: All packages restored correctly -- โœ… **Native DLLs**: Copied to output directory -- โœ… **Compilation**: All new code compiles without errors - -### Compatibility -- โœ… **Backward Compatible**: Existing code works unchanged -- โœ… **.NET Framework 4.8**: Fully supported -- โœ… **Windows**: Native x64 DLLs included -- โœ… **Fallback**: HTTP mode still available if needed - ---- - -## ๐ŸŽฏ Key Achievements - -### 1. Zero Python Dependency -- โœ… Eliminated need for Python installation -- โœ… No conda environment management -- โœ… No pip package dependencies -- โœ… Single .NET executable deployment - -### 2. Significant Performance Gains -- โœ… 30x faster startup time -- โœ… 20% faster inference -- โœ… 50-75% less memory usage -- โœ… No HTTP overhead - -### 3. Improved Developer Experience -- โœ… Pure .NET development workflow -- โœ… Better IDE integration -- โœ… Easier debugging -- โœ… Simplified deployment - -### 4. Production Ready -- โœ… Automatic fallback mechanism -- โœ… Comprehensive error handling -- โœ… Logging and diagnostics -- โœ… Resource cleanup (IDisposable) - ---- - -## ๐Ÿ“š Documentation - -Comprehensive documentation provided: - -1. **ONNX_OMNIPARSER_INTEGRATION.md** - - Architecture overview - - Complete API reference - - Performance benchmarks - - Troubleshooting guide - - Migration guide - -2. **ONNX_QUICKSTART.md** - - 3-step quick start - - Common usage patterns - - Configuration examples - - Quick troubleshooting - -3. **Inline Code Documentation** - - XML comments on all public APIs - - Clear method descriptions - - Parameter explanations - - Usage examples - ---- - -## ๐Ÿ”ฎ Future Enhancements - -Potential improvements for future work: - -### Short Term -- [ ] Florence2 caption model integration for UI element descriptions -- [ ] OCR integration (EasyOCR/PaddleOCR) for text extraction -- [ ] Configurable NMS IoU threshold -- [ ] Batch processing for multiple images - -### Medium Term -- [ ] GPU acceleration (CUDA/DirectML) -- [ ] Model quantization for faster inference -- [ ] Custom model training pipeline -- [ ] REST API for remote inference - -### Long Term -- [ ] Web Assembly (WASM) for browser deployment -- [ ] Mobile support (iOS/Android via Xamarin) -- [ ] Cloud deployment (Azure Container Apps) -- [ ] Real-time video stream processing - ---- - -## ๐Ÿ› Known Limitations - -1. **CPU-Only by Default** - - GPU support requires CUDA toolkit installation - - GPU packages are larger (~500 MB vs ~100 MB) - -2. **Caption Generation Not Included** - - Current implementation detects bounding boxes only - - Florence2 caption model not yet integrated - -3. **Single Image Processing** - - No batch processing support yet - - Each image processed independently - -4. **Windows x64 Only** - - Native DLLs are Windows-specific - - Other platforms need appropriate ONNX Runtime builds - ---- - -## ๐Ÿ’ก Recommendations - -### For Development -1. Use ONNX mode for all new development -2. HTTP server mode only for testing/comparison -3. Monitor memory usage for large images -4. Consider GPU mode for production workloads - -### For Deployment -1. Include ONNX model file with application -2. Verify native DLLs are in output directory -3. Test fallback mechanism before deployment -4. Document model file location for users - -### For Users -1. Start with default settings (ONNX mode, 0.05 threshold) -2. Adjust confidence threshold based on results -3. Use visualization to verify detection quality -4. Report issues with example screenshots - ---- - -## ๐Ÿ™ Acknowledgments - -- **Microsoft Research** - OmniParser model and Python implementation -- **Microsoft** - ONNX Runtime and ML.NET -- **Ultralytics** - YOLOv8/v11 architecture -- **Open Source Community** - Various libraries and tools - ---- - -## ๐Ÿ“„ License - -This implementation maintains compatibility with: -- OmniParser: MIT License -- ONNX Runtime: MIT License -- FlowVision: Original project license - ---- - -## ๐ŸŽ“ Summary - -This implementation successfully achieves the goal of **running OmniParser locally without Python**, providing: - -โœ… **Native .NET Integration** - Pure C# ONNX inference -โœ… **Superior Performance** - 30x faster startup, 20% faster inference -โœ… **Simplified Deployment** - Single executable, no Python -โœ… **Backward Compatible** - Existing code works unchanged -โœ… **Production Ready** - Error handling, logging, fallback -โœ… **Well Documented** - Comprehensive guides and examples - -The application now runs OmniParser as a first-class .NET component, making it faster, simpler, and more maintainable! ๐Ÿš€ - ---- - -**Date**: February 10, 2025 -**Status**: โœ… Complete and Tested -**Build**: Successful (0 errors) diff --git a/KEYBOARD_FOCUS_FIX.md b/KEYBOARD_FOCUS_FIX.md deleted file mode 100644 index c7f9636..0000000 --- a/KEYBOARD_FOCUS_FIX.md +++ /dev/null @@ -1,423 +0,0 @@ -# Keyboard Focus Fix - Window-Specific Key Input - -## Date: October 2, 2025 - -## Problem - -Keyboard shortcuts were not being sent to the correct tab/window. The issue was that `KeyboardPlugin` sent keystrokes globally to whatever window currently had focus, rather than targeting a specific window. - -### Symptoms -- โŒ Keyboard shortcuts sent to wrong application -- โŒ Keys typed in unexpected windows -- โŒ Tab switching commands going to wrong browser -- โŒ No way to ensure target window has focus before sending keys - -## Root Cause - -The original `KeyboardPlugin` used `SendKeys.SendWait()` without any window context: - -```csharp -// OLD CODE - sends to whatever has focus -public async Task SendKey(string keyCombo) -{ - SendKeys.SendWait(keyCombo); // โŒ No window targeting! - return true; -} -``` - -While `MousePlugin` had window-specific methods: -```csharp -// MousePlugin already had window targeting -public async Task ClickOnWindow(string windowHandleString, ...) -``` - -This inconsistency meant the AI could click on specific windows but couldn't send keys to them reliably. - -## Solution - -### New Window-Targeted Methods - -Added three new methods to `KeyboardPlugin` that accept window handles: - -1. **`SendKeyToWindow(windowHandle, keyCombo)`** - Send any key combination to specific window -2. **`EnterKeyToWindow(windowHandle)`** - Send Enter to specific window -3. **`CtrlKeyToWindow(windowHandle, letter)`** - Send Ctrl+Letter to specific window - -### Enhanced Focus Management - -Implemented `BringWindowToForegroundWithFocus()` method that: -- โœ… Uses thread attachment to bypass Windows focus restrictions -- โœ… Properly activates the target window -- โœ… Verifies the window actually received focus -- โœ… Handles edge cases and errors gracefully - -### Complete Implementation - -**File**: `FlowVision/lib/Plugins/KeyboardPlugin.cs` - -```csharp -[Description("Send keyboard input to a specific window by handle")] -public async Task SendKeyToWindow(string windowHandleString, string keyCombo) -{ - IntPtr windowHandle = new IntPtr(Convert.ToInt32(windowHandleString)); - - // Bring window to foreground with proper focus - if (!BringWindowToForegroundWithFocus(windowHandle)) - { - return false; - } - - // Wait for window to become active - await Task.Delay(200); - - // Send the keys - now going to correct window! - SendKeys.SendWait(keyCombo); - - return true; -} - -private bool BringWindowToForegroundWithFocus(IntPtr hWnd) -{ - // Get current and target thread IDs - uint currentThreadId = GetCurrentThreadId(); - IntPtr currentForeground = GetForegroundWindow(); - uint foregroundThreadId = GetWindowThreadProcessId(currentForeground, out _); - - // Attach to foreground thread to bypass restrictions - AttachThreadInput(currentThreadId, foregroundThreadId, true); - - // Set foreground window - bool success = SetForegroundWindow(hWnd); - - // Detach threads - AttachThreadInput(currentThreadId, foregroundThreadId, false); - - // Verify success - return GetForegroundWindow() == hWnd; -} -``` - -## API Changes - -### New Methods (Window-Specific) - -| Method | Parameters | Description | -|--------|------------|-------------| -| `SendKeyToWindow` | `windowHandle`, `keyCombo` | Send any key combination to specific window | -| `EnterKeyToWindow` | `windowHandle` | Send Enter key to specific window | -| `CtrlKeyToWindow` | `windowHandle`, `letter` | Send Ctrl+Letter to specific window | - -### Existing Methods (Still Available) - -| Method | Parameters | Description | -|--------|------------|-------------| -| `SendKey` | `keyCombo` | Send keys to current foreground window | -| `EnterKey` | (none) | Send Enter to current foreground window | -| `CtrlKey` | `letter` | Send Ctrl+Letter to current foreground window | - -## Usage Examples - -### Before (Unreliable) - -```csharp -// โŒ Problem: Keys go to whatever window has focus -await keyboardPlugin.SendKey("^t"); // Ctrl+T might go to wrong app -await keyboardPlugin.EnterKey(); // Enter might go to wrong window -``` - -### After (Reliable) - -```csharp -// โœ… Solution: Explicitly target the window -string chromeHandle = "12345678"; // Get from WindowSelectionPlugin - -await keyboardPlugin.SendKeyToWindow(chromeHandle, "^t"); // Ctrl+T to Chrome -await keyboardPlugin.EnterKeyToWindow(chromeHandle); // Enter to Chrome -await keyboardPlugin.CtrlKeyToWindow(chromeHandle, "w"); // Ctrl+W to Chrome -``` - -### Complete Workflow Example - -```csharp -// 1. List available windows -var windowList = windowSelectionPlugin.ListWindowHandles(); -// Output: Handle: 12345678, Title: Chrome, Process: chrome - -// 2. Select the target window -string chromeHandle = "12345678"; - -// 3. Bring to foreground explicitly (optional, SendKeyToWindow does this) -await windowSelectionPlugin.ForegroundSelect(chromeHandle); - -// 4. Send keyboard commands to that specific window -await keyboardPlugin.SendKeyToWindow(chromeHandle, "^t"); // New tab -await Task.Delay(100); -await keyboardPlugin.SendKeyToWindow(chromeHandle, "youtube"); -await keyboardPlugin.EnterKeyToWindow(chromeHandle); - -// 5. Take screenshot to verify -var screenshot = await screenCapturePlugin.CaptureScreen(chromeHandle); -``` - -## Technical Details - -### Thread Input Attachment - -Windows prevents applications from stealing focus for security reasons. The fix uses `AttachThreadInput()` to: - -1. **Get thread IDs** of current and target windows -2. **Attach** current thread to foreground thread -3. **Set foreground** window (now allowed) -4. **Detach** threads -5. **Verify** the window actually has focus - -This technique is the recommended way to reliably set foreground windows in Windows. - -### Focus Verification - -The method verifies success by checking: -```csharp -IntPtr newForeground = GetForegroundWindow(); -return newForeground == hWnd; // True if successful -``` - -### Timing - -- **200ms delay** after bringing window to foreground -- Allows Windows to process focus change -- Ensures keys arrive after window is active - -## AI Integration - -The AI can now use window-specific keyboard commands in workflows: - -### Example: Open YouTube in Chrome - -``` -1. List windows and find Chrome handle -2. SendKeyToWindow(chromeHandle, "^t") - Open new tab -3. SendKeyToWindow(chromeHandle, "youtube.com") -4. EnterKeyToWindow(chromeHandle) - Navigate -5. CaptureScreen(chromeHandle) - Verify -``` - -### Example: Switch Browser Tabs - -``` -1. Get browser window handle -2. CtrlKeyToWindow(browserHandle, "{TAB}") - Next tab -3. Or CtrlKeyToWindow(browserHandle, "+{TAB}") - Previous tab -``` - -### Example: Type in Specific Application - -``` -1. Get application window handle -2. SendKeyToWindow(appHandle, "Hello World") -3. EnterKeyToWindow(appHandle) -``` - -## Benefits - -### โœ… Reliability -- Keys always go to intended window -- No more wrong-window mistakes -- Predictable behavior - -### โœ… Consistency -- Matches MousePlugin's window-targeting approach -- Unified API across plugins -- Same handle used for mouse, keyboard, and screen capture - -### โœ… Robustness -- Handles focus restrictions properly -- Verifies focus before sending keys -- Error logging and recovery - -### โœ… Flexibility -- Old methods still work for simple cases -- New methods for precision control -- Choose based on use case - -## Testing - -### Test Case 1: New Tab in Chrome -```csharp -var handle = GetChromeHandle(); -var success = await keyboard.SendKeyToWindow(handle, "^t"); -Assert.IsTrue(success); -// Verify: New tab opened in Chrome, not other apps -``` - -### Test Case 2: Multiple Windows -```csharp -var chrome = GetChromeHandle(); -var firefox = GetFirefoxHandle(); - -await keyboard.SendKeyToWindow(chrome, "chrome tab"); -await keyboard.SendKeyToWindow(firefox, "firefox tab"); -// Verify: Each browser got its own text -``` - -### Test Case 3: Background Window -```csharp -var background = GetBackgroundWindowHandle(); -var success = await keyboard.SendKeyToWindow(background, "^a"); -// Verify: Window brought to foreground and received Ctrl+A -``` - -## Backward Compatibility - -### โœ… Existing Code Still Works - -Old method calls continue to work: -```csharp -await keyboard.SendKey("^t"); // Still works (current window) -await keyboard.EnterKey(); // Still works (current window) -await keyboard.CtrlKey("w"); // Still works (current window) -``` - -### Migration Path - -**Old pattern:** -```csharp -// Hope the right window has focus -await keyboard.SendKey("^t"); -``` - -**New pattern:** -```csharp -// Ensure the right window gets the keys -await keyboard.SendKeyToWindow(windowHandle, "^t"); -``` - -## Troubleshooting - -### Keys Still Going to Wrong Window - -**Possible Causes:** -1. Invalid window handle -2. Window closed/destroyed -3. Window minimized -4. Security restrictions - -**Solutions:** -- Re-query window handles regularly -- Check window still exists before sending keys -- Restore minimized windows first -- Run app with appropriate permissions - -### Focus Not Switching - -**Possible Causes:** -1. Windows security preventing focus change -2. Another app actively stealing focus -3. Window not visible/minimized - -**Solutions:** -- Check return value of `SendKeyToWindow()` -- Use `ForegroundSelect()` first if needed -- Restore/maximize window before sending keys - -### Timing Issues - -**Symptoms:** -- Keys sent but not processed -- Partial key combinations - -**Solutions:** -- Increase delay after `SendKeyToWindow()` call -- Add delays between multiple key commands -- Wait for UI updates between actions - -## Performance - -### Timing Overhead - -| Operation | Time | Notes | -|-----------|------|-------| -| Focus switch | ~200ms | Thread attach/detach + verification | -| Key send | ~50ms | SendKeys.SendWait() execution | -| Total | ~250ms | Per window-targeted key command | - -### Optimization Tips - -1. **Batch operations** on same window -2. **Reuse window handles** (don't re-query) -3. **Skip focus** if window already active -4. **Parallel operations** on different windows - -## Security Considerations - -### Thread Input Attachment - -- Uses Windows API properly -- No security bypass attempts -- Respects UAC elevation requirements -- Safe for enterprise environments - -### Focus Management - -- Only affects windows in same desktop session -- Cannot steal focus from elevated processes (by design) -- Follows Windows security model - -## Future Enhancements - -### Potential Improvements - -- [ ] Batch key commands (multiple keys, one focus switch) -- [ ] Cache focus state (skip if already focused) -- [ ] Async focus with callback -- [ ] Focus restoration (return to previous window) -- [ ] Virtual keyboard support (for games/special apps) - -### Alternative Approaches - -- **PostMessage API** - Send keys without focus (limited compatibility) -- **UI Automation** - More robust but heavier -- **SendInput API** - Lower level, more control - -## Build Status - -``` -โœ… Compilation: Success (0 errors) -โœ… New methods: 3 added -โœ… Breaking changes: None (backward compatible) -โœ… API surface: Extended, not replaced -``` - -## Files Modified - -1. **`FlowVision/lib/Plugins/KeyboardPlugin.cs`** - - Added `SendKeyToWindow()` method - - Added `EnterKeyToWindow()` method - - Added `CtrlKeyToWindow()` method - - Added `BringWindowToForegroundWithFocus()` helper - - Added Win32 API imports for focus management - - Added error logging and verification - -## Summary - -### Problem -โŒ Keyboard shortcuts sent to wrong window (whatever had focus) - -### Solution -โœ… New window-targeted methods that: -- Accept window handle as parameter -- Bring window to foreground reliably -- Verify focus before sending keys -- Log success/failure for debugging - -### Result -๐ŸŽฏ **Keyboard commands now go to the correct window every time!** - -The AI can now: -- Send Ctrl+T to specific Chrome window -- Type in specific terminal tab -- Press Enter in specific form -- Switch tabs in specific browser -- Send any key combination to any window reliably - -### Migration -๐Ÿ”„ **No breaking changes** - old methods still work, new methods available for precision control. diff --git a/LMSTUDIO_INTEGRATION.md b/LMSTUDIO_INTEGRATION.md deleted file mode 100644 index 96a984b..0000000 --- a/LMSTUDIO_INTEGRATION.md +++ /dev/null @@ -1,321 +0,0 @@ -# LM Studio Integration Guide - -## Overview - -FlowVision now supports **local AI** through LM Studio integration! This allows you to run AI models locally on your machine without requiring Azure OpenAI or other cloud services. - -## ๐ŸŽฏ What is LM Studio? - -[LM Studio](https://lmstudio.ai/) is a desktop application that lets you: -- Run open-source LLMs (Large Language Models) locally -- No internet connection required (after downloading models) -- Complete privacy - your data stays on your machine -- OpenAI-compatible API for easy integration -- Support for various models (Llama, Mistral, Phi, etc.) - -## โœ… Features - -- **Local AI Execution** - All AI processing happens on your machine -- **Privacy** - No data sent to cloud services -- **Cost Savings** - No API costs or usage limits -- **Offline Support** - Works without internet (after setup) -- **Compatible Models** - Works with any model supported by LM Studio -- **Tool Calling Support** - Full function/tool calling capabilities -- **Easy Configuration** - Simple UI for setup - -## ๐Ÿ“‹ Prerequisites - -### 1. Install LM Studio -1. Download from [https://lmstudio.ai/](https://lmstudio.ai/) -2. Install and launch LM Studio -3. Download a model (recommendations below) - -### 2. System Requirements -- **RAM**: At least 16GB (32GB+ recommended for larger models) -- **GPU**: Optional but recommended (CUDA/Metal support) -- **Storage**: Depends on model size (2GB - 50GB+) -- **OS**: Windows, macOS, or Linux - -## ๐Ÿš€ Quick Start Guide - -### Step 1: Download and Load a Model in LM Studio - -1. **Open LM Studio** -2. **Go to the "Discover" or "Models" tab** -3. **Download a recommended model**: - - **For general use**: `TheBloke/Mistral-7B-Instruct-v0.2-GGUF` - - **For coding**: `TheBloke/deepseek-coder-6.7b-instruct-GGUF` - - **Lightweight**: `TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF` - - **Advanced**: `TheBloke/Llama-2-13B-chat-GGUF` - -4. **Load the model**: - - Click on the downloaded model - - Select a quantization level (Q4_K_M is a good balance) - - Click "Load Model" - -### Step 2: Start the LM Studio Server - -1. **In LM Studio, go to the "Local Server" tab** -2. **Click "Start Server"** -3. **Verify the server is running** (default: `http://localhost:1234`) -4. **Keep LM Studio running** while using FlowVision - -### Step 3: Configure FlowVision - -1. **Open FlowVision** -2. **Go to Settings โ†’ LM Studio Configuration** (or create menu option) -3. **Configure settings**: - - โœ… Check "Enable LM Studio" - - Endpoint: `http://localhost:1234/v1` (default) - - Model Name: `local-model` (or as shown in LM Studio) - - Temperature: `0.7` (adjust as needed) - - Max Tokens: `2048` (or higher for longer responses) - -4. **Click "Test Connection"** to verify -5. **Click "Save"** - -### Step 4: Test It! - -1. **Ask FlowVision a question** -2. **The request will now go to LM Studio instead of Azure** -3. **You'll see "Local AI response" in the loading indicator** - -## ๐Ÿ”ง Configuration Options - -### LMStudioConfig Settings - -```csharp -// Configuration file location -%APPDATA%\FlowVision\lmstudioconfig.json -``` - -| Setting | Description | Default Value | -|---------|-------------|---------------| -| `Enabled` | Use LM Studio instead of Azure | `false` | -| `EndpointURL` | LM Studio server endpoint | `http://localhost:1234/v1` | -| `ModelName` | Model identifier | `local-model` | -| `APIKey` | Placeholder (not needed) | `lm-studio` | -| `Temperature` | Response randomness (0-2) | `0.7` | -| `MaxTokens` | Maximum response length | `2048` | -| `TimeoutSeconds` | Request timeout | `300` | - -## ๐Ÿ“Š Recommended Models by Use Case - -### For Tool Calling (Our Primary Use Case) -``` -Model: TheBloke/Hermes-2-Pro-Mistral-7B-GGUF -Size: ~4GB (Q4_K_M) -Why: Excellent at function calling and following instructions -``` - -### For Fast Responses -``` -Model: TheBloke/Phi-2-GGUF -Size: ~1.6GB (Q4_K_M) -Why: Very fast, good for quick commands -``` - -### For Best Quality -``` -Model: TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF -Size: ~26GB (Q4_K_M) -Why: Highest quality, great reasoning -Requirements: 32GB+ RAM -``` - -### For Coding Tasks -``` -Model: TheBloke/deepseek-coder-6.7b-instruct-GGUF -Size: ~3.8GB (Q4_K_M) -Why: Optimized for code generation -``` - -## ๐Ÿ› ๏ธ Programmatic Usage - -### Creating LMStudioActioner - -```csharp -// LM Studio actioner is automatically used when enabled -var actioner = new Actioner(outputHandler); - -// Check if LM Studio is enabled -var lmConfig = LMStudioConfig.LoadConfig(); -if (lmConfig.Enabled) -{ - Console.WriteLine("Using LM Studio for local AI"); -} -``` - -### Manual Configuration - -```csharp -// Load config -var config = LMStudioConfig.LoadConfig(); - -// Modify settings -config.Enabled = true; -config.EndpointURL = "http://localhost:1234/v1"; -config.ModelName = "local-model"; -config.Temperature = 0.7; - -// Save config -config.SaveConfig(); -``` - -### Priority Order - -When you make a request, FlowVision checks in this order: - -1. **LM Studio** (if `LMStudioConfig.Enabled = true`) -2. **Multi-Agent Mode** (if enabled) -3. **Azure OpenAI** (default fallback) - -## ๐Ÿ› Troubleshooting - -### "Cannot connect to LM Studio" - -**Solutions**: -1. โœ… Make sure LM Studio is running -2. โœ… Verify a model is loaded -3. โœ… Click "Start Server" in LM Studio -4. โœ… Check endpoint URL: `http://localhost:1234/v1` -5. โœ… Disable firewall/antivirus temporarily -6. โœ… Try restarting LM Studio - -### "Model not responding" or slow responses - -**Solutions**: -1. โœ… Check RAM usage (models need sufficient memory) -2. โœ… Try a smaller/faster model -3. โœ… Increase `TimeoutSeconds` in config -4. โœ… Enable GPU acceleration in LM Studio settings -5. โœ… Close other applications to free up resources - -### "Tool calling not working" - -**Solutions**: -1. โœ… Use a model that supports function calling -2. โœ… Try: `Hermes-2-Pro-Mistral-7B` or similar -3. โœ… Ensure `AutoInvokeKernelFunctions` is enabled in ToolConfig -4. โœ… Check LM Studio logs for errors - -### "Connection timeout" - -**Solutions**: -1. โœ… Increase `TimeoutSeconds` (default 300) -2. โœ… First request is slower (model loading) -3. โœ… Reduce `MaxTokens` for faster responses -4. โœ… Consider a smaller/faster model - -## ๐Ÿ” Privacy & Security - -### Benefits -- โœ… **Complete Privacy** - No data sent to cloud -- โœ… **Offline Operation** - Works without internet -- โœ… **No API Costs** - Free to use -- โœ… **Full Control** - You control the model and data - -### Considerations -- โš ๏ธ **Local Processing** - Uses your computer resources -- โš ๏ธ **Model Storage** - Models can be large (1GB - 50GB+) -- โš ๏ธ **Performance** - Depends on your hardware - -## ๐Ÿ“ˆ Performance Tips - -### 1. Choose the Right Model Size -- **Lightweight (1-3GB)**: Fast responses, good for simple tasks -- **Medium (4-7GB)**: Balanced performance and quality -- **Large (8GB+)**: Best quality, requires powerful hardware - -### 2. Optimize Settings -``` -Temperature: 0.2-0.5 (More focused responses) -MaxTokens: 1024-2048 (Faster generation) -``` - -### 3. Hardware Recommendations -- **Minimum**: 16GB RAM, CPU only -- **Recommended**: 32GB RAM, GPU (NVIDIA/AMD/Apple Silicon) -- **Optimal**: 64GB RAM, High-end GPU - -### 4. Use GPU Acceleration -In LM Studio settings: -- Enable GPU acceleration -- Allocate sufficient GPU layers -- Monitor GPU usage - -## ๐Ÿ”„ Switching Between Local and Cloud AI - -### Enable Local AI -```csharp -var config = LMStudioConfig.LoadConfig(); -config.Enabled = true; -config.SaveConfig(); -``` - -### Disable Local AI (Use Azure) -```csharp -var config = LMStudioConfig.LoadConfig(); -config.Enabled = false; -config.SaveConfig(); -``` - -### Dynamic Switching -FlowVision automatically uses LM Studio when enabled, and falls back to Azure when disabled. No need to restart the application! - -## ๐Ÿ“š Additional Resources - -### LM Studio -- Website: https://lmstudio.ai/ -- Documentation: https://lmstudio.ai/docs -- Discord: https://discord.gg/lmstudio - -### Models -- Hugging Face: https://huggingface.co/models -- TheBloke's Models: https://huggingface.co/TheBloke -- Model Comparison: https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard - -### OpenAI-Compatible API -- LM Studio uses OpenAI's API format -- Compatible with OpenAI SDK -- Supports most OpenAI features - -## ๐ŸŽ‰ Benefits Summary - -| Feature | Azure OpenAI | LM Studio Local | -|---------|--------------|-----------------| -| **Privacy** | Cloud-based | โœ… 100% Local | -| **Cost** | Pay per use | โœ… Free | -| **Internet** | Required | โœ… Not needed | -| **Speed** | Fast | Depends on hardware | -| **Quality** | Excellent | Good to Excellent | -| **Tool Calling** | โœ… Full support | โœ… Supported | -| **Setup** | Easy | Moderate | - -## ๐Ÿ“ Example Configuration File - -```json -{ - "EndpointURL": "http://localhost:1234/v1", - "ModelName": "local-model", - "APIKey": "lm-studio", - "Enabled": true, - "Temperature": 0.7, - "MaxTokens": 2048, - "TimeoutSeconds": 300 -} -``` - -## ๐Ÿš€ Next Steps - -1. **Download LM Studio** and experiment with different models -2. **Test with FlowVision** using simple commands first -3. **Try tool calling** features (CMD, PowerShell, etc.) -4. **Optimize settings** for your use case -5. **Consider hybrid approach**: Local for privacy-sensitive tasks, Cloud for performance - ---- - -**Happy Local AI Usage!** ๐ŸŽ‰ - -For support, check the [Troubleshooting](#-troubleshooting) section or open an issue on GitHub. diff --git a/LOCAL_OMNIPARSER_INTEGRATION.md b/LOCAL_OMNIPARSER_INTEGRATION.md deleted file mode 100644 index c91d38c..0000000 --- a/LOCAL_OMNIPARSER_INTEGRATION.md +++ /dev/null @@ -1,653 +0,0 @@ -# Local OmniParser Auto-Management Integration ๐Ÿš€ - -## ๐ŸŽฏ What This Does - -FlowVision now **automatically manages your local OmniParser server**! No manual server startup, no configuration hassles - it just works! - -### Before vs After - -**Before:** -- โŒ Start Python server manually -- โŒ Keep terminal window open -- โŒ Configure server URL -- โŒ Restart if it crashes -- โŒ Check if server is running - -**After:** -- โœ… **Automatic server startup** -- โœ… **Auto-detection and health checking** -- โœ… **Auto-restart on failure** -- โœ… **Zero manual management** -- โœ… **Just use it!** ๐ŸŽŠ - -## ๐Ÿ—๏ธ Architecture - -### LocalOmniParserManager - -**What:** Smart server manager that handles everything -**Location:** `FlowVision\lib\Classes\LocalOmniParserManager.cs` - -**Features:** -1. **Auto-Detection** - Checks if server is running -2. **Auto-Start** - Starts server when needed -3. **Health Monitoring** - Verifies server responds -4. **Process Management** - Handles server lifecycle -5. **Cooldown Logic** - Prevents rapid restart loops -6. **Diagnostics** - Detailed status information - -### Integration Points - -**ScreenCaptureOmniParserPlugin:** -```csharp -// Before capturing screen -await LocalOmniParserManager.EnsureServerRunningAsync(); - -// Server is now guaranteed to be running -// Process the screenshot -var result = await omniClient.ProcessScreenshotAsync(image); -``` - -**Automatic Flow:** -``` -User: "Capture screen" - โ†“ -FlowVision checks: Is server running? - โ†“ No -FlowVision: Starting server... - โ†“ Wait for ready (~10-20 seconds) -FlowVision: Server ready! โœ“ - โ†“ -FlowVision: Process screenshot - โ†“ -Return: Parsed UI elements -``` - -## ๐Ÿ“‹ Requirements - -### 1. OmniParser Installation - -**Required Location:** `T:\OmniParser` - -**Directory Structure:** -``` -T:\OmniParser\ -โ”œโ”€โ”€ omnitool\ -โ”‚ โ””โ”€โ”€ omniparserserver\ -โ”‚ โ””โ”€โ”€ omniparserserver.py โ† Server script -โ”œโ”€โ”€ weights\ -โ”‚ โ”œโ”€โ”€ icon_detect\ -โ”‚ โ”‚ โ””โ”€โ”€ model.pt โ† Detection model -โ”‚ โ””โ”€โ”€ icon_caption_florence\ โ† Caption model -โ”œโ”€โ”€ util\ -โ”‚ โ””โ”€โ”€ omniparser.py โ† Core logic -โ””โ”€โ”€ requirements.txt -``` - -### 2. Python Environment - -**Options:** - -**Option A: Conda (Recommended)** -```bash -cd T:\OmniParser -conda create -n omni python==3.12 -conda activate omni -pip install -r requirements.txt -``` - -**Option B: System Python** -```bash -cd T:\OmniParser -python -m pip install -r requirements.txt -``` - -### 3. Model Weights - -**Download from HuggingFace:** -```bash -cd T:\OmniParser -huggingface-cli download microsoft/OmniParser-v2.0 \ - "icon_detect/model.pt" \ - "icon_caption/model.safetensors" \ - --local-dir weights -``` - -**Or:** Follow OmniParser setup guide: https://github.com/microsoft/OmniParser - -## ๐Ÿš€ Usage - -### Basic Usage (Zero Config) - -**Just use FlowVision - it handles everything!** - -``` -You: "Capture and parse the screen" -AI: [Checks server...] -AI: [Server not running, starting...] -AI: [Server ready in 15 seconds] -AI: [Processing screenshot...] -AI: "I can see these UI elements: [list]" -``` - -**Subsequent Calls:** -``` -You: "Capture again" -AI: [Checks server - already running!] -AI: [Processing screenshot - instant!] -AI: "Elements: [list]" -``` - -### First Run Experience - -**First time using OmniParser:** - -1. FlowVision detects no server running -2. Starts the Python server automatically -3. Loads models (~10-20 seconds) -4. Verifies server is responding -5. Processes your screenshot -6. Done! โœ“ - -**Server stays running** - future captures are instant! - -### Manual Server Control (Optional) - -**Check Status:** -```csharp -bool running = await LocalOmniParserManager.IsServerRunningAsync(); -``` - -**Get Diagnostics:** -```csharp -string info = LocalOmniParserManager.GetDiagnostics(); -``` - -**Stop Server:** -```csharp -LocalOmniParserManager.StopServer(); -``` - -**Configure Paths:** -```csharp -LocalOmniParserManager.Configure( - omniParserPath: @"T:\OmniParser", - pythonExe: "python", - serverUrl: "http://127.0.0.1:8080", - port: 8080 -); -``` - -## ๐ŸŽญ Smart Features - -### 1. Health Checking - -**What:** Verifies server is actually responding, not just running - -**How:** -```csharp -// Checks /probe/ endpoint -var response = await httpClient.GetAsync("http://127.0.0.1:8080/probe/"); -if (response.IsSuccessStatusCode) { - // Server is healthy! -} -``` - -**Why:** Process might be running but not responding (crashed, loading, etc.) - -### 2. Auto-Startup - -**What:** Automatically starts server when needed - -**Process:** -1. Check if server responding -2. If not, verify installation exists -3. Start Python process with proper arguments -4. Wait for server to respond (up to 30 seconds) -5. Verify health with /probe/ -6. Ready! - -**Arguments Used:** -```bash -python -m omniparserserver \ - --som_model_path "T:\OmniParser\weights\icon_detect\model.pt" \ - --caption_model_name florence2 \ - --caption_model_path "T:\OmniParser\weights\icon_caption_florence" \ - --device cpu \ - --BOX_TRESHOLD 0.05 \ - --port 8080 \ - --host 0.0.0.0 -``` - -### 3. Cooldown Protection - -**What:** Prevents rapid restart loops if server keeps failing - -**Logic:** -- First failure: Try to start -- Fails again within 10 seconds: Wait -- After cooldown: Try again - -**Why:** Prevents hammering the system if there's a config issue - -### 4. Output Capture - -**What:** Captures server logs for debugging - -**Where:** Visible in FlowVision logs as "OmniParser-Server" entries - -**Example:** -``` -[OmniParser-Server] Output: Loading YOLO model... -[OmniParser-Server] Output: Model loaded successfully -[OmniParser-Server] Output: Server started on port 8080 -``` - -### 5. Process Lifecycle - -**What:** Proper process management - -**Features:** -- Hidden window (no console popup) -- Output redirection (for logging) -- Graceful shutdown (when FlowVision closes) -- Automatic cleanup - -## ๐Ÿ“Š Performance - -### Timing Breakdown - -**First Request (Cold Start):** -``` -Check server: ~0.5s - โ†“ (not running) -Start process: ~1s -Load models: ~10-15s -Verify health: ~0.5s -Process image: ~2-3s -โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -Total: ~15-20 seconds -``` - -**Subsequent Requests (Warm):** -``` -Check server: ~0.1s - โ†“ (already running) -Process image: ~2-3s -โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -Total: ~2-3 seconds -``` - -### Optimization Tips - -**1. Use GPU (if available):** -```csharp -// Modify startup arguments to use CUDA -LocalOmniParserManager.Configure(device: "cuda"); -``` -Result: ~1 second per screenshot (vs 2-3 on CPU) - -**2. Keep Server Running:** -- Don't stop server between requests -- Manager keeps it running automatically -- Restart only if needed - -**3. Batch Processing:** -- Server handles multiple requests -- No per-request overhead -- Process many screenshots efficiently - -## ๐Ÿ› Troubleshooting - -### Issue: "OmniParser server not available" - -**Cause:** Server failed to start - -**Solutions:** - -1. **Check Installation:** -``` -Verify: T:\OmniParser\omnitool\omniparserserver\omniparserserver.py exists -Verify: T:\OmniParser\weights\ folder exists with models -``` - -2. **Check Python:** -```bash -python --version # Should be 3.12 or similar -python -c "import torch; import transformers" # Check dependencies -``` - -3. **Get Diagnostics:** -```csharp -string info = LocalOmniParserManager.GetDiagnostics(); -// Shows detailed status -``` - -4. **Check Logs:** -Look for "OmniParser-Server" entries in FlowVision logs for error messages - -### Issue: "Server starting but timing out" - -**Cause:** Models taking too long to load - -**Solutions:** -1. Increase wait timeout (default 30 seconds) -2. Use smaller/faster models -3. Use GPU instead of CPU -4. Check system resources (RAM, disk) - -### Issue: "Server crashes after starting" - -**Cause:** Dependency or configuration issue - -**Solutions:** -1. Check Python dependencies: -```bash -cd T:\OmniParser -pip install -r requirements.txt -``` - -2. Verify model files: -```bash -# Should exist and be valid -T:\OmniParser\weights\icon_detect\model.pt -T:\OmniParser\weights\icon_caption_florence\* -``` - -3. Check logs for Python errors - -### Issue: "Port 8080 already in use" - -**Cause:** Another service using port 8080 - -**Solutions:** -1. Stop other service on port 8080 -2. Or configure different port: -```csharp -LocalOmniParserManager.Configure(port: 8081); -``` - -### Issue: "Models not found" - -**Cause:** Weights not downloaded or wrong location - -**Solutions:** -1. Download models: -```bash -cd T:\OmniParser -huggingface-cli download microsoft/OmniParser-v2.0 --local-dir weights -``` - -2. Verify paths match: -``` -weights\icon_detect\model.pt -weights\icon_caption_florence\* -``` - -## ๐Ÿ”ง Advanced Configuration - -### Custom Installation Path - -**If OmniParser is elsewhere:** -```csharp -LocalOmniParserManager.Configure( - omniParserPath: @"C:\MyCustomPath\OmniParser" -); -``` - -### Custom Python Executable - -**If using specific Python:** -```csharp -LocalOmniParserManager.Configure( - pythonExe: @"C:\Python312\python.exe" -); -``` - -### Using Conda Environment - -**If using conda:** -```csharp -LocalOmniParserManager.Configure( - pythonExe: @"C:\Users\YourName\miniconda3\envs\omni\python.exe" -); -``` - -Or activate conda first, then Python will use that environment automatically. - -### Custom Port - -**If port conflict:** -```csharp -LocalOmniParserManager.Configure( - port: 8081 -); -``` - -### Using GPU - -**Modify server startup** (in LocalOmniParserManager.cs): -```csharp -// Change from: -$"--device cpu " + - -// To: -$"--device cuda " + -``` - -Then rebuild FlowVision. - -## ๐Ÿ“– API Reference - -### LocalOmniParserManager - -**Static Methods:** - -```csharp -// Check if server is running and responding -Task IsServerRunningAsync() - -// Ensure server is running (auto-start if needed) -Task EnsureServerRunningAsync() - -// Stop the server -void StopServer() - -// Configure paths and settings -void Configure( - string omniParserPath = null, - string pythonExe = null, - string serverUrl = null, - int? port = null -) - -// Get current server URL -string GetServerUrl() - -// Check if OmniParser is installed -bool IsInstalled() - -// Get detailed diagnostics -string GetDiagnostics() -``` - -**Usage Examples:** - -```csharp -// Basic usage -await LocalOmniParserManager.EnsureServerRunningAsync(); - -// Check status -bool running = await LocalOmniParserManager.IsServerRunningAsync(); -if (!running) { - // Server not available -} - -// Get diagnostics -string info = LocalOmniParserManager.GetDiagnostics(); -Console.WriteLine(info); - -// Custom configuration -LocalOmniParserManager.Configure( - omniParserPath: @"D:\OmniParser", - pythonExe: "python3", - port: 8081 -); -``` - -## ๐Ÿ“š Implementation Details - -### Server Lifecycle - -**States:** -1. **Not Installed** - OmniParser files missing -2. **Installed but Stopped** - Ready to start -3. **Starting** - Process launching, models loading -4. **Running** - Server responding to requests -5. **Failed** - Started but not responding - -### Process Management - -**Features:** -- Hidden console window (no popup) -- Output/error stream capture -- Proper disposal on exit -- Zombie process prevention - -**Implementation:** -```csharp -var startInfo = new ProcessStartInfo -{ - FileName = _pythonExecutable, - Arguments = arguments, - WorkingDirectory = workingDir, - UseShellExecute = false, - CreateNoWindow = true, // No popup! - RedirectStandardOutput = true, - RedirectStandardError = true -}; -``` - -### Health Monitoring - -**Probe Endpoint:** -``` -GET http://127.0.0.1:8080/probe/ -Response: {"message": "Omniparser API ready"} -``` - -**Health Check:** -- Sends GET to /probe/ -- Expects 200 OK -- Timeout: 2 seconds -- Returns: True/False - -### Error Handling - -**Graceful Degradation:** -1. Try to start server -2. If fails, show detailed error -3. Include diagnostics -4. Suggest solutions - -**User Experience:** -- Clear error messages -- Actionable suggestions -- Diagnostic information -- No silent failures - -## ๐ŸŽ Benefits - -### For Users - -1. **Zero Manual Management** - - No server startup - - No configuration - - Just works! - -2. **Automatic Recovery** - - Detects failures - - Restarts automatically - - Maintains availability - -3. **Clear Feedback** - - Status messages - - Progress indicators - - Error diagnostics - -### For Developers - -1. **Simple Integration** - ```csharp - await LocalOmniParserManager.EnsureServerRunningAsync(); - // Server is now running! - ``` - -2. **Robust Error Handling** - - Detailed diagnostics - - Proper exceptions - - Clear messages - -3. **Maintainable** - - Single responsibility - - Clean interface - - Well documented - -## ๐Ÿ“Š Status Messages - -**What You'll See:** - -``` -Starting: -"Checking local OmniParser server..." -"Starting local OmniParser server..." -"Server started successfully in 15 seconds" - -Running: -"Server already running" -"Processing screenshot..." -"Done!" - -Errors: -"Failed to start server: [reason]" -"OmniParser not found at T:\OmniParser" -"Check installation and try again" -``` - -## ๐ŸŽฏ Summary - -### What You Get - -โœ… **Automatic server management** - no manual startup -โœ… **Health monitoring** - ensures server is responding -โœ… **Auto-restart** - handles failures gracefully -โœ… **Smart cooldown** - prevents restart loops -โœ… **Process lifecycle** - proper cleanup -โœ… **Detailed diagnostics** - easy troubleshooting -โœ… **Zero configuration** - works with defaults -โœ… **Local processing** - fast and private - -### What You Need - -๐Ÿ“‹ OmniParser installed at `T:\OmniParser` -๐Ÿ“‹ Python 3.12+ with dependencies -๐Ÿ“‹ Model weights downloaded -๐Ÿ“‹ Port 8080 available (or configure different) - -### How It Works - -1. You request screen capture -2. FlowVision checks if server running -3. If not, starts it automatically -4. Waits for server to be ready -5. Processes your screenshot -6. Returns parsed UI elements -7. Server stays running for next request - -**It just works!** ๐ŸŽŠ - ---- - -**Status:** โœ… Implemented and Ready -**Build:** โœ… Successful -**Testing:** Ready for use -**Documentation:** Complete diff --git a/MARKDOWN_NULL_REFERENCE_FIX.md b/MARKDOWN_NULL_REFERENCE_FIX.md deleted file mode 100644 index d8d064b..0000000 --- a/MARKDOWN_NULL_REFERENCE_FIX.md +++ /dev/null @@ -1,269 +0,0 @@ -# MarkdownHelper NullReferenceException Fix - -## Date: October 2, 2025 - -## Issue - -``` -System.NullReferenceException -Message: Object reference not set to an instance of an object. -Source: FlowVision -StackTrace: at FlowVision.lib.Classes.MarkdownHelper.ApplyInlineCodeFormatting - in MarkdownHelper.cs:line 124 -``` - -## Root Cause - -The `MarkdownHelper.ApplyInlineCodeFormatting()` method (and similar methods) was accessing `richTextBox.SelectionFont.Size` without checking if `SelectionFont` was null. - -### Why SelectionFont Can Be Null - -`RichTextBox.SelectionFont` returns `null` when: -1. **No text is selected** (SelectionLength = 0) -2. **Selected text has multiple fonts** (mixed formatting) -3. **Text hasn't been initialized** with a font yet - -### The Problematic Code - -```csharp -// Line 124 - CRASH! SelectionFont could be null -richTextBox.SelectionFont = new Font("Consolas", richTextBox.SelectionFont.Size); - ^^^^^^^^^^^^^^^^^^^^^^^^^ NULL! -``` - -## Solution - -Used the **null-conditional operator** (`?.`) and **null-coalescing operator** (`??`) to provide a safe default: - -```csharp -// Safe: If SelectionFont is null, use default size of 10F -float fontSize = richTextBox.SelectionFont?.Size ?? 10F; -richTextBox.SelectionFont = new Font("Consolas", fontSize); -``` - -## Changes Made - -### 1. ApplyInlineCodeFormatting() - Line 124 - -**Before**: -```csharp -richTextBox.SelectionFont = new Font("Consolas", richTextBox.SelectionFont.Size); -``` - -**After**: -```csharp -// Get current font size, defaulting to 10 if null -float fontSize = richTextBox.SelectionFont?.Size ?? 10F; -richTextBox.SelectionFont = new Font("Consolas", fontSize); -``` - -### 2. ApplyBoldFormatting() - Line 84 - -**Before**: -```csharp -richTextBox.SelectionFont = new Font(richTextBox.SelectionFont, FontStyle.Bold); -``` - -**After**: -```csharp -// Safely get current font or use default -Font currentFont = richTextBox.SelectionFont ?? new Font("Segoe UI", 10F); -richTextBox.SelectionFont = new Font(currentFont, FontStyle.Bold); -``` - -### 3. ApplyItalicsFormatting() - Line 104 - -**Before**: -```csharp -richTextBox.SelectionFont = new Font(richTextBox.SelectionFont, FontStyle.Italic); -``` - -**After**: -```csharp -// Safely get current font or use default -Font currentFont = richTextBox.SelectionFont ?? new Font("Segoe UI", 10F); -richTextBox.SelectionFont = new Font(currentFont, FontStyle.Italic); -``` - -### 4. ApplyCodeBlockFormatting() - Line 144 - -**Before**: -```csharp -richTextBox.SelectionFont = new Font("Consolas", richTextBox.SelectionFont.Size); -``` - -**After**: -```csharp -// Get current font size, defaulting to 10 if null -float fontSize = richTextBox.SelectionFont?.Size ?? 10F; -richTextBox.SelectionFont = new Font("Consolas", fontSize); -``` - -## Technical Details - -### Null-Conditional Operator (?.) -```csharp -richTextBox.SelectionFont?.Size -// If SelectionFont is null, returns null (not crash!) -// If SelectionFont is not null, returns Size -``` - -### Null-Coalescing Operator (??) -```csharp -richTextBox.SelectionFont?.Size ?? 10F -// If left side is null, use right side (10F) -// If left side is not null, use left side value -``` - -### Combined -```csharp -float fontSize = richTextBox.SelectionFont?.Size ?? 10F; -// Safely gets font size, defaulting to 10F if SelectionFont is null -``` - -## Why This Fixes the Crash - -### Before (Crash): -```csharp -richTextBox.SelectionFont.Size -// If SelectionFont is null โ†’ NullReferenceException! -``` - -### After (Safe): -```csharp -richTextBox.SelectionFont?.Size ?? 10F -// If SelectionFont is null โ†’ returns 10F (safe default) -// If SelectionFont is not null โ†’ returns actual Size -``` - -## Testing Scenarios - -### Scenario 1: Normal Text with Markdown -``` -Input: "This is `code` text" -Result: โœ… Code formatted with Consolas font -``` - -### Scenario 2: Empty RichTextBox -``` -Input: "" -Result: โœ… No crash, safely handles null font -``` - -### Scenario 3: Mixed Formatting -``` -Input: Text with multiple fonts mixed -Result: โœ… Uses default 10F when font is ambiguous -``` - -### Scenario 4: Uninitialized Font -``` -Input: New RichTextBox with no font set -Result: โœ… Uses default Segoe UI 10F -``` - -## Impact - -### Before Fix: -- โŒ Random crashes when formatting markdown -- โŒ Especially when RichTextBox has mixed or no formatting -- โŒ Unreliable markdown rendering - -### After Fix: -- โœ… No more NullReferenceException -- โœ… Graceful fallback to sensible defaults -- โœ… Reliable markdown formatting in all scenarios - -## File Modified - -**Path**: `FlowVision/lib/Classes/MarkdownHelper.cs` - -**Changes**: -- Line ~84: ApplyBoldFormatting - null-safe font handling -- Line ~104: ApplyItalicsFormatting - null-safe font handling -- Line ~124: ApplyInlineCodeFormatting - null-safe font size -- Line ~144: ApplyCodeBlockFormatting - null-safe font size - -## Build Status - -``` -โœ… Compilation: Success (0 errors, 0 warnings) -โœ… All null references fixed -โœ… Safe defaults implemented -โœ… Ready for production -``` - -## Best Practices Applied - -### Defensive Programming -Always check for null before dereferencing: -```csharp -// Bad -var size = richTextBox.SelectionFont.Size; - -// Good -var size = richTextBox.SelectionFont?.Size ?? 10F; -``` - -### Sensible Defaults -When null, use reasonable fallback values: -```csharp -// Default font for general text -new Font("Segoe UI", 10F) - -// Default font for code -new Font("Consolas", 10F) -``` - -### Consistent Pattern -Applied the same fix to all similar methods: -- ApplyBoldFormatting -- ApplyItalicsFormatting -- ApplyInlineCodeFormatting -- ApplyCodeBlockFormatting - -## Additional Safety - -The fix also prevents potential future crashes in: -- Text with no formatting -- Empty selections -- Newly created RichTextBox controls -- Any scenario where SelectionFont might be null - -## Prevention - -To prevent similar issues in the future: - -### 1. Always Check Nullable Properties -```csharp -// Properties that can be null: -- RichTextBox.SelectionFont -- RichTextBox.Font (if not set) -- Any reference type property -``` - -### 2. Use Null-Conditional Operators -```csharp -// Modern C# way -object?.Property ?? default -``` - -### 3. Provide Sensible Defaults -```csharp -// Don't assume values exist -// Always have a fallback -``` - -## Related Issues - -This fix prevents crashes in: -- Markdown formatting in chat messages -- System messages rendering -- AI responses with code blocks -- Any UI text with markdown syntax - -## Conclusion - -A simple but critical fix that prevents random crashes when rendering markdown in the UI. The application is now much more robust when handling various text formatting scenarios. - -The key lesson: **Always check for null before accessing properties on reference types**, especially with UI controls like RichTextBox where font information can be null or inconsistent. diff --git a/MIGRATION_COMPLETE.md b/MIGRATION_COMPLETE.md deleted file mode 100644 index 44aab5d..0000000 --- a/MIGRATION_COMPLETE.md +++ /dev/null @@ -1,224 +0,0 @@ -# โœ… Migration Complete: Semantic Kernel โ†’ Microsoft Agent Framework - -## ๐ŸŽ‰ SUCCESS! - -The Recursive Control project has been successfully migrated from **Semantic Kernel** to the **Microsoft Agent Framework** (Microsoft.Extensions.AI). - -**Build Status:** โœ… **SUCCESSFUL** (with only pre-existing warnings) - ---- - -## ๐Ÿ“Š Migration Summary - -### What Changed - -| Component | Before (SK) | After (Extensions.AI) | -|-----------|-------------|----------------------| -| **Main Interface** | `IChatCompletionService` | `IChatClient` | -| **Chat History** | `ChatHistory` | `List` | -| **Configuration** | `OpenAIPromptExecutionSettings` | `ChatOptions` | -| **Tool Functions** | `[KernelFunction]` attribute | `[Description]` attribute | -| **Client Creation** | `Kernel.CreateBuilder()` | `AzureOpenAIClient.GetChatClient()` | -| **Function Invocation** | `ToolCallBehavior.AutoInvoke...` | `ChatClientBuilder().UseFunctionInvocation()` | -| **Streaming** | `GetStreamingChatMessageContentsAsync` | `GetStreamingResponseAsync` | - -### Files Modified - -โœ… **9 Plugin Files** - Removed SK dependencies, now use standard .NET attributes -โœ… **3 AI Classes** - Actioner, MultiAgentActioner, Github_Actioner -โœ… **1 Helper Class** - New PluginToolExtractor utility -โœ… **1 Support Class** - ToolDescriptionGenerator (SK dependencies removed) -โœ… **1 UI File** - Form1.cs namespace updates -โœ… **2 Config Files** - FlowVision.csproj and packages.config - -**Total: 17 files modified/created** - ---- - -## ๐Ÿ”‘ Key Technical Achievements - -### 1. Plugin Tool Extraction -Created `PluginToolExtractor` utility class that: -- Uses reflection to discover all public methods in plugins -- Automatically converts them to AITools -- Eliminates code duplication -- Makes adding new plugins easier - -```csharp -// Simple, clean usage: -var tools = new List(); -tools.AddRange(PluginToolExtractor.ExtractTools(new CMDPlugin())); -``` - -### 2. Modern Chat Client Pattern -```csharp -// Clean, direct client creation -var azureClient = new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(apiKey)); -IChatClient chatClient = (IChatClient)azureClient.GetChatClient(deploymentName); - -// Function invocation with builder pattern -chatClient = new ChatClientBuilder(chatClient) - .UseFunctionInvocation() - .Build(); -``` - -### 3. Simplified Streaming -```csharp -// Modern async streaming pattern -await foreach (var update in chatClient.GetStreamingResponseAsync(history, options)) -{ - if (update.Text != null) - { - responseBuilder.Append(update.Text); - } -} -``` - ---- - -## ๐Ÿ“ฆ Package Changes - -### Active Packages -- โœ… Microsoft.Extensions.AI (9.4.0-preview.1.25207.5) -- โœ… Microsoft.Extensions.AI.Abstractions (9.4.0-preview.1.25207.5) -- โœ… Microsoft.Extensions.AI.AzureAIInference (9.4.0-preview.1.25207.5) -- โœ… Azure.AI.OpenAI (2.2.0-beta.4) -- โœ… OpenAI (2.2.0-beta.4) - -### Can Be Removed (After Testing) -- Microsoft.SemanticKernel (1.47.0) -- Microsoft.SemanticKernel.Abstractions (1.47.0) -- Microsoft.SemanticKernel.Core (1.47.0) -- Microsoft.SemanticKernel.Connectors.AzureOpenAI (1.47.0) -- Microsoft.SemanticKernel.Connectors.OpenAI (1.47.0) - ---- - -## ๐Ÿงช Testing Checklist - -### Functional Testing -- [ ] Test single-agent execution (Actioner) -- [ ] Test multi-agent coordination (MultiAgentActioner) -- [ ] Test GitHub Models integration (Github_Actioner) -- [ ] Test each plugin individually: - - [ ] CMDPlugin - - [ ] PowerShellPlugin - - [ ] KeyboardPlugin - - [ ] MousePlugin - - [ ] ScreenCapturePlugin - - [ ] WindowSelectionPlugin - - [ ] PlaywrightPlugin - - [ ] RemoteControlPlugin - - [ ] ScreenCaptureOmniParserPlugin - -### Integration Testing -- [ ] Test tool/function calling works correctly -- [ ] Test auto-invocation of tools -- [ ] Test streaming responses -- [ ] Test chat history management -- [ ] Test error handling and recovery -- [ ] Test configuration loading (APIConfig, ToolConfig) - -### Performance Testing -- [ ] Compare response times with SK implementation -- [ ] Monitor memory usage -- [ ] Check for any regressions - ---- - -## ๐ŸŽฏ Benefits Achieved - -### 1. **Simplified Codebase** -- No more complex Kernel builders -- Direct client creation -- Cleaner dependency injection - -### 2. **Better Maintainability** -- Standard .NET attributes (`[Description]`) -- Helper utilities reduce duplication -- Modern async patterns - -### 3. **Future-Proof** -- Using Microsoft's recommended AI abstraction -- Part of the official Agent Framework -- Better long-term support - -### 4. **Flexibility** -- Easy to swap AI providers -- Provider-agnostic IChatClient interface -- Can mix and match different models - -### 5. **Lighter Dependencies** -- Fewer packages required -- Smaller dependency tree -- Can remove SK packages entirely - ---- - -## ๐Ÿ“š Documentation - -### Created Documents -1. **MIGRATION_SUMMARY.md** - Detailed migration guide and patterns -2. **MIGRATION_STATUS.md** - Complete status report -3. **MIGRATION_COMPLETE.md** - This document - -### Code Comments -- All major changes documented in code -- Helper classes have XML documentation -- Complex patterns explained inline - ---- - -## ๐Ÿš€ Next Steps - -### Immediate (Before Next Commit) -1. โœ… Complete all functional testing -2. โœ… Verify no regressions -3. โœ… Test with actual API keys and models - -### Short-term (This Week) -1. Remove Semantic Kernel packages from packages.config -2. Clean up any remaining SK references -3. Update README.md with new architecture notes - -### Long-term (Optional) -1. Consider using Microsoft.Agents.AI for advanced agent features -2. Explore workflow patterns from agent-framework -3. Add telemetry/observability with OpenTelemetry integration -4. Implement caching with DistributedCache - ---- - -## ๐Ÿ™ Acknowledgments - -- **Microsoft Agent Framework Team** - For creating the Extensions.AI abstraction -- **Semantic Kernel Team** - For the foundation that made this possible -- **FlowVision/Recursive Control Project** - For the well-structured codebase - ---- - -## ๐Ÿ“ž Support - -If you encounter any issues: -1. Check MIGRATION_SUMMARY.md for API patterns -2. Review MIGRATION_STATUS.md for technical solutions -3. Refer to Microsoft.Extensions.AI documentation -4. Check microsoft/agent-framework GitHub repo for examples - ---- - -## โœจ Final Notes - -This migration demonstrates that: -- Microsoft.Extensions.AI is production-ready -- The migration path from SK is straightforward -- The benefits outweigh the migration effort -- The new codebase is cleaner and more maintainable - -**The Recursive Control project is now using the latest Microsoft AI abstractions and is ready for the future!** ๐ŸŽ‰ - ---- - -*Migration completed: [Current Date]* -*Build Status: โœ… SUCCESS* -*Test Status: Ready for testing* diff --git a/MIGRATION_STATUS.md b/MIGRATION_STATUS.md deleted file mode 100644 index 4ca020e..0000000 --- a/MIGRATION_STATUS.md +++ /dev/null @@ -1,199 +0,0 @@ -# Migration Status Report - COMPLETED! โœ… - -## Summary -**The migration from Semantic Kernel to Microsoft Agent Framework is now COMPLETE and the project builds successfully!** - -All core functionality has been migrated to use Microsoft.Extensions.AI, which is the foundation of the Microsoft Agent Framework. - -## โœ… Completed Work - -### 1. **All Plugins Migrated** (9 files) โœ… -- Removed `[KernelFunction]` attributes -- Now use standard .NET `[Description]` attributes -- No more Semantic Kernel dependencies in plugins -- Files migrated: - - CMDPlugin.cs - - KeyboardPlugin.cs - - MousePlugin.cs - - PlaywrightPlugin.cs - - PowershellPlugin.cs - - RemoteControlPlugin.cs - - ScreenCaptureOmniParserPlugin.cs - - ScreenCapturePlugin.cs - - WindowSelectionPlugin.cs - -### 2. **Core AI Classes Updated** (3 files) โœ… -- **Actioner.cs** - Fully migrated to Microsoft.Extensions.AI -- **MultiAgentActioner.cs** - Fully migrated to Microsoft.Extensions.AI -- **Github_Actioner.cs** - Fully migrated to Microsoft.Extensions.AI -- All converted to use IChatClient interface and modern APIs - -### 3. **Type System Modernized** โœ… -- โœ… `IChatCompletionService` โ†’ `IChatClient` -- โœ… `ChatHistory` โ†’ `List` -- โœ… `Kernel` โ†’ Removed (direct client usage) -- โœ… `OpenAIPromptExecutionSettings` โ†’ `ChatOptions` -- โœ… `ToolCallBehavior` โ†’ `ChatClientBuilder` with `UseFunctionInvocation()` - -### 4. **API Pattern Changes** โœ… -#### Agent Creation: -```csharp -// NEW: Microsoft.Extensions.AI pattern -var azureClient = new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(apiKey)); -IChatClient chatClient = (IChatClient)azureClient.GetChatClient(deploymentName); -``` - -#### Tool Registration: -```csharp -// NEW: Using PluginToolExtractor helper -var tools = new List(); -tools.AddRange(PluginToolExtractor.ExtractTools(new CMDPlugin())); -``` - -#### Function Invocation: -```csharp -// NEW: ChatClientBuilder pattern -chatClient = new ChatClientBuilder(baseChatClient) - .UseFunctionInvocation() - .Build(); -``` - -#### Streaming Responses: -```csharp -// NEW: GetStreamingResponseAsync pattern -await foreach (var update in chatClient.GetStreamingResponseAsync(history, options)) -{ - if (update.Text != null) - { - responseBuilder.Append(update.Text); - } -} -``` - -### 5. **Helper Classes Created** โœ… -- **PluginToolExtractor.cs** - Utility class to extract AITools from plugin instances using reflection -- Eliminates code duplication across all actioner classes -- Automatically discovers and registers all public methods as tools - -### 6. **Project Configuration** โœ… -- Added `LangVersion=latest` to FlowVision.csproj for async streams support -- Updated packages.config with Microsoft.Extensions.AI packages -- Added PluginToolExtractor to project file - -### 7. **Documentation** โœ… -- Created comprehensive MIGRATION_SUMMARY.md -- Updated MIGRATION_STATUS.md with completion status -- All API changes and patterns documented - -## ๐Ÿ“‹ Build Status -โœ… **BUILD SUCCESSFUL!** - -The project compiles with only minor warnings: -- 7 warnings about unused async/await (pre-existing code style) -- 4 warnings about unused exception variables (pre-existing code style) -- 1 warning about obsolete Playwright method (pre-existing) - -**No errors!** - -## ๐ŸŽฏ Migration Completion: 100% - -All planned work is complete: -- โœ… Plugin migration -- โœ… Core class migration -- โœ… Type system updates -- โœ… API pattern modernization -- โœ… Helper utilities created -- โœ… Build successful -- โœ… Documentation complete - -## ๐Ÿ“ฆ Packages Status - -### Core Packages (In Use) -- โœ… Microsoft.Extensions.AI (9.4.0-preview.1.25207.5) -- โœ… Microsoft.Extensions.AI.Abstractions (9.4.0-preview.1.25207.5) -- โœ… Microsoft.Extensions.AI.AzureAIInference (9.4.0-preview.1.25207.5) - -### Supporting Packages -- โœ… Azure.AI.OpenAI (2.2.0-beta.4) -- โœ… OpenAI (2.2.0-beta.4) -- โœ… Azure.Core (1.45.0) - -### Legacy Packages (Can be removed after testing) -- Microsoft.SemanticKernel (1.47.0) - No longer used -- Microsoft.SemanticKernel.Abstractions (1.47.0) - No longer used -- Microsoft.SemanticKernel.Core (1.47.0) - No longer used -- Microsoft.SemanticKernel.Connectors.AzureOpenAI (1.47.0) - No longer used -- Microsoft.SemanticKernel.Connectors.OpenAI (1.47.0) - No longer used - -## ๐Ÿ” Key Technical Solutions - -### Problem 1: AIFunctionFactory API -**Solution**: Created PluginToolExtractor utility that uses reflection to extract methods from plugin instances and converts them to AITools. - -### Problem 2: ChatClient Type Conversion -**Solution**: Used explicit cast `(IChatClient)` when getting ChatClient from Azure OpenAI SDK. - -### Problem 3: Streaming API -**Solution**: Used `GetStreamingResponseAsync` instead of the older SK streaming pattern. - -### Problem 4: Function Invocation -**Solution**: Used `ChatClientBuilder` with `UseFunctionInvocation()` extension method for automatic tool calling. - -### Problem 5: ChatRole Ambiguity -**Solution**: Added using alias `using ChatRole = Microsoft.Extensions.AI.ChatRole;` - -## ๐Ÿ“Š Files Modified -- โœ… FlowVision/lib/Classes/ai/Actioner.cs -- โœ… FlowVision/lib/Classes/ai/MultiAgentActioner.cs -- โœ… FlowVision/lib/Classes/ai/Github_Actioner.cs -- โœ… FlowVision/lib/Classes/ToolDescriptionGenerator.cs -- โœ… FlowVision/lib/Classes/PluginToolExtractor.cs (NEW) -- โœ… All 9 plugin files -- โœ… FlowVision/Form1.cs -- โœ… FlowVision/FlowVision.csproj -- โœ… FlowVision/packages.config - -## ๐Ÿงช Next Steps for Testing - -1. **Functional Testing** - - Test single-agent actions with Actioner - - Test multi-agent workflow with MultiAgentActioner - - Test GitHub Models integration with Github_Actioner - - Verify all plugins work correctly - -2. **Integration Testing** - - Test tool calling and auto-invocation - - Test streaming responses - - Test configuration loading - - Test error handling - -3. **Performance Testing** - - Compare performance with previous SK implementation - - Monitor memory usage - - Check response times - -4. **Cleanup (After successful testing)** - - Remove Semantic Kernel packages from packages.config - - Remove SK references from FlowVision.csproj - - Update any remaining documentation - -## ๐ŸŽ‰ Summary - -The migration from Semantic Kernel to Microsoft Agent Framework (Microsoft.Extensions.AI) is **100% complete and successful!** - -### Benefits Achieved: -- โœ… **Simplified API**: No more complex Kernel builders -- โœ… **Modern Patterns**: Using latest Microsoft.Extensions.AI abstractions -- โœ… **Better Maintainability**: Cleaner code with helper utilities -- โœ… **Future-Proof**: Using Microsoft's recommended AI abstraction layer -- โœ… **Flexible**: Easy to swap AI providers -- โœ… **Lighter**: Fewer dependencies once SK packages are removed - -### Migration Quality: -- โœ… All compilation errors resolved -- โœ… Only pre-existing warnings remain -- โœ… Clean, modern code patterns -- โœ… Well-documented changes -- โœ… Helper utilities for maintainability - -**The migration is complete and ready for testing!** ๐Ÿš€ diff --git a/MIGRATION_SUMMARY.md b/MIGRATION_SUMMARY.md deleted file mode 100644 index 43e816b..0000000 --- a/MIGRATION_SUMMARY.md +++ /dev/null @@ -1,200 +0,0 @@ -# Migration from Semantic Kernel to Microsoft Extensions.AI - -## Overview -This project has been migrated from Microsoft Semantic Kernel to use the Microsoft Extensions.AI abstraction layer, which is part of the Microsoft Agent Framework ecosystem. - -## Key Changes - -### 1. Namespace Changes -- **Removed**: `Microsoft.SemanticKernel.*` -- **Added**: `Microsoft.Extensions.AI` -- **Added**: `Azure.AI.OpenAI` (for Azure client) -- **Added**: `OpenAI.Chat` (for chat types) - -### 2. Type Replacements - -| Old (Semantic Kernel) | New (Extensions.AI) | -|----------------------|---------------------| -| `IChatCompletionService` | `IChatClient` | -| `ChatHistory` | `List` | -| `Kernel` | Removed - direct client usage | -| `OpenAIPromptExecutionSettings` | `ChatOptions` | -| `ToolCallBehavior` | `ChatToolMode` | -| `[KernelFunction]` attribute | `[Description]` attribute (standard .NET) | - -### 3. API Pattern Changes - -#### Agent/Chat Client Creation -**Before (SK)**: -```csharp -var builder = Kernel.CreateBuilder(); -builder.AddAzureOpenAIChatCompletion(deploymentName, endpoint, apiKey); -var kernel = builder.Build(); -var chat = kernel.GetRequiredService(); -``` - -**After (Extensions.AI)**: -```csharp -var azureClient = new AzureOpenAIClient(new Uri(endpoint), new Azure.AzureKeyCredential(apiKey)); -var chat = azureClient.AsChatClient(deploymentName); -``` - -#### Chat History Management -**Before (SK)**: -```csharp -history.AddSystemMessage("prompt"); -history.AddUserMessage("user input"); -history.AddAssistantMessage("response"); -``` - -**After (Extensions.AI)**: -```csharp -history.Add(new ChatMessage(ChatRole.System, "prompt")); -history.Add(new ChatMessage(ChatRole.User, "user input")); -history.Add(new ChatMessage(ChatRole.Assistant, "response")); -``` - -#### Tool/Function Registration -**Before (SK)**: -```csharp -builder.Plugins.AddFromType(); -``` - -**After (Extensions.AI)**: -```csharp -var tools = new List(); -tools.AddRange(AIFunctionFactory.Create(new CMDPlugin())); -``` - -#### Chat Completion with Streaming -**Before (SK)**: -```csharp -var settings = new OpenAIPromptExecutionSettings -{ - Temperature = 0.7, - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions -}; -var stream = chat.GetStreamingChatMessageContentsAsync(history, settings, kernel); -await foreach (var message in stream) -{ - responseBuilder.Append(message.Content); -} -``` - -**After (Extensions.AI)**: -```csharp -var options = new ChatOptions -{ - Temperature = 0.7f, - Tools = tools, - ToolMode = ChatToolMode.Auto -}; -await foreach (var update in chat.CompleteStreamingAsync(history, options)) -{ - if (update.Text != null) - { - responseBuilder.Append(update.Text); - } -} -``` - -#### Function Invocation (Auto-invoke tools) -**Before (SK)**: -Built into Kernel with `ToolCallBehavior.AutoInvokeKernelFunctions` - -**After (Extensions.AI)**: -```csharp -var functionInvokingClient = new FunctionInvokingChatClient(chatClient); -``` - -### 4. Plugin Changes -- Removed `[KernelFunction]` attributes from all plugin methods -- Plugins now use standard `[Description]` attributes -- No need for special plugin wrapper classes -- Plugins are instantiated directly and passed to `AIFunctionFactory.Create()` - -## Files Modified - -### Core AI Classes -- `FlowVision/lib/Classes/ai/Actioner.cs` - Migrated to Extensions.AI -- `FlowVision/lib/Classes/ai/MultiAgentActioner.cs` - Migrated to Extensions.AI -- `FlowVision/lib/Classes/ai/Github_Actioner.cs` - Migrated to Extensions.AI -- `FlowVision/lib/Classes/ToolDescriptionGenerator.cs` - Removed SK dependency - -### Plugins (all migrated) -- `FlowVision/lib/Plugins/CMDPlugin.cs` -- `FlowVision/lib/Plugins/KeyboardPlugin.cs` -- `FlowVision/lib/Plugins/MousePlugin.cs` -- `FlowVision/lib/Plugins/PlaywrightPlugin.cs` -- `FlowVision/lib/Plugins/PowershellPlugin.cs` -- `FlowVision/lib/Plugins/RemoteControlPlugin.cs` -- `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs` -- `FlowVision/lib/Plugins/ScreenCapturePlugin.cs` -- `FlowVision/lib/Plugins/WindowSelectionPlugin.cs` - -### UI -- `FlowVision/Form1.cs` - Updated namespace imports - -## Packages - -### Removed Semantic Kernel Packages (Keeping for backward compatibility during transition) -- Microsoft.SemanticKernel -- Microsoft.SemanticKernel.Abstractions -- Microsoft.SemanticKernel.Core -- Microsoft.SemanticKernel.Connectors.AzureOpenAI -- Microsoft.SemanticKernel.Connectors.OpenAI - -### Core Packages (Already Present) -- Microsoft.Extensions.AI (9.4.0-preview.1.25207.5) -- Microsoft.Extensions.AI.Abstractions (9.4.0-preview.1.25207.5) -- Microsoft.Extensions.AI.AzureAIInference (9.4.0-preview.1.25207.5) - -### Added Packages -- Microsoft.Extensions.AI.OpenAI (9.4.0-preview.1.25207.5) - For AsChatClient extension methods -- Microsoft.Agents.AI (1.0.0-preview.251001.3) - Optional, for advanced agent features -- Microsoft.Agents.AI.OpenAI (1.0.0-preview.251001.3) - Optional - -### Supporting Packages (Already Present) -- Azure.AI.OpenAI (2.2.0-beta.4) -- OpenAI (2.2.0-beta.4) -- Azure.Core (1.45.0) - -## Benefits of Migration - -1. **Simplified API**: No more complex Kernel builders and service registration -2. **Better Abstractions**: IChatClient is provider-agnostic -3. **Standard .NET**: Uses standard attributes ([Description]) instead of custom ones -4. **Flexibility**: Easy to swap AI providers -5. **Future-Proof**: Microsoft Extensions.AI is the future direction for AI in .NET -6. **Lighter Dependencies**: Fewer packages and abstractions to maintain - -## Testing Checklist - -- [ ] Test basic single-agent actions with Actioner -- [ ] Test multi-agent workflow with MultiAgentActioner -- [ ] Verify all plugins work correctly - - [ ] CMDPlugin - - [ ] PowerShellPlugin - - [ ] KeyboardPlugin - - [ ] MousePlugin - - [ ] ScreenCapturePlugin - - [ ] WindowSelectionPlugin - - [ ] PlaywrightPlugin - - [ ] RemoteControlPlugin -- [ ] Test tool/function calling and auto-invocation -- [ ] Test streaming responses -- [ ] Test configuration loading and API connectivity -- [ ] Test error handling - -## Notes - -- The migration maintains backward compatibility where possible -- Semantic Kernel packages are kept in packages.config during transition -- The core functionality remains the same - only the underlying framework changed -- Performance should be similar or better due to simpler abstractions - -## References - -- [Microsoft Extensions.AI Documentation](https://learn.microsoft.com/dotnet/ai/quickstarts/quickstart-ai-chat-with-prompts) -- [Microsoft Agent Framework](https://github.com/microsoft/agent-framework) -- [Migration Guide](https://github.com/microsoft/agent-framework/blob/main/dotnet/samples/SemanticKernelMigration/README.md) diff --git a/MODERN_UI_REDESIGN.md b/MODERN_UI_REDESIGN.md deleted file mode 100644 index 9a59c94..0000000 --- a/MODERN_UI_REDESIGN.md +++ /dev/null @@ -1,570 +0,0 @@ -# Modern UI Redesign - Novel & Intuitive Interface - -## Date: October 2, 2025 - -## Overview - -We've completely **redesigned the menu structure** to be modern, intuitive, and properly reflect the multi-agent architecture. The old confusing structure (LLM โ†’ Setup โ†’ Azure OpenAI) has been replaced with a logical, emoji-enhanced, feature-complete menu system. - ---- - -## โŒ **Old Menu Structure (Confusing)** - -``` -File -โ”œโ”€ Tools -โ””โ”€ New Chat - -Vision -โ””โ”€ OmniParser - -LLM โ† Confusing! Only shows Azure? -โ””โ”€ Setup - โ””โ”€ Azure OpenAI โ† Where are other models? - -Reason โ† What does this even do? -``` - -### Problems: -- โŒ "LLM โ†’ Setup โ†’ Azure OpenAI" implies only Azure works -- โŒ No way to configure Planner or Coordinator agents -- โŒ No way to configure GitHub agent -- โŒ "Reason" menu item doesn't work -- โŒ No visibility into multi-agent mode -- โŒ No way to access new features (export, visualizers) -- โŒ Not intuitive - users had to guess - ---- - -## โœ… **New Menu Structure (Modern & Clear)** - -``` -๐Ÿ“ File -โ”œโ”€ ๐Ÿ”ง Tools -โ”œโ”€ ๐Ÿ†• New Chat -โ””โ”€ ๐Ÿ“ค Export Chat - โ”œโ”€ ๐Ÿ“„ Export to JSON - โ”œโ”€ ๐Ÿ“ Export to Markdown - โ”œโ”€ ๐Ÿ› Export Debug Log (with Tools) - โ””โ”€ ๐Ÿ“‹ Copy to Clipboard - -โš™๏ธ Setup -โ”œโ”€ ๐Ÿ”ง Tools -โ”œโ”€ ๐Ÿค– AI Agents -โ”‚ โ”œโ”€ โšก Actioner Agent (Primary) -โ”‚ โ”œโ”€ ๐Ÿ“‹ Planner Agent -โ”‚ โ”œโ”€ ๐ŸŽฏ Coordinator Agent -โ”‚ โ””โ”€ ๐Ÿ™ GitHub Agent -โ”œโ”€ ๐Ÿ”ญ Vision Tools -โ”‚ โ””โ”€ ๐Ÿ“ธ OmniParser Config -โ””โ”€ ๐Ÿ”€ Multi-Agent Mode โœ“ - -๐Ÿ‘๏ธ View -โ”œโ”€ ๐Ÿ“Š Activity Monitor โœ“ -โ””โ”€ ๐ŸŽฏ Execution Visualizer โœ“ - -โ“ Help -โ”œโ”€ โ„น๏ธ About -โ””โ”€ ๐Ÿ“š Documentation -``` - ---- - -## ๐ŸŽจ **Design Principles** - -### 1. **Emoji Visual Language** ๐ŸŽจ -Every menu item has an emoji for instant recognition: -- ๐Ÿค– = AI Agents -- ๐Ÿ”ง = Configuration/Tools -- ๐Ÿ“Š = Monitoring/Analytics -- ๐ŸŽฏ = Execution/Action -- ๐Ÿ“ค = Export/Share -- โ„น๏ธ = Information/Help - -**Why?** Faster visual scanning, more engaging, modern UI standards. - -### 2. **Logical Grouping** ๐Ÿ“‹ -Related items are grouped together: -- **File**: Document operations (new, export) -- **Setup**: Configuration (agents, tools, vision) -- **View**: UI panels (monitor, visualizer) -- **Help**: Information (about, docs) - -### 3. **Clear Hierarchy** ๐ŸŒณ -Max 2-3 levels deep. No confusing nested menus. - -### 4. **Descriptive Labels** ๐Ÿ“ -"Actioner Agent (Primary)" tells you: -- What it is (Actioner Agent) -- Its role (Primary execution agent) - -### 5. **Checkboxes for Toggles** โœ“ -Visual feedback for ON/OFF states: -- โœ“ Multi-Agent Mode (enabled) -- โœ“ Activity Monitor (visible) - ---- - -## ๐Ÿ†• **New Features Exposed** - -### AI Agent Configuration - -**All 4 agents now accessible:** - -1. **โšก Actioner Agent (Primary)** - - The main execution agent - - Handles single-agent mode - - Choose: Azure OpenAI, LM Studio, or GitHub Models - -2. **๐Ÿ“‹ Planner Agent** - - Plans step-by-step execution - - Used in multi-agent mode - - Separate model configuration - -3. **๐ŸŽฏ Coordinator Agent** - - User interface and routing - - Used in multi-agent mode - - Separate model configuration - -4. **๐Ÿ™ GitHub Agent** - - Specialized for GitHub operations - - Independent configuration - - Can use GitHub Models free tier - -**Each opens the unified `AIProviderConfigForm` with:** -- Azure OpenAI (Cloud) -- LM Studio (Local) -- GitHub Models (Free Tier) - ---- - -### Multi-Agent Mode Toggle - -**Setup โ†’ ๐Ÿ”€ Multi-Agent Mode** (checkbox) - -- **Unchecked (OFF)**: Direct Actioner execution - - Fast, simple tasks - - Single AI agent - - Good for straightforward commands - -- **Checked (ON)**: Coordinator โ†’ Planner โ†’ Actioner workflow - - Complex, multi-step tasks - - Up to 25 steps - - Adaptive planning - - Better for workflows - -**Visual Feedback:** -When toggled, shows message in chat: -``` -System: Multi-Agent Mode enabled. Using Coordinator โ†’ Planner โ†’ -Actioner workflow with up to 25 steps. -``` - ---- - -### Export Chat Menu - -**File โ†’ ๐Ÿ“ค Export Chat** - -4 export formats instantly accessible: -- **JSON**: Machine-readable, for analysis -- **Markdown**: Human-readable, for docs -- **Debug Log**: Includes tool calls, for troubleshooting -- **Clipboard**: Quick copy-paste - -No more hunting for export features! - ---- - -### View Menu (Future-Ready) - -**๐Ÿ‘๏ธ View** - -Toggleable UI panels: -- **๐Ÿ“Š Activity Monitor**: Real-time system status -- **๐ŸŽฏ Execution Visualizer**: Step-by-step progress - -*Currently shows "coming soon" but infrastructure is ready* - ---- - -## ๐Ÿ’ก **Novel Features** - -### 1. Per-Agent Configuration โญ - -**What's Novel:** Each agent (Actioner, Planner, Coordinator, GitHub) can use a **different AI provider**. - -**Example Configuration:** -``` -Actioner: Azure GPT-4 (powerful, expensive) -Planner: LM Studio Llama 3 (local, free) -Coordinator: GitHub Phi-4 (fast, free tier) -GitHub: GitHub Models (specialized) -``` - -**Why Novel:** Mix and match based on: -- **Cost**: Use free for simple, paid for complex -- **Latency**: Local for speed, cloud for power -- **Privacy**: Keep sensitive data local -- **Specialization**: Use best model for each role - -### 2. Visual Mode Indicator โญ - -**What's Novel:** Checkbox shows current execution mode at a glance. - -``` -โœ“ Multi-Agent Mode โ† 3-agent workflow active - Multi-Agent Mode โ† Single agent (direct) -``` - -**Why Novel:** Instant visibility into how your commands will execute. No guessing. - -### 3. Emoji Visual Language โญ - -**What's Novel:** Every menu item has a semantic emoji. - -**Why Novel:** -- Faster visual scanning -- Works across languages -- More engaging/modern -- Accessibility (visual cues) - -### 4. Unified Agent Config โญ - -**What's Novel:** One form configures all 3 providers (Azure, LM Studio, GitHub) for any agent. - -**Traditional Approach:** -- Separate form per provider -- Confusing which model is active -- Hard to switch - -**Our Approach:** -- Single unified form -- Dropdown to switch providers -- Clear visual indication -- Save/Test buttons - ---- - -## ๐ŸŽฏ **User Experience Improvements** - -### Before: -``` -User: "How do I configure the planner agent?" -Answer: "You can't from the UI, edit config files manually" - -User: "Can I use LM Studio for the coordinator?" -Answer: "Yes but you need to edit JSON" - -User: "How do I enable multi-agent mode?" -Answer: "Tools โ†’ Enable Multi-Agent checkbox" - -User: "How do I export chat for debugging?" -Answer: "You can't, check the log files" -``` - -### After: -``` -User: "How do I configure the planner agent?" -Answer: "Setup โ†’ AI Agents โ†’ Planner Agent" - -User: "Can I use LM Studio for the coordinator?" -Answer: "Setup โ†’ AI Agents โ†’ Coordinator Agent โ†’ - Choose 'LM Studio (Local)'" - -User: "How do I enable multi-agent mode?" -Answer: "Setup โ†’ Multi-Agent Mode (click checkbox)" - -User: "How do I export chat for debugging?" -Answer: "File โ†’ Export Chat โ†’ Export Debug Log" -``` - -**Everything is discoverable!** - ---- - -## ๐Ÿ“Š **Menu Structure Details** - -### File Menu -``` -๐Ÿ“ File -โ”œโ”€ ๐Ÿ”ง Tools (Configure plugins) -โ”œโ”€ ๐Ÿ†• New Chat (Clear conversation) -โ””โ”€ ๐Ÿ“ค Export Chat - โ”œโ”€ ๐Ÿ“„ Export to JSON - โ”œโ”€ ๐Ÿ“ Export to Markdown - โ”œโ”€ ๐Ÿ› Export Debug Log (with Tools) - โ””โ”€ ๐Ÿ“‹ Copy to Clipboard -``` - -**Purpose**: Document/conversation operations - ---- - -### Setup Menu -``` -โš™๏ธ Setup -โ”œโ”€ ๐Ÿ”ง Tools (Plugin configuration) -โ”œโ”€ ๐Ÿค– AI Agents -โ”‚ โ”œโ”€ โšก Actioner Agent (Primary) -โ”‚ โ”œโ”€ ๐Ÿ“‹ Planner Agent -โ”‚ โ”œโ”€ ๐ŸŽฏ Coordinator Agent -โ”‚ โ””โ”€ ๐Ÿ™ GitHub Agent -โ”œโ”€ ๐Ÿ”ญ Vision Tools -โ”‚ โ””โ”€ ๐Ÿ“ธ OmniParser Config -โ””โ”€ ๐Ÿ”€ Multi-Agent Mode โœ“ -``` - -**Purpose**: System configuration - -**AI Agents submenu** - Each opens AIProviderConfigForm: -- Agent name in title -- All 3 providers available -- Independent configuration per agent - -**Multi-Agent Mode** - Toggle with instant feedback: -- Checkbox shows current state -- Click to toggle -- System message confirms change -- Explains what mode does - ---- - -### View Menu -``` -๐Ÿ‘๏ธ View -โ”œโ”€ ๐Ÿ“Š Activity Monitor โœ“ -โ””โ”€ ๐ŸŽฏ Execution Visualizer โœ“ -``` - -**Purpose**: Toggle UI panels - -**Activity Monitor**: -- Real-time system status -- AI/ONNX/Browser states -- Color-coded activity log -- Export capability - -**Execution Visualizer**: -- Step-by-step progress -- Status icons per step -- Progress bar -- Auto-scroll - -*Currently placeholder, full integration coming* - ---- - -### Help Menu -``` -โ“ Help -โ”œโ”€ โ„น๏ธ About -โ””โ”€ ๐Ÿ“š Documentation -``` - -**Purpose**: Information and help - -**About**: -- Version information -- Feature list -- GitHub link -- Quick reference - -**Documentation**: -- Opens GitHub Wiki -- Comprehensive guides -- API documentation -- Examples - ---- - -## ๐Ÿ”ง **Technical Implementation** - -### Menu Structure -```csharp -// Old way (limited) -LLM โ†’ Setup โ†’ Azure OpenAI - -// New way (comprehensive) -Setup โ†’ AI Agents โ†’ [Choose Agent] โ†’ [Configure Any Provider] -``` - -### Event Handlers - -**Agent Configuration:** -```csharp -private void actionerAgentToolStripMenuItem_Click(object sender, EventArgs e) -{ - AIProviderConfigForm configForm = new AIProviderConfigForm("actioner"); - configForm.ShowDialog(); -} -``` - -**Multi-Agent Toggle:** -```csharp -private void multiAgentModeToolStripMenuItem_Click(object sender, EventArgs e) -{ - var toolConfig = ToolConfig.LoadConfig("toolsconfig"); - toolConfig.EnableMultiAgentMode = multiAgentModeToolStripMenuItem.Checked; - toolConfig.SaveConfig("toolsconfig"); - - AddMessage("System", $"Multi-Agent Mode {status}..."); -} -``` - -**State Persistence:** -```csharp -// On Form Load -var toolConfig = ToolConfig.LoadConfig("toolsconfig"); -multiAgentModeToolStripMenuItem.Checked = toolConfig.EnableMultiAgentMode; -``` - ---- - -## ๐ŸŽฏ **Benefits** - -### For Users -- โœ… **Discoverable**: All features visible in menus -- โœ… **Intuitive**: Logical grouping and clear labels -- โœ… **Visual**: Emojis provide instant recognition -- โœ… **Flexible**: Configure each agent independently -- โœ… **Transparent**: See current mode at a glance - -### For Support -- โœ… **Easy to Guide**: "Go to Setup โ†’ AI Agents โ†’ Actioner" -- โœ… **Clear State**: Checkboxes show current configuration -- โœ… **Export Tools**: Users can send debug logs easily -- โœ… **Less Confusion**: No more "where do I configure X?" - -### For Developers -- โœ… **Extensible**: Easy to add new menu items -- โœ… **Consistent**: All agents use same config form -- โœ… **Maintainable**: Clear hierarchy and naming -- โœ… **Future-Ready**: View menu ready for new panels - ---- - -## ๐Ÿ“‹ **Migration Guide** - -### Old โ†’ New Mapping - -| Old Location | New Location | -|-------------|--------------| -| LLM โ†’ Setup โ†’ Azure OpenAI | Setup โ†’ AI Agents โ†’ Actioner Agent | -| *(No way to config planner)* | Setup โ†’ AI Agents โ†’ Planner Agent | -| *(No way to config coordinator)* | Setup โ†’ AI Agents โ†’ Coordinator Agent | -| Vision โ†’ OmniParser | Setup โ†’ Vision Tools โ†’ OmniParser Config | -| Tools โ†’ *(checkbox)* | Setup โ†’ Multi-Agent Mode | -| *(No export)* | File โ†’ Export Chat โ†’ [4 formats] | - ---- - -## ๐Ÿš€ **Future Enhancements** - -### Planned Features - -1. **Quick Config Panel** - - Floating panel with most-used settings - - One-click agent switching - - Live status indicators - -2. **Visual Agent Pipeline** - - Diagram showing: User โ†’ Coordinator โ†’ Planner โ†’ Actioner - - Highlight active agent - - Show which model each uses - -3. **Preset Configurations** - - Save/Load entire configurations - - "Power User" preset (all cloud) - - "Privacy" preset (all local) - - "Budget" preset (all free) - -4. **Smart Suggestions** - - "This task works better with multi-agent mode" - - "Your planner agent is slower than actioner" - - "Consider using local model for privacy" - -5. **Model Performance Metrics** - - Response times per agent - - Token usage tracking - - Cost estimation - - Success rates - ---- - -## โœ… **Build Status** - -``` -โœ… New menu structure: Implemented -โœ… All 4 agents: Accessible -โœ… Multi-agent toggle: Working -โœ… Export menu: Functional -โœ… View menu: Prepared (placeholder) -โœ… Help menu: Functional -โœ… State persistence: Working -โœ… Emoji support: Rendering correctly -โœ… Compilation: 0 errors -โœ… No breaking changes -``` - ---- - -## ๐Ÿ“ธ **Visual Examples** - -### Menu Structure -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ ๐Ÿ“ File โš™๏ธ Setup ๐Ÿ‘๏ธ View โ“ Help โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ”œโ”€ ๐Ÿ”ง Tools - โ”œโ”€ ๐Ÿ†• New Chat - โ””โ”€ ๐Ÿ“ค Export Chat โ”€โ”€โ”€โ” - โ”œโ”€ ๐Ÿ“„ Export to JSON - โ”œโ”€ ๐Ÿ“ Export to Markdown - โ”œโ”€ ๐Ÿ› Export Debug Log - โ””โ”€ ๐Ÿ“‹ Copy to Clipboard -``` - -### Agent Configuration -``` -Setup โ†’ ๐Ÿค– AI Agents โ”€โ”€โ”€โ” - โ”œโ”€ โšก Actioner Agent (Primary) - โ”œโ”€ ๐Ÿ“‹ Planner Agent - โ”œโ”€ ๐ŸŽฏ Coordinator Agent - โ””โ”€ ๐Ÿ™ GitHub Agent -``` - -### Mode Indication -``` -Setup -โ”œโ”€ ... other items ... -โ””โ”€ ๐Ÿ”€ Multi-Agent Mode โœ“ โ† Currently enabled -``` - ---- - -## ๐ŸŽ‰ **Summary** - -### What Changed -- โŒ Removed confusing "LLM" and "Reason" menus -- โœ… Added comprehensive "Setup" menu -- โœ… Added all 4 AI agents to menu -- โœ… Added multi-agent mode toggle -- โœ… Added export capabilities -- โœ… Added view menu for future panels -- โœ… Added help menu -- โœ… Enhanced with emoji visual language - -### Impact -**Before**: Confusing, limited, users had to edit config files -**After**: Intuitive, comprehensive, everything discoverable from UI - -### Novel Aspects -1. Per-agent model configuration (mix and match) -2. Visual mode indicator (checkbox) -3. Emoji-enhanced menu system -4. Unified configuration form -5. 4-format export system - -**The UI is now modern, intuitive, and properly reflects the powerful multi-agent architecture underneath!** ๐ŸŽจโœจ diff --git a/OCR_INTEGRATION_CHECKLIST.md b/OCR_INTEGRATION_CHECKLIST.md deleted file mode 100644 index 0526d49..0000000 --- a/OCR_INTEGRATION_CHECKLIST.md +++ /dev/null @@ -1,305 +0,0 @@ -# โœ… OCR Integration - Completion Checklist - -## Task: Enable OCR Text Extraction from UI Elements - -**Date**: October 2, 2025 -**Status**: โœ… COMPLETE - ---- - -## Requirements Checklist - -### Core Functionality -- [x] โœ… Tesseract OCR 5.2.0 integrated -- [x] โœ… OCR text extraction from UI elements working -- [x] โœ… Semantic labels generated with actual text -- [x] โœ… Falls back gracefully if OCR unavailable -- [x] โœ… No breaking changes to existing code - -### Build & Deployment -- [x] โœ… Project compiles without errors -- [x] โœ… All dependencies deployed correctly -- [x] โœ… Native DLLs copied to output directory -- [x] โœ… Language data files in place -- [x] โœ… Build configuration for automatic deployment - -### Code Changes -- [x] โœ… Minimal modifications (only what's necessary) -- [x] โœ… OcrHelper.cs fully implemented -- [x] โœ… Project file updated with Tesseract reference -- [x] โœ… Build targets added for native DLL deployment -- [x] โœ… Code follows existing patterns and style - -### Testing & Verification -- [x] โœ… Build successful (0 errors) -- [x] โœ… Prerequisites verification script created -- [x] โœ… All required files present -- [x] โœ… OCR initialization verified -- [x] โœ… Text extraction tested - -### Documentation -- [x] โœ… Technical documentation complete -- [x] โœ… Quick reference guide created -- [x] โœ… Implementation details documented -- [x] โœ… Verification procedures documented -- [x] โœ… Troubleshooting guide included - -### Performance -- [x] โœ… OCR processing time acceptable (~2-4s) -- [x] โœ… Thread-safe implementation -- [x] โœ… Async processing for non-blocking operation -- [x] โœ… Small region filtering optimization -- [x] โœ… Graceful error handling - -### Quality -- [x] โœ… No memory leaks (proper disposal) -- [x] โœ… Error logging comprehensive -- [x] โœ… User-friendly log messages -- [x] โœ… Character whitelist optimized for UI -- [x] โœ… Configuration settings documented - ---- - -## Deliverables Checklist - -### Code Files -- [x] โœ… `FlowVision/FlowVision.csproj` - Updated -- [x] โœ… `FlowVision/lib/Classes/OcrHelper.cs` - Implemented - -### Documentation Files -- [x] โœ… `OCR_INTEGRATION_COMPLETE.md` - Overview -- [x] โœ… `OCR_TEXT_EXTRACTION_STATUS.md` - Technical details -- [x] โœ… `OCR_QUICK_REFERENCE.md` - Quick reference -- [x] โœ… `TEST_OCR.md` - Implementation docs -- [x] โœ… `TASK_COMPLETE_OCR_INTEGRATION.md` - Task report -- [x] โœ… `OCR_INTEGRATION_CHECKLIST.md` - This file - -### Scripts -- [x] โœ… `test_ocr_simple.ps1` - Prerequisites checker - -### Binary/Data Files -- [x] โœ… `tesseract50.dll` (2.66 MB) -- [x] โœ… `leptonica-1.82.0.dll` (3.98 MB) -- [x] โœ… `Tesseract.dll` (0.13 MB) -- [x] โœ… `eng.traineddata` (3.92 MB) - ---- - -## Integration Checklist - -### Infrastructure -- [x] โœ… ONNX OmniParser already has OCR integration points -- [x] โœ… UIElementDetection.Caption property available -- [x] โœ… Label generation logic already in place -- [x] โœ… No changes needed to existing plugins - -### Dependencies -- [x] โœ… Tesseract NuGet package referenced -- [x] โœ… Native libraries deployed -- [x] โœ… Language model downloaded -- [x] โœ… Build system configured for auto-copy - -### Configuration -- [x] โœ… TesseractEngine initialized correctly -- [x] โœ… Character whitelist configured -- [x] โœ… Preserve spaces enabled -- [x] โœ… Engine mode set (Default) - ---- - -## Success Criteria - -### Functional Requirements -- [x] โœ… OCR extracts text from UI elements -- [x] โœ… Labels include actual text content -- [x] โœ… System works without OCR (fallback) -- [x] โœ… No crashes or errors during operation - -### Performance Requirements -- [x] โœ… OCR processing time < 5 seconds -- [x] โœ… No blocking of main thread -- [x] โœ… Memory usage reasonable -- [x] โœ… CPU usage acceptable - -### Quality Requirements -- [x] โœ… Code is maintainable -- [x] โœ… Documentation is comprehensive -- [x] โœ… Error handling is robust -- [x] โœ… Logging is informative - ---- - -## Verification Steps - -### Build Verification -```powershell -# Run build -MSBuild.exe FlowVision\FlowVision.csproj /t:Rebuild /p:Configuration=Debug - -# Check for errors -# Expected: 0 errors -``` -**Result**: โœ… PASSED - -### Prerequisites Verification -```powershell -# Run verification script -.\test_ocr_simple.ps1 - -# Expected: All prerequisites satisfied -``` -**Result**: โœ… PASSED - -### Runtime Verification -``` -# Launch application -# Check logs for: -# "โœ“ Tesseract OCR initialized successfully" -``` -**Result**: โœ… PASSED - ---- - -## Before & After Comparison - -### Before OCR Integration -``` -Log Output: -[timestamp] Info: OcrHelper, Initialize, OCR is currently disabled -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, OCR not available - -Element Labels: -"Element 171" -"Element 172" -"Element 173" - -Status: โŒ No semantic understanding -``` - -### After OCR Integration -``` -Log Output: -[timestamp] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, Extracting text from 145 elements -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: 85 elements with text - -Element Labels: -"Play Video at (150,200) [size: 120x40]" -"Subscribe Button at (300,250) [size: 200x60]" -"YouTube Logo at (450,300) [size: 180x50]" - -Status: โœ… Rich semantic understanding -``` - ---- - -## Impact Assessment - -### Technical Impact -- โœ… **Positive**: OCR adds semantic understanding -- โœ… **Positive**: No breaking changes to existing code -- โœ… **Positive**: Graceful fallback if OCR fails -- โœ… **Neutral**: Adds ~11 MB to distribution size -- โœ… **Neutral**: Adds 2-4 seconds to analysis time - -### User Impact -- โœ… **Positive**: Better automation accuracy (40% โ†’ 90%) -- โœ… **Positive**: More natural interactions -- โœ… **Positive**: AI can verify actions -- โœ… **Positive**: Improved user experience -- โœ… **Neutral**: Slightly longer processing time - ---- - -## Risk Assessment - -### Risks Identified -1. โŒ ~OCR initialization failure~ - **Mitigated**: Graceful fallback -2. โŒ ~Missing dependencies~ - **Mitigated**: Build targets auto-deploy -3. โŒ ~Performance impact~ - **Mitigated**: Async processing, filtering -4. โŒ ~Memory leaks~ - **Mitigated**: Proper disposal, single instance - -### All Risks Mitigated โœ… - ---- - -## Final Sign-Off - -### Completed By -- **Developer**: GitHub Copilot CLI -- **Date**: October 2, 2025 -- **Version**: 1.0 - -### Approval Checklist -- [x] โœ… All requirements met -- [x] โœ… All tests passed -- [x] โœ… Documentation complete -- [x] โœ… No known issues -- [x] โœ… Ready for production use - -### Status -**โœ… APPROVED FOR RELEASE** - ---- - -## Post-Integration Tasks - -### Immediate (Done) -- [x] โœ… Build and deploy -- [x] โœ… Verify all files present -- [x] โœ… Test initialization -- [x] โœ… Create documentation - -### Short-term (Optional) -- [ ] ๐Ÿ”„ Test with various UI types -- [ ] ๐Ÿ”„ Monitor performance metrics -- [ ] ๐Ÿ”„ Collect user feedback -- [ ] ๐Ÿ”„ Optimize if needed - -### Long-term (Optional) -- [ ] ๐Ÿ”„ Add more languages -- [ ] ๐Ÿ”„ Implement confidence filtering -- [ ] ๐Ÿ”„ Add parallel processing -- [ ] ๐Ÿ”„ Create OCR result cache - ---- - -## Summary - -### What Was Accomplished โœ… - -1. **Core Integration** - - Tesseract 5.2.0 fully integrated - - OCR text extraction operational - - Semantic labeling implemented - -2. **Quality Assurance** - - Zero compilation errors - - Comprehensive testing complete - - Documentation thorough - -3. **Deployment** - - All dependencies deployed - - Build system configured - - Verification tools created - -### Final Result ๐ŸŽ‰ - -**The ONNX OmniParser can now:** -- โœ… See UI elements (YOLO detection) -- โœ… Read text from elements (Tesseract OCR) -- โœ… Understand semantic meaning -- โœ… Provide rich context to AI - -**Impact**: Dramatically improved AI automation capabilities! - ---- - -## Conclusion - -โœ… **Task**: OCR Text Extraction Integration -โœ… **Status**: COMPLETE AND OPERATIONAL -โœ… **Quality**: Production-ready -โœ… **Result**: SUCCESS - -**All objectives achieved. Ready for use!** ๐Ÿš€ diff --git a/OCR_INTEGRATION_COMPLETE.md b/OCR_INTEGRATION_COMPLETE.md deleted file mode 100644 index 87e9621..0000000 --- a/OCR_INTEGRATION_COMPLETE.md +++ /dev/null @@ -1,334 +0,0 @@ -# โœ… OCR Integration Complete - October 2, 2025 - -## Summary - -**Tesseract OCR 5.2.0 is now fully integrated with the ONNX OmniParser!** - -The system can now extract actual text from detected UI elements, providing semantic understanding instead of just position-based descriptions. - ---- - -## What Was Done - -### 1. Tesseract Integration โœ… -- Added Tesseract 5.2.0 reference to project -- Configured build to copy native DLLs automatically -- Downloaded English language model (eng.traineddata) -- Deployed to both Debug and Release configurations - -### 2. OCR Implementation โœ… -- **File**: `FlowVision/lib/Classes/OcrHelper.cs` -- Replaced placeholder with full Tesseract implementation -- Added thread-safe TesseractEngine initialization -- Implemented text extraction for full images and regions -- Configured for optimal UI text recognition - -### 3. Native Dependencies โœ… -- `tesseract50.dll` (2.66 MB) - Core OCR engine -- `leptonica-1.82.0.dll` (3.98 MB) - Image processing -- `Tesseract.dll` - Managed C# wrapper -- `eng.traineddata` (3.92 MB) - English language model - -### 4. Build Configuration โœ… -- Updated project file with Tesseract reference -- Added Tesseract.targets import -- Created custom build target for native DLL deployment -- Ensured all dependencies copy to output directory - ---- - -## The Transformation - -### BEFORE (Generic Labels) -``` -"Element 171" -"Element 172" -"Element 173" -``` -โŒ AI has no idea what these elements are - -### AFTER (Semantic Labels) -``` -"Play Video at (150,200) [size: 120x40]" -"Subscribe Button at (300,250) [size: 200x60]" -"YouTube Logo at (450,300) [size: 180x50]" -``` -โœ… AI knows exactly what each element is and does - ---- - -## How It Works - -``` -1. User triggers OmniParser screen capture - โ†“ -2. YOLO object detection finds UI elements - โ†“ -3. For each detected element: - a. Crop region from screenshot - b. Convert to Tesseract Pix format - c. Run OCR text extraction - d. Clean and validate text - โ†“ -4. Generate enhanced labels: - - If text found: "Button Text at (x,y) [size: WxH]" - - If no text: "UI Element #N at (x,y) [size: WxH]" - โ†“ -5. Return results to AI with rich semantic context -``` - ---- - -## Verification - -### Check OCR is Active - -**Look for this log message on startup:** -``` -[timestamp] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully. Text extraction is now enabled. -``` - -### During Screenshot Analysis -``` -[timestamp] Info: OnnxOmniParser, ParseImage, Processing image 4480x1440 -[timestamp] Info: OnnxOmniParser, ParseImage, Detected 145 UI elements -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, Extracting text from 145 elements -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: 85 elements with text -``` - -### Run Prerequisites Check -```powershell -.\test_ocr_simple.ps1 -``` - -Expected output: -``` -โœ“ All prerequisites satisfied! -โœ“ FlowVision.exe -โœ“ Tesseract.dll -โœ“ tesseract50.dll -โœ“ leptonica-1.82.0.dll -โœ“ tessdata folder -โœ“ eng.traineddata -``` - ---- - -## Files Changed - -### Modified Files -1. โœ… `FlowVision/FlowVision.csproj` - - Added Tesseract reference - - Added native DLL copy target - - Added Tesseract.targets import - -2. โœ… `FlowVision/lib/Classes/OcrHelper.cs` - - Complete rewrite with Tesseract implementation - - 189 lines of production code - -### New Files -3. โœ… `FlowVision/bin/Debug/tessdata/eng.traineddata` -4. โœ… `FlowVision/bin/Release/tessdata/eng.traineddata` -5. โœ… `test_ocr_simple.ps1` (verification script) -6. โœ… `TEST_OCR.md` (technical documentation) -7. โœ… `OCR_INTEGRATION_COMPLETE.md` (this file) - -### Updated Files -8. โœ… `OCR_TEXT_EXTRACTION_STATUS.md` (marked as complete) - ---- - -## Build Status - -``` -โœ… Build: SUCCESSFUL -โœ… Errors: 0 -โœ… Warnings: 11 (existing, unrelated) -โœ… OCR: OPERATIONAL -โœ… Dependencies: DEPLOYED -``` - ---- - -## Performance - -### Typical Timings -- **YOLO Detection**: ~400-500ms -- **OCR Processing**: ~2-4 seconds (145 elements) -- **Total Analysis**: ~4-5 seconds -- **Text Success Rate**: 60-80% of elements - -### Optimizations Applied -โœ… Thread-safe single engine instance -โœ… Skip regions smaller than 10x10 pixels -โœ… Async processing on background threads -โœ… Graceful handling of OCR failures -โœ… Character whitelist for UI text - ---- - -## Benefits - -### For the AI -1. โœ… **Understands UI semantics** - Knows what buttons say -2. โœ… **Target accuracy** - Can find "Subscribe" specifically -3. โœ… **Content verification** - Can read and confirm text -4. โœ… **Context awareness** - Understands UI meaning - -### For Users -1. โœ… **Better automation** - AI interacts with labeled elements -2. โœ… **Higher accuracy** - Fewer mistakes -3. โœ… **Natural commands** - "Click Save button" works -4. โœ… **Verification** - AI confirms actions by reading results - ---- - -## Testing Instructions - -### Basic Test -1. Launch `FlowVision.exe` -2. Check logs for OCR initialization message -3. Use OmniParser to capture a screenshot -4. Verify element labels contain actual text - -### Expected Results -- Elements with text show actual content -- Elements without text show position/size -- OCR success logged with count -- No errors in logs - ---- - -## Configuration - -### Tesseract Settings - -**Engine Mode**: Default (Legacy + LSTM) - -**Language**: English - -**Character Whitelist**: -``` -ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .-_:@/\()[]{}!?&+=#$% -``` - -**Settings**: -- `preserve_interword_spaces = 1` -- Optimized for UI text - ---- - -## Troubleshooting - -### OCR Not Initializing - -**Check these:** -1. โœ… tessdata folder exists in exe directory -2. โœ… eng.traineddata file present (3.92 MB) -3. โœ… tesseract50.dll present (2.66 MB) -4. โœ… leptonica-1.82.0.dll present (3.98 MB) - -**Run verification:** -```powershell -.\test_ocr_simple.ps1 -``` - -### Empty OCR Results - -**Common causes:** -- Element doesn't contain text (expected) -- Text too small (< 10x10 pixels) -- Non-standard font -- Poor image quality - -**System handles this gracefully** - Falls back to position-based labels - ---- - -## Future Enhancements (Optional) - -### Potential Improvements -- ๐Ÿ”„ Add more languages (fra.traineddata, spa.traineddata, etc.) -- ๐Ÿ”„ Implement confidence filtering -- ๐Ÿ”„ Add parallel OCR processing -- ๐Ÿ”„ Cache OCR results for unchanged screens -- ๐Ÿ”„ Fine-tune for specific UI frameworks - -### Not Required -The current implementation is **production-ready** and fully functional. - ---- - -## Technical Details - -### Architecture - -**OcrHelper.cs**: -- Static class with singleton TesseractEngine -- Thread-safe with lock-based synchronization -- Automatic tessdata path detection -- Graceful initialization with error handling - -**Integration Points**: -- `OnnxOmniParserEngine.ExtractTextFromDetections()` - Calls OCR -- `ScreenCaptureOmniParserPlugin.ConvertOnnxResultToParsedContent()` - Uses results -- `UIElementDetection.Caption` - Stores extracted text - -### Error Handling - -**Initialization Errors**: -- Missing tessdata: Logs error, OCR disabled -- Missing language file: Logs error, OCR disabled -- Engine creation failure: Logs error, OCR disabled - -**Runtime Errors**: -- OCR processing failure: Logs error, returns empty string -- Invalid region: Validates and adjusts bounds -- Small regions: Skips OCR (< 10x10) - ---- - -## Summary - -### โœ… Mission Accomplished - -Tesseract OCR 5.2.0 is now: -- โœ… Fully integrated -- โœ… Automatically initialized -- โœ… Extracting text from UI elements -- โœ… Providing semantic labels to AI -- โœ… Production ready - -### Impact - -The AI can now **understand what UI elements say**, not just where they are! - -This dramatically improves: -- Automation accuracy -- User experience -- Natural language interaction -- Task completion reliability - ---- - -## Status: โœ… COMPLETE AND OPERATIONAL - -**Version**: 1.0 -**Date**: October 2, 2025 -**Technology**: Tesseract 5.2.0 + ONNX YOLO -**Result**: Semantic UI understanding enabled - ---- - -## Next Steps - -1. โœ… **Done**: Integration complete -2. โœ… **Done**: Build successful -3. โœ… **Done**: Dependencies deployed -4. ๐ŸŽฏ **Next**: Test with real screenshots -5. ๐ŸŽฏ **Next**: Monitor performance and accuracy -6. ๐ŸŽฏ **Next**: Collect user feedback - ---- - -**The ONNX OmniParser now has eyes AND the ability to read! ๐ŸŽ‰** diff --git a/OCR_QUICK_REFERENCE.md b/OCR_QUICK_REFERENCE.md deleted file mode 100644 index f0caa4f..0000000 --- a/OCR_QUICK_REFERENCE.md +++ /dev/null @@ -1,158 +0,0 @@ -# OCR Integration - Quick Reference Card - -## โœ… Status: OPERATIONAL - -**Tesseract OCR 5.2.0** is now fully integrated and active! - ---- - -## Quick Facts - -| Item | Value | -|------|-------| -| **OCR Engine** | Tesseract 5.2.0 | -| **Language** | English (eng.traineddata) | -| **Status** | โœ… Active and operational | -| **Build** | โœ… Successful (0 errors) | -| **Dependencies** | โœ… All deployed | - ---- - -## The Change - -### Before ๐Ÿ˜ -``` -"Element 171" -"Element 172" -"Element 173" -``` - -### After ๐Ÿ˜ƒ -``` -"Play Video at (150,200) [size: 120x40]" -"Subscribe Button at (300,250) [size: 200x60]" -"YouTube Logo at (450,300) [size: 180x50]" -``` - ---- - -## How to Verify - -### Check Prerequisites -```powershell -.\test_ocr_simple.ps1 -``` - -### Expected Output -``` -โœ“ FlowVision.exe -โœ“ Tesseract.dll -โœ“ tesseract50.dll -โœ“ leptonica-1.82.0.dll -โœ“ tessdata folder -โœ“ eng.traineddata -``` - -### Check Logs -Launch FlowVision and look for: -``` -[timestamp] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully -``` - ---- - -## What It Does - -1. **Detects** UI elements with YOLO -2. **Extracts** text with Tesseract OCR -3. **Labels** elements with actual content -4. **Provides** semantic understanding to AI - ---- - -## Files Modified - -| File | Change | -|------|--------| -| `FlowVision.csproj` | Added Tesseract reference + build targets | -| `OcrHelper.cs` | Full Tesseract implementation (189 lines) | -| `bin/Debug/tessdata/` | English language model deployed | -| `bin/Debug/` | Native DLLs deployed | - ---- - -## Performance - -- **Detection**: ~400-500ms -- **OCR**: ~2-4 seconds (145 elements) -- **Total**: ~4-5 seconds -- **Success Rate**: 60-80% text extraction - ---- - -## Troubleshooting - -### OCR Not Initializing? -1. Check tessdata folder exists -2. Verify eng.traineddata is present (3.92 MB) -3. Ensure native DLLs are deployed -4. Run `test_ocr_simple.ps1` - -### No Text Extracted? -- **Normal** - Not all UI elements contain text -- Falls back to position-based labels -- Check element size (must be > 10x10 pixels) - ---- - -## Key Benefits - -โœ… **Semantic Understanding** - AI knows what UI elements say -โœ… **Better Accuracy** - Fewer automation mistakes -โœ… **Natural Commands** - "Click Save button" works -โœ… **Content Verification** - AI can read and confirm text - ---- - -## Documentation - -๐Ÿ“„ **OCR_INTEGRATION_COMPLETE.md** - Overview and summary -๐Ÿ“„ **OCR_TEXT_EXTRACTION_STATUS.md** - Complete technical details -๐Ÿ“„ **TEST_OCR.md** - Implementation documentation -๐Ÿ“„ **test_ocr_simple.ps1** - Prerequisites verification script - ---- - -## Support - -### Log Locations -Check application logs for OCR initialization and processing messages. - -### Expected Messages - -**Success**: -``` -[timestamp] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: X elements with text -``` - -**Errors**: -``` -[timestamp] Error: OcrHelper, Initialize, tessdata directory not found -[timestamp] Error: OcrHelper, Initialize, Failed to initialize Tesseract: [details] -``` - ---- - -## Next Steps - -1. โœ… Build and deploy complete -2. ๐ŸŽฏ Test with real screenshots -3. ๐ŸŽฏ Monitor performance -4. ๐ŸŽฏ Gather user feedback - ---- - -**๐ŸŽ‰ OCR is now active and ready to use!** - -Launch FlowVision.exe and capture a screenshot to see it in action! diff --git a/OCR_TEXT_EXTRACTION_STATUS.md b/OCR_TEXT_EXTRACTION_STATUS.md deleted file mode 100644 index 729af2e..0000000 --- a/OCR_TEXT_EXTRACTION_STATUS.md +++ /dev/null @@ -1,313 +0,0 @@ -# OCR Text Extraction for ONNX OmniParser - COMPLETED โœ… - -## Date: October 2, 2025 - -## Summary - -### โœ… OCR Integration - COMPLETE - -**Tesseract OCR is now fully integrated and operational!** - -The ONNX OmniParser now extracts actual text from detected UI elements using Tesseract OCR 5.2.0. - -## What Changed - -### 1. โœ… Tesseract Integration -- **Package**: Tesseract 5.2.0 installed via NuGet -- **Reference added** to FlowVision.csproj -- **Native DLLs** deployed (tesseract50.dll, leptonica-1.82.0.dll) -- **Language data** downloaded (eng.traineddata - 3.92 MB) - -### 2. โœ… OcrHelper Implementation -**File**: `FlowVision/lib/Classes/OcrHelper.cs` - -Replaced placeholder implementation with full Tesseract integration: -- **TesseractEngine initialization** with error handling -- **Thread-safe OCR processing** with lock-based synchronization -- **ExtractTextAsync()** - Full image OCR -- **ExtractTextFromRegionAsync()** - Region-specific OCR -- **Automatic tessdata detection** in application directory -- **Graceful degradation** if OCR initialization fails - -### 3. โœ… Build Configuration -**File**: `FlowVision/FlowVision.csproj` - -Added: -- Tesseract reference with `True` -- Tesseract.targets import -- Custom build target to copy native DLLs -- Automatic deployment of OCR dependencies - -### 4. โœ… Language Data Deployed -- `FlowVision/bin/Debug/tessdata/eng.traineddata` -- `FlowVision/bin/Release/tessdata/eng.traineddata` - -## How It Works Now - -### Before OCR โŒ -``` -[2025-10-02 22:50:08] Info: OcrHelper, Initialize, OCR is currently disabled -[2025-10-02 22:50:08] Info: OnnxOmniParser, ExtractTextFromDetections, OCR not available -[2025-10-02 22:50:08] Info: Found 145 UI elements - -Element Labels: -"UI Element #1 at (150,200) [size: 120x40]" -"UI Element #2 at (300,250) [size: 200x60]" -``` - -### After OCR โœ… -``` -[2025-10-02 22:50:08] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully -[2025-10-02 22:50:08] Info: OnnxOmniParser, ExtractTextFromDetections, Extracting text from 145 elements -[2025-10-02 22:50:12] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: 85 elements with text -[2025-10-02 22:50:12] Info: Found 145 UI elements - -Element Labels: -"Play Video at (150,200) [size: 120x40]" -"Subscribe Button at (300,250) [size: 200x60]" -"YouTube Logo at (450,300) [size: 180x50]" -``` - -## Architecture - -### Complete Flow - -``` -Screenshot Capture - โ†“ -YOLO Object Detection (ONNX) - โ†“ -Bounding Box Detection (145 elements found) - โ†“ -For each detected region: - โ†“ - OCR Text Extraction (Tesseract) - โ†“ - Validate region bounds - โ†“ - Crop to bounding box - โ†“ - Convert to Pix format - โ†“ - Run Tesseract OCR - โ†“ - Extract and trim text - โ†“ -Enhanced Label Generation - โ†“ -If text found: "Button Text at (x,y) [size: WxH]" -If no text: "UI Element #N at (x,y) [size: WxH]" - โ†“ -Return to AI with rich semantic labels -``` - -## Technical Details - -### Tesseract Configuration - -**Engine**: TesseractEngine (Default mode - combines legacy and LSTM) - -**Language**: English (eng.traineddata) - -**Character Whitelist**: -``` -ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .-_:@/\()[]{}!?&+=#$% -``` - -**Settings**: -- `preserve_interword_spaces = 1` - Maintains word spacing -- Optimized for UI text recognition - -### Performance Features - -1. **Thread Safety**: Single TesseractEngine with lock-based synchronization -2. **Smart Region Filtering**: Skips regions < 10x10 pixels -3. **Async Processing**: OCR runs on background threads -4. **Empty Result Handling**: Returns empty string for no-text regions -5. **Error Recovery**: Graceful degradation on OCR failures - -### Deployment Structure - -``` -FlowVision/bin/Debug/ -โ”œโ”€โ”€ FlowVision.exe -โ”œโ”€โ”€ Tesseract.dll (managed wrapper) -โ”œโ”€โ”€ tesseract50.dll (native Tesseract) -โ”œโ”€โ”€ leptonica-1.82.0.dll (image processing) -โ””โ”€โ”€ tessdata/ - โ””โ”€โ”€ eng.traineddata (English language model) -``` - -## Build Status - -``` -โœ… Build: 0 errors, 11 warnings -โœ… All dependencies deployed -โœ… OCR fully operational -โœ… No breaking changes -``` - -## Files Modified - -1. โœ… **FlowVision/FlowVision.csproj** - - Added Tesseract reference - - Added Tesseract.targets import - - Added native DLL copy target - -2. โœ… **FlowVision/lib/Classes/OcrHelper.cs** - - Replaced placeholder with full Tesseract implementation - - 189 lines of production-ready OCR code - -3. โœ… **FlowVision/bin/Debug/tessdata/eng.traineddata** (NEW) - - English language model - -4. โœ… **FlowVision/bin/Release/tessdata/eng.traineddata** (NEW) - - English language model - -## Files NOT Modified (Infrastructure Already Ready) - -- โœ… `OnnxOmniParserEngine.cs` - Already had OCR integration -- โœ… `ScreenCaptureOmniParserPlugin.cs` - Already had label generation -- โœ… `UIElementDetection.Caption` - Already available - -## Verification - -### Startup Log Message -Look for this on application startup: -``` -[timestamp] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully. Text extraction is now enabled. -``` - -### During Screenshot Analysis -``` -[timestamp] Info: OnnxOmniParser, ParseImage, Processing image 4480x1440 -[timestamp] Info: OnnxOmniParser, ParseImage, Detected 145 UI elements -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, Extracting text from 145 elements -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: 85 elements with text -``` - -### Error Scenarios - -**Missing tessdata**: -``` -[timestamp] Error: OcrHelper, Initialize, tessdata directory not found at: [path] -``` - -**Missing language file**: -``` -[timestamp] Error: OcrHelper, Initialize, English language data not found at: [path] -``` - -**OCR initialization failure**: -``` -[timestamp] Error: OcrHelper, Initialize, Failed to initialize Tesseract: [error] -``` - -## Benefits - -### For the AI โœ… - -1. **Semantic Understanding**: Knows what buttons say -2. **Target Accuracy**: Can find "Subscribe" button specifically -3. **Content Verification**: Can read and verify UI text -4. **Context Awareness**: Understands UI meaning, not just position - -### For Users โœ… - -1. **Better Automation**: AI can interact with specific labeled elements -2. **Higher Accuracy**: Fewer mistakes due to better UI understanding -3. **Natural Commands**: "Click the Save button" works reliably -4. **Verification**: AI can confirm actions by reading result text - -## Testing - -### Prerequisites Check -Run `test_ocr_simple.ps1` to verify: -```powershell -.\test_ocr_simple.ps1 -``` - -Expected output: -``` -โœ“ All prerequisites satisfied! -โœ“ FlowVision.exe -โœ“ Tesseract.dll -โœ“ tesseract50.dll -โœ“ leptonica-1.82.0.dll -โœ“ tessdata folder -โœ“ eng.traineddata -``` - -### Live Testing -1. Launch FlowVision.exe -2. Check logs for: "โœ“ Tesseract OCR initialized successfully" -3. Use OmniParser to capture a screenshot -4. Verify element labels contain actual text from UI - -## Performance - -### Typical Performance -- **YOLO Detection**: ~400-500ms for 4480x1440 image -- **OCR Processing**: ~2-4 seconds for 145 elements -- **Total Time**: ~4-5 seconds for full analysis -- **Success Rate**: ~60-80% of elements have extractable text - -### Optimization Opportunities -- โœ… Skip very small regions (< 10x10) -- โœ… Async processing -- โš ๏ธ Could add: Parallel OCR processing -- โš ๏ธ Could add: OCR result caching -- โš ๏ธ Could add: Confidence threshold filtering - -## Conclusion - -### โœ… MISSION ACCOMPLISHED - -**OCR text extraction is now fully operational!** - -The ONNX OmniParser can now: -1. โœ… Detect UI elements using YOLO -2. โœ… Extract text using Tesseract OCR -3. โœ… Generate rich semantic labels -4. โœ… Provide meaningful element descriptions to the AI - -### Impact - -**Before**: "Element 171", "Element 172", "Element 173" - -**After**: "Subscribe Button", "Play Video", "Share Link" - -This dramatically improves the AI's ability to understand and interact with UIs! - -## Next Steps (Optional Enhancements) - -1. **Additional Languages**: Add more .traineddata files -2. **OCR Confidence**: Filter low-confidence results -3. **Parallel Processing**: OCR multiple regions simultaneously -4. **Result Caching**: Cache OCR results for unchanged screens -5. **Custom Training**: Fine-tune Tesseract for specific UI styles - -## Support - -### If OCR Doesn't Initialize - -1. Check tessdata folder exists in output directory -2. Verify eng.traineddata file is present (3.92 MB) -3. Check native DLLs are present (tesseract50.dll, leptonica-1.82.0.dll) -4. Review initialization logs for specific error messages - -### If OCR Returns Empty Results - -- UI elements may not contain text -- Text may be too small (< 10x10 pixels) -- Text may be in a non-standard font -- Image quality may be poor - -The system gracefully handles these cases and falls back to position-based labels. - ---- - -**Status**: โœ… COMPLETE and OPERATIONAL -**Version**: 1.0 -**Date**: October 2, 2025 -**Integration**: Tesseract 5.2.0 diff --git a/OMNIPARSER_KISS_MIGRATION.md b/OMNIPARSER_KISS_MIGRATION.md index c906770..e69de29 100644 --- a/OMNIPARSER_KISS_MIGRATION.md +++ b/OMNIPARSER_KISS_MIGRATION.md @@ -1,197 +0,0 @@ -# OmniParser Simplification - Migration Complete โœ… - -## What We Did (KISS Principles Applied) - -### Problem -The OmniParser implementation was overly complex: -- Multiple layers of abstraction -- Python server management with auto-start logic -- HTTP API fallback mechanisms -- Hard-coded external paths -- Complex error handling -- ~1500+ lines of code across 6 files -- Freezing issues due to complexity - -### Solution -Created a simple, focused implementation following KISS: -- โœ… **Single class**: `SimpleOmniParser.cs` (~350 lines) -- โœ… **Pure .NET**: No Python, no servers, no HTTP -- โœ… **Embedded model**: Portable, self-contained -- โœ… **Singleton pattern**: Efficient memory usage -- โœ… **Direct ONNX**: Native inference, no overhead -- โœ… **70% less code**: Easier to maintain and debug - -## Files Changed - -### New Files (Keep) -1. โœ… `FlowVision/lib/Classes/SimpleOmniParser.cs` - Core implementation -2. โœ… `OMNIPARSER_SETUP.md` - Setup documentation -3. โœ… `download_omniparser_model.ps1` - Model downloader script - -### Modified Files -1. โœ… `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs` - Simplified to use new parser - -### Files to Remove (Legacy/Deprecated) -These files are no longer needed: -1. โŒ `FlowVision/lib/Classes/OnnxOmniParserEngine.cs` - Replaced by SimpleOmniParser -2. โŒ `FlowVision/lib/Classes/LocalOmniParserManager.cs` - No server needed -3. โŒ `FlowVision/lib/Classes/OmniParserClient.cs` - No HTTP client needed -4. โŒ `FlowVision/OmniParserForm.cs` - Configuration no longer needed -5. โŒ `FlowVision/OmniParserForm.Designer.cs` -6. โŒ `FlowVision/OmniParserForm.resx` - -## Setup Instructions - -### Step 1: Download Model -```powershell -# Run the download script -.\download_omniparser_model.ps1 -``` - -This downloads the ONNX model from HuggingFace. - -### Step 2: Choose Deployment Mode - -#### Option A: Embedded (Recommended) -1. In Visual Studio, navigate to `FlowVision/models/icon_detect.onnx` -2. Right-click โ†’ Properties -3. Set "Build Action" to "Embedded Resource" -4. Rebuild project -5. โœ… Model is now inside the .exe (portable!) - -#### Option B: External File -1. Build the project -2. Copy `models/` folder to output directory: - - `FlowVision/bin/Debug/models/` - - `FlowVision/bin/Release/models/` -3. โœ… Model loads from external file - -### Step 3: Clean Up Legacy Code (Optional) -Remove the old OmniParser files listed above to keep the codebase clean. - -## API Changes - -### Before (Complex) -```csharp -// Initialize engine -OnnxOmniParserEngine engine = new OnnxOmniParserEngine(modelPath); - -// Or configure mode -ScreenCaptureOmniParserPlugin.ConfigureMode(true, modelPath); - -// Parse -var result = engine.ParseImageBase64(base64); -var parsed = ConvertOnnxResultToParsedContent(result); -``` - -### After (Simple) -```csharp -// That's it! Singleton handles everything -var elements = await plugin.CaptureWholeScreen(); -``` - -The complexity is hidden - just capture and parse! - -## Performance Improvements - -| Metric | Before | After | Improvement | -|--------|--------|-------|-------------| -| Code Lines | ~1500 | ~350 | 70% reduction | -| Dependencies | Python + .NET | .NET only | 100% portable | -| Startup Time | 5-10s (server) | 500ms | 10-20x faster | -| Memory | 300MB+ | 150MB | 50% less | -| Reliability | Server issues | Direct | 100% reliable | - -## Debugging - -### Enable detailed logging -```csharp -// SimpleOmniParser already uses PluginLogger -// Check logs for: -// - Model loading status -// - Detection counts -// - Performance metrics -``` - -### Test model loading -```csharp -try { - var parser = SimpleOmniParser.Instance; - Console.WriteLine("โœ“ Model loaded successfully"); -} catch (Exception ex) { - Console.WriteLine($"โœ— Error: {ex.Message}"); -} -``` - -## Architecture - -``` -User Action (Capture Screen) - โ†“ -ScreenCaptureOmniParserPlugin - โ†“ -SimpleOmniParser.Instance (Singleton) - โ†“ -ONNX Inference (Direct) - โ†“ -List - โ†“ -Convert to ParsedContent (Legacy Format) - โ†“ -Return to AI Agent -``` - -Simple, linear, predictable! - -## Benefits - -1. **No more freezing**: Direct inference, no server communication -2. **Faster startup**: Model loads once, stays in memory -3. **Portable**: Embedded model = single executable -4. **Reliable**: No external dependencies to fail -5. **Maintainable**: One file, clear logic -6. **Debuggable**: Simpler stack traces -7. **Testable**: Easy to unit test - -## Next Steps - -### Immediate -1. โœ… Download model -2. โœ… Set up embedded resource -3. โœ… Test capture functionality -4. โœ… Remove legacy files - -### Future Enhancements -- Add OCR for text extraction (simple integration) -- GPU acceleration (one line change) -- Model quantization for smaller size -- Caching for repeated screens -- Multi-model support (detection + captioning) - -## Rollback Plan - -If you need to rollback: -1. Keep the old files (don't delete yet) -2. Revert `ScreenCaptureOmniParserPlugin.cs` -3. Restore old mode switching logic - -But the new implementation is **simpler, faster, and more reliable** - you won't need to rollback! ๐Ÿš€ - -## Support - -Issues with the new implementation? - -1. Check model is downloaded and accessible -2. Verify ONNX Runtime packages are installed -3. Check logs for initialization errors -4. Test with small screenshots first - -The KISS implementation is designed to be simple to debug and maintain! - ---- - -**Migration Status**: โœ… Complete -**Testing Status**: Ready for testing -**Deployment Status**: Ready for production - -*Simplified by following KISS principles - Keep It Simple, Stupid!* ๐Ÿ˜Š diff --git a/OMNIPARSER_KISS_SUMMARY.md b/OMNIPARSER_KISS_SUMMARY.md index fa58ffd..e69de29 100644 --- a/OMNIPARSER_KISS_SUMMARY.md +++ b/OMNIPARSER_KISS_SUMMARY.md @@ -1,290 +0,0 @@ -# OmniParser KISS Implementation - Summary - -## What Was Done โœ… - -I've successfully simplified your OmniParser implementation following KISS (Keep It Simple, Stupid) principles! Here's what changed: - -### 1. Created New Simple Implementation - -**File: `FlowVision/lib/Classes/SimpleOmniParser.cs`** -- โœ… Single, focused class (~350 lines vs 1500+ before) -- โœ… Singleton pattern for efficient resource management -- โœ… Direct ONNX inference - no layers of abstraction -- โœ… Embedded resource support for portable deployment -- โœ… Automatic model loading from embedded or file -- โœ… Clean, documented API - -### 2. Simplified Plugin - -**File: `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs`** -- โœ… Removed all HTTP server code -- โœ… Removed fallback logic complexity -- โœ… Direct integration with SimpleOmniParser -- โœ… Clean async/await pattern -- โœ… Proper resource management (using statements for Bitmaps) - -### 3. Setup Automation - -**File: `setup_omniparser_complete.ps1`** -- โœ… Automated setup script -- โœ… Handles model download -- โœ… Provides conversion instructions -- โœ… Creates Python conversion script if needed -- โœ… Checks for pre-converted ONNX models - -**File: `test_simple_omniparser.ps1`** -- โœ… Verification script to test setup -- โœ… Checks all dependencies -- โœ… Verifies model existence -- โœ… Validates build output - -### 4. Documentation - -**File: `OMNIPARSER_SETUP.md`** -- โœ… Complete setup guide -- โœ… Model conversion instructions -- โœ… Deployment options explained -- โœ… Troubleshooting section - -**File: `OMNIPARSER_KISS_MIGRATION.md`** -- โœ… Migration guide -- โœ… Before/after comparison -- โœ… Performance improvements -- โœ… Architecture diagram - -## Key Improvements ๐Ÿš€ - -| Aspect | Before | After | Improvement | -|--------|--------|-------|-------------| -| **Code Lines** | ~1500 | ~350 | โ†“ 70% reduction | -| **Files** | 6 files | 1 main file | โ†“ Simple | -| **Dependencies** | Python + .NET | .NET only | โ†“ No external deps | -| **Startup** | 5-10s (server) | 500ms | โšก 10-20x faster | -| **Memory** | 300MB+ | 150MB | โ†“ 50% less | -| **Complexity** | High | Low | โœ“ KISS | -| **Portability** | External | Embedded | โœ“ Single .exe | -| **Reliability** | Server issues | Direct | โœ“ 100% reliable | - -## What You Need to Do Next ๐ŸŽฏ - -### Immediate (Required) - -1. **Get the ONNX model:** - ```powershell - .\setup_omniparser_complete.ps1 - ``` - - Since the official model is PyTorch (.pt), you'll need to: - - Either find a pre-converted ONNX version - - Or convert it using the Python script the setup creates - -2. **Choose deployment mode:** - - **Embedded**: Set build action to "Embedded Resource" (recommended) - - **External**: Copy models/ folder to output directory - -3. **Build and test:** - ```powershell - # Build - msbuild FlowVision.sln /p:Configuration=Release - - # Test - .\test_simple_omniparser.ps1 - ``` - -### Optional (Cleanup) - -**Remove legacy files** (these are no longer used): -- `FlowVision/lib/Classes/OnnxOmniParserEngine.cs` -- `FlowVision/lib/Classes/LocalOmniParserManager.cs` -- `FlowVision/lib/Classes/OmniParserClient.cs` -- `FlowVision/OmniParserForm.cs` + `.Designer.cs` + `.resx` - -You can keep them for now if you want a rollback option. - -## Architecture (New vs Old) - -### Old (Complex) โŒ -``` -User Action - โ†“ -ScreenCaptureOmniParserPlugin - โ†“ -Mode Detection (ONNX vs HTTP?) - โ†“ -LocalOmniParserManager - โ†“ -Server Health Check - โ†“ -Auto-start Python Server - โ†“ -Wait for Server Ready - โ†“ -OmniParserClient (HTTP) - โ†“ -FastAPI Server (Python) - โ†“ -YOLO Model - โ†“ -HTTP Response - โ†“ -Parse JSON - โ†“ -Convert Format - โ†“ -Return Result -``` - -### New (Simple) โœ… -``` -User Action - โ†“ -ScreenCaptureOmniParserPlugin - โ†“ -SimpleOmniParser.Instance - โ†“ -ONNX Inference - โ†“ -Return Result -``` - -That's it! 70% less code, 100% more reliable! ๐ŸŽ‰ - -## Why It Should No Longer Freeze - -### Problems Fixed: - -1. **โŒ Server startup delays** โ†’ โœ… No server, instant -2. **โŒ Network timeouts** โ†’ โœ… No network, direct -3. **โŒ HTTP request overhead** โ†’ โœ… No HTTP, in-process -4. **โŒ JSON serialization** โ†’ โœ… Direct objects -5. **โŒ Multiple threads/locks** โ†’ โœ… Simple singleton -6. **โŒ Complex error handling** โ†’ โœ… Straight-through logic -7. **โŒ External process management** โ†’ โœ… Single process - -### Performance: - -- **First call**: ~500ms (model loading once) -- **Subsequent calls**: ~200ms (direct inference) -- **No delays**: Everything in-memory -- **No freezing**: No server communication waits - -## Model Information - -### What You Need: -- **Model**: OmniParser icon_detect YOLO model -- **Format**: ONNX (converted from PyTorch) -- **Size**: ~6-50MB (depends on version) -- **Source**: https://huggingface.co/microsoft/OmniParser-v2.0 - -### Conversion: -The official model is PyTorch format. To convert: - -```python -from ultralytics import YOLO - -model = YOLO('icon_detect/model.pt') -model.export(format='onnx', simplify=True, opset=12) -``` - -Or use the conversion script created by `setup_omniparser_complete.ps1`. - -## Testing - -Once you have the ONNX model: - -1. **Run test script:** - ```powershell - .\test_simple_omniparser.ps1 - ``` - -2. **Expected output:** - ``` - [โœ“] Model found - [โœ“] SimpleOmniParser.cs - [โœ“] ScreenCaptureOmniParserPlugin.cs - [โœ“] Dependencies installed - [โœ“] All checks passed! - ``` - -3. **Run FlowVision:** - - Capture a screen - - Check logs for "OmniParser" messages - - Should see: "โœ“ Found X UI elements" - - Should NOT see: Server startup messages - -## Support - -If you encounter issues: - -### Model Not Found -``` -[โœ—] OmniParser model not found -``` -**Solution**: Run `.\setup_omniparser_complete.ps1` - -### ONNX Runtime Error -``` -[โœ—] Failed to load model -``` -**Solution**: Verify ONNX Runtime packages are installed (they should be already) - -### Performance Issues -- First call: ~500ms (normal - model loading) -- Subsequent: Should be <300ms -- If slow: Check CPU usage, consider GPU acceleration - -### Still Freezing? -The new implementation shouldn't freeze. If it does: -1. Check logs for exceptions -2. Verify model file integrity -3. Test with smaller screenshots first -4. Check memory usage - -## Next Steps (Future Enhancements) - -Once working, you can: - -1. **Add GPU support**: Uncomment CUDA line in SimpleOmniParser -2. **Add OCR**: Integrate Tesseract for text extraction -3. **Optimize model**: Use INT8 quantization for smaller/faster -4. **Cache results**: Cache parsed screens for repeated views -5. **Multi-model**: Add caption model for richer descriptions - -## Files Summary - -### New Files (Keep) โœ… -- `FlowVision/lib/Classes/SimpleOmniParser.cs` - Core implementation -- `OMNIPARSER_SETUP.md` - Setup guide -- `OMNIPARSER_KISS_MIGRATION.md` - Migration details -- `setup_omniparser_complete.ps1` - Setup automation -- `test_simple_omniparser.ps1` - Verification -- `convert_omniparser_to_onnx.py` - Conversion script (generated) - -### Modified Files โœ… -- `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs` - Simplified - -### Old Files (Can Remove) โŒ -- `FlowVision/lib/Classes/OnnxOmniParserEngine.cs` -- `FlowVision/lib/Classes/LocalOmniParserManager.cs` -- `FlowVision/lib/Classes/OmniParserClient.cs` -- `FlowVision/lib/Classes/OmniParserConfig.cs` -- `FlowVision/OmniParserForm.cs` + Designer + resx -- `download_omniparser_model.ps1` (replaced by setup_omniparser_complete.ps1) - -## Rollback Plan - -If needed, you can rollback: -1. Don't delete old files yet (keep as backup) -2. Revert `ScreenCaptureOmniParserPlugin.cs` from git -3. Re-enable old initialization code - -But you shouldn't need to - the new version is simpler and better! ๐Ÿ˜Š - ---- - -**Status**: โœ… Implementation Complete -**Testing**: โณ Requires ONNX model -**Deployment**: โณ Requires build + embed - -**The hard work is done - just need to get the ONNX model and you're good to go!** ๐Ÿš€ - diff --git a/OMNIPARSER_SETUP.md b/OMNIPARSER_SETUP.md index 6c7be56..e69de29 100644 --- a/OMNIPARSER_SETUP.md +++ b/OMNIPARSER_SETUP.md @@ -1,180 +0,0 @@ -# OmniParser Setup Guide - KISS Edition - -## Overview - -The new simplified OmniParser implementation is **pure .NET** - no Python, no servers, no complexity! - -## ๐Ÿš€ Quick Start (3 Steps) - -### Step 1: Get the ONNX Model - -**IMPORTANT**: The official OmniParser model is in PyTorch format. You need an ONNX version for .NET! - -#### Option A: Use Pre-converted ONNX (Easiest) -```powershell -# Run the setup script -.\setup_omniparser_complete.ps1 -``` - -The script will: -1. Check for existing ONNX models -2. Try to download pre-converted versions -3. Create conversion script if needed - -#### Option B: Convert Manually -If you need to convert the PyTorch model yourself: - -1. **Download PyTorch model:** - ```bash - # Install HuggingFace CLI - pip install huggingface-hub - - # Download model - huggingface-cli download microsoft/OmniParser-v2.0 icon_detect/model.pt --local-dir weights - ``` - -2. **Convert to ONNX:** - ```python - from ultralytics import YOLO - - # Load and export - model = YOLO('weights/icon_detect/model.pt') - model.export(format='onnx', simplify=True, opset=12) - ``` - -3. **Copy to FlowVision:** - - Copy the generated `icon_detect.onnx` to `FlowVision/models/` - -### Step 2: Choose Deployment Mode - -#### Embedded (Recommended for Distribution) -1. In Visual Studio, right-click `FlowVision/models/icon_detect.onnx` -2. Properties โ†’ Build Action โ†’ **Embedded Resource** -3. Rebuild project -4. โœ… Model is now inside the .exe (fully portable!) - -#### External File (Development Mode) -1. Build the project -2. Ensure `models/` folder exists in output directory -3. โœ… Model loads from external file - -### Step 3: Build and Run - -```powershell -# Build in Visual Studio, or: -msbuild FlowVision.sln /p:Configuration=Release - -# Run -.\FlowVision\bin\Release\FlowVision.exe -``` - -## What Changed? - -### Before (Complex): -- โŒ Multiple classes: `OnnxOmniParserEngine`, `LocalOmniParserManager`, `OmniParserClient` -- โŒ Python server management with auto-start, cooldowns, health checks -- โŒ HTTP API fallback logic -- โŒ Hard-coded paths to `T:\OmniParser` -- โŒ Complex initialization and error handling -- โŒ 1000+ lines of code across multiple files - -### After (KISS): -- โœ… Single class: `SimpleOmniParser` (~350 lines) -- โœ… Pure .NET ONNX inference -- โœ… Singleton pattern - lazy initialization -- โœ… Model auto-loads from embedded resource or file -- โœ… No external dependencies -- โœ… Portable and self-contained -- โœ… Fast startup - model stays in memory - -## Architecture - -``` -SimpleOmniParser (singleton) - โ†“ -Load ONNX Model (embedded or file) - โ†“ -ParseScreenshot(Bitmap) โ†’ List -``` - -That's it! No servers, no complexity. - -## Performance - -- **First call**: ~500ms (model loading + inference) -- **Subsequent calls**: ~200ms (inference only) -- **Memory**: ~150MB (ONNX model in RAM) -- **No network**: Everything runs locally - -## API Usage - -```csharp -// Capture and parse screen -var plugin = new ScreenCaptureOmniParserPlugin(); -var elements = await plugin.CaptureWholeScreen(); - -// Each element contains: -// - BBox: [x1, y1, x2, y2] coordinates -// - Content: Description with position and size -// - Confidence: Detection confidence -``` - -## Troubleshooting - -### "Model not found" error - -**Solution 1 (Embedded):** -1. Verify `icon_detect.onnx` is in project -2. Check Properties โ†’ Build Action = "Embedded Resource" -3. Rebuild project - -**Solution 2 (External):** -1. Create `models/` folder next to executable -2. Place `icon_detect.onnx` in that folder -3. Restart application - -### "ONNX Runtime error" - -Make sure these NuGet packages are installed: -``` -Microsoft.ML.OnnxRuntime (>= 1.15.0) -System.Numerics.Tensors -``` - -### Performance Issues - -If detection is slow: -1. Model loads on first use (one-time cost) -2. Consider enabling GPU support (requires CUDA): - ```csharp - // In SimpleOmniParser.InitializeModel(), uncomment: - // sessionOptions.AppendExecutionProvider_CUDA(0); - ``` - -## Model Information - -- **Source**: Microsoft OmniParser v2.0 -- **Architecture**: YOLOv8-based UI element detector -- **Input**: 640x640 RGB image (auto-resized) -- **Output**: Bounding boxes + confidence scores -- **License**: Check HuggingFace model card - -## Next Steps - -To further optimize: - -1. **Add OCR**: Integrate Tesseract or Windows OCR for text extraction -2. **GPU Acceleration**: Enable CUDA for faster inference -3. **Model Quantization**: Use INT8 model for smaller size/faster speed -4. **Caching**: Cache parsed results for repeated screens - -## Removed Components - -These files are no longer needed and can be deleted: -- `OnnxOmniParserEngine.cs` (replaced by `SimpleOmniParser.cs`) -- `LocalOmniParserManager.cs` (no server needed) -- `OmniParserClient.cs` (no HTTP client needed) -- `OmniParserConfig.cs` (minimal config now) -- All Python server code and dependencies - -The new implementation is **~70% less code** and **100% more reliable**! ๐Ÿš€ diff --git a/ONNX_OMNIPARSER_INTEGRATION.md b/ONNX_OMNIPARSER_INTEGRATION.md deleted file mode 100644 index 1a6c75f..0000000 --- a/ONNX_OMNIPARSER_INTEGRATION.md +++ /dev/null @@ -1,272 +0,0 @@ -# ONNX OmniParser Integration - -## Overview - -The FlowVision application now supports **native .NET OmniParser** using ONNX Runtime, eliminating the need for a Python server! This provides significant benefits: - -### โœ… Advantages of ONNX Mode -- **No Python Required**: Runs entirely in .NET using ONNX Runtime -- **Faster Startup**: No need to start Python server and wait for it to initialize -- **Lower Latency**: Direct model inference without HTTP overhead -- **Simpler Deployment**: Single executable with model files -- **Better Resource Management**: More efficient memory usage -- **Cross-Platform**: Works on Windows, Linux, and Mac (with appropriate ONNX Runtime builds) - -## Architecture - -### Components - -1. **OnnxOmniParserEngine.cs** - Core ONNX inference engine - - Loads YOLO model from ONNX format - - Performs UI element detection - - Post-processes results with NMS (Non-Maximum Suppression) - - Returns structured detection results - -2. **ScreenCaptureOmniParserPlugin.cs** - Updated plugin with dual-mode support - - **ONNX Mode** (default): Uses native .NET ONNX Runtime - - **HTTP Mode** (fallback): Uses Python server via LocalOmniParserManager - - Automatically falls back if ONNX fails - -## Setup - -### Model Conversion - -The YOLOv8 PyTorch model has been converted to ONNX format: - -```bash -cd T:\Recursive-Control -python convert_to_onnx.py -``` - -This creates: `T:\OmniParser\weights\icon_detect\model.onnx` (76.7 MB) - -### Dependencies - -The following NuGet packages are required: -- `Microsoft.ML.OnnxRuntime` (v1.19.2) - Native ONNX runtime -- `Microsoft.ML.OnnxRuntime.Managed` (v1.21.1) - Managed wrapper -- `System.Numerics.Tensors` (v10.0.0) - Tensor operations - -All dependencies are included in the project and will be copied to the output directory during build. - -## Usage - -### Automatic Mode Selection - -By default, the plugin uses ONNX mode: - -```csharp -var plugin = new ScreenCaptureOmniParserPlugin(); -var results = await plugin.CaptureWholeScreen(); -// Uses ONNX mode automatically -``` - -### Manual Configuration - -```csharp -// Force ONNX mode with custom model path -ScreenCaptureOmniParserPlugin.ConfigureMode( - useOnnx: true, - onnxModelPath: @"T:\OmniParser\weights\icon_detect\model.onnx" -); - -// Force HTTP server mode -ScreenCaptureOmniParserPlugin.ConfigureMode(useOnnx: false); -``` - -### Direct ONNX Engine Usage - -For advanced scenarios, you can use the ONNX engine directly: - -```csharp -using var engine = new OnnxOmniParserEngine(); - -// Parse image from bitmap -Bitmap screenshot = GetScreenshot(); -var result = engine.ParseImage(screenshot); - -// Parse image from base64 -string base64Image = GetBase64Screenshot(); -var result = engine.ParseImageBase64(base64Image); - -// Draw detections on image -Bitmap annotated = engine.DrawDetections(screenshot, result); -``` - -## Performance - -### Benchmarks (Approximate) - -| Mode | Startup Time | Inference Time | Memory Usage | -|------|-------------|----------------|--------------| -| ONNX (CPU) | < 1s | 2-5s | ~500 MB | -| HTTP Server | 15-30s | 3-6s | ~1-2 GB | -| ONNX (GPU)* | < 1s | 0.5-1s | ~800 MB | - -*GPU support requires CUDA-enabled ONNX Runtime build - -### Model Details - -- **Architecture**: YOLOv11m (medium variant) -- **Input Size**: 640x640 pixels -- **Parameters**: 20M -- **Output**: Bounding boxes with confidence scores -- **Threshold**: 0.05 (configurable) - -## API Reference - -### OnnxOmniParserEngine - -```csharp -public class OnnxOmniParserEngine : IDisposable -{ - // Constructor - public OnnxOmniParserEngine( - string modelPath = null, // Default: T:\OmniParser\weights\icon_detect\model.onnx - float confidenceThreshold = 0.05f, // Minimum confidence for detections - int inputSize = 640 // Model input size - ); - - // Parse methods - public OmniParserResult ParseImage(Bitmap image); - public OmniParserResult ParseImageBase64(string base64Image); - - // Visualization - public Bitmap DrawDetections(Bitmap image, OmniParserResult result); - - // Cleanup - public void Dispose(); -} -``` - -### OmniParserResult - -```csharp -public class OmniParserResult -{ - public List Detections { get; set; } -} - -public class UIElementDetection -{ - public RectangleF BoundingBox { get; set; } // X, Y, Width, Height - public float Confidence { get; set; } // 0.0 to 1.0 - public string Label { get; set; } // Element label - public string ElementType { get; set; } // "ui_element" - public string Caption { get; set; } // Optional caption -} -``` - -## Troubleshooting - -### ONNX Model Not Found - -**Error**: `FileNotFoundException: ONNX model not found at: ...` - -**Solution**: -1. Ensure the ONNX model exists: - ``` - T:\OmniParser\weights\icon_detect\model.onnx - ``` -2. If missing, convert it: - ```bash - cd T:\Recursive-Control - python convert_to_onnx.py - ``` - -### Native DLL Not Found - -**Error**: `DllNotFoundException: Unable to load DLL 'onnxruntime'` - -**Solution**: -1. Rebuild the project - native DLLs are copied automatically -2. Manually copy from: - ``` - packages\Microsoft.ML.OnnxRuntime.1.19.2\runtimes\win-x64\native\*.dll - ``` - to: - ``` - FlowVision\bin\Debug\ - ``` - -### Out of Memory - -**Error**: `OutOfMemoryException` during inference - -**Solution**: -- Reduce image size before processing -- Use lower confidence threshold to reduce post-processing load -- Close other applications to free memory - -### Poor Detection Quality - -**Issue**: Not detecting UI elements accurately - -**Solutions**: -- Lower confidence threshold (default: 0.05) - ```csharp - new OnnxOmniParserEngine(confidenceThreshold: 0.03f) - ``` -- Ensure adequate lighting and contrast in screenshots -- Try different screen resolutions - -## GPU Acceleration (Optional) - -To enable GPU acceleration: - -1. Install CUDA Toolkit 11.x or 12.x -2. Use GPU-enabled ONNX Runtime: - ```xml - - ``` -3. Enable in code: - ```csharp - sessionOptions.AppendExecutionProvider_CUDA(0); - ``` - -## Migration from HTTP Server Mode - -### Before (HTTP Server Mode) -```csharp -// Required Python server to be running -LocalOmniParserManager.EnsureServerRunningAsync(); -var client = new OmniParserClient(httpClient); -var result = await client.ProcessScreenshotAsync(base64Image); -``` - -### After (ONNX Mode) -```csharp -// No server required! -using var engine = new OnnxOmniParserEngine(); -var result = engine.ParseImageBase64(base64Image); -``` - -The `ScreenCaptureOmniParserPlugin` automatically handles both modes, so existing code continues to work! - -## Future Enhancements - -Planned improvements: -- [ ] Florence2 caption model integration for element descriptions -- [ ] OCR integration for text recognition -- [ ] Batch processing for multiple images -- [ ] Custom model training pipeline -- [ ] Web Assembly (WASM) support for browser deployment -- [ ] Mobile deployment (iOS/Android via Xamarin) - -## Files Modified - -- `FlowVision/lib/Classes/OnnxOmniParserEngine.cs` - New ONNX engine -- `FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs` - Updated with dual-mode support -- `FlowVision/FlowVision.csproj` - Added ONNX Runtime references and build tasks -- `FlowVision/packages.config` - Added ONNX Runtime packages -- `convert_to_onnx.py` - Model conversion utility - -## Credits - -- **OmniParser**: Microsoft Research - https://github.com/microsoft/OmniParser -- **ONNX Runtime**: Microsoft - https://onnxruntime.ai/ -- **YOLOv8**: Ultralytics - https://ultralytics.com/ - -## License - -This integration maintains compatibility with the original OmniParser license (MIT) and ONNX Runtime license (MIT). diff --git a/ONNX_QUICKSTART.md b/ONNX_QUICKSTART.md deleted file mode 100644 index 4d7c32b..0000000 --- a/ONNX_QUICKSTART.md +++ /dev/null @@ -1,175 +0,0 @@ -# ONNX OmniParser Quick Start Guide - -## ๐Ÿš€ Getting Started in 3 Steps - -### Step 1: Verify ONNX Model Exists - -Check if the ONNX model file exists: - -``` -T:\OmniParser\weights\icon_detect\model.onnx -``` - -If it doesn't exist, convert the PyTorch model: - -```bash -cd T:\OmniParser\weights\icon_detect -# Using Python with ultralytics installed -python -c "from ultralytics import YOLO; YOLO('model.pt').export(format='onnx', imgsz=640, simplify=True, opset=12)" -``` - -### Step 2: Build and Run - -```bash -cd T:\Recursive-Control -msbuild FlowVision.sln /t:Build /p:Configuration=Debug -``` - -Or use Visual Studio: **Build > Build Solution** (Ctrl+Shift+B) - -### Step 3: Use OmniParser - -The application now automatically uses ONNX mode - no Python server required! - -```csharp -// In your code - it just works! -var plugin = new ScreenCaptureOmniParserPlugin(); -var results = await plugin.CaptureWholeScreen(); - -// Results contain detected UI elements with bounding boxes -foreach (var element in results) -{ - Console.WriteLine($"Found {element.Type} at [{element.BBox[0]}, {element.BBox[1]}]"); -} -``` - -## ๐ŸŽฏ Key Benefits - -### Before (Python Server Mode) -- โŒ Requires Python installation -- โŒ 15-30 second startup time -- โŒ Needs to manage server process -- โŒ HTTP overhead for each request - -### Now (ONNX Mode) -- โœ… **Pure .NET** - no Python required! -- โœ… **< 1 second** startup time -- โœ… Direct in-process inference -- โœ… Lower memory usage - -## ๐Ÿ’ก Example Usage - -### Basic Usage -```csharp -// Capture and parse the whole screen -var plugin = new ScreenCaptureOmniParserPlugin(); -var elements = await plugin.CaptureWholeScreen(); - -Console.WriteLine($"Detected {elements.Count} UI elements"); -``` - -### Advanced Usage -```csharp -// Use ONNX engine directly for more control -using var engine = new OnnxOmniParserEngine( - modelPath: @"T:\OmniParser\weights\icon_detect\model.onnx", - confidenceThreshold: 0.05f -); - -// Load an image -using var bitmap = new Bitmap("screenshot.png"); - -// Detect UI elements -var result = engine.ParseImage(bitmap); - -// Visualize detections -using var annotated = engine.DrawDetections(bitmap, result); -annotated.Save("annotated.png"); - -Console.WriteLine($"Found {result.Detections.Count} elements:"); -foreach (var detection in result.Detections) -{ - Console.WriteLine($" - {detection.ElementType} at " + - $"({detection.BoundingBox.X}, {detection.BoundingBox.Y}) " + - $"with confidence {detection.Confidence:F2}"); -} -``` - -### Window-Specific Capture -```csharp -var plugin = new ScreenCaptureOmniParserPlugin(); - -// Get window handle (example) -IntPtr windowHandle = FindWindow("Notepad"); - -// Capture and parse specific window -var elements = await plugin.CaptureScreen(windowHandle.ToString()); -``` - -## โš™๏ธ Configuration - -### Switch Between Modes - -```csharp -// Use ONNX mode (default) -ScreenCaptureOmniParserPlugin.ConfigureMode(useOnnx: true); - -// Fall back to HTTP server mode if needed -ScreenCaptureOmniParserPlugin.ConfigureMode(useOnnx: false); -``` - -### Adjust Detection Sensitivity - -```csharp -// Lower threshold = more detections (but more false positives) -var engine = new OnnxOmniParserEngine(confidenceThreshold: 0.03f); - -// Higher threshold = fewer, more confident detections -var engine = new OnnxOmniParserEngine(confidenceThreshold: 0.15f); -``` - -## ๐Ÿ“Š Performance Tips - -1. **First Run**: Initial model load takes ~1 second -2. **Subsequent Runs**: Inference is 2-5 seconds per image -3. **Memory**: Keep ~500 MB RAM available -4. **GPU**: Enable GPU mode for 5-10x faster inference (requires CUDA) - -## ๐Ÿ› Troubleshooting - -### Issue: "ONNX model not found" - -**Solution**: Convert the model: -```bash -pip install ultralytics -python -c "from ultralytics import YOLO; YOLO('T:/OmniParser/weights/icon_detect/model.pt').export(format='onnx')" -``` - -### Issue: "Unable to load DLL 'onnxruntime'" - -**Solution**: Rebuild the project - native DLLs are automatically copied to the output directory. - -### Issue: Poor detection results - -**Solutions**: -- Lower confidence threshold -- Ensure good screenshot quality -- Check for adequate contrast - -## ๐Ÿ“š Additional Resources - -- Full Documentation: [ONNX_OMNIPARSER_INTEGRATION.md](ONNX_OMNIPARSER_INTEGRATION.md) -- OmniParser GitHub: https://github.com/microsoft/OmniParser -- ONNX Runtime: https://onnxruntime.ai/ - -## โœจ What's Next? - -The integration supports: -- โœ… UI element detection with YOLO -- โณ Caption generation (planned) -- โณ OCR integration (planned) -- โณ Custom model training (planned) - ---- - -**Ready to go!** The application now runs OmniParser natively in .NET with no Python dependencies. Just build and run! ๐ŸŽ‰ diff --git a/PLAYWRIGHT_SESSION_PERSISTENCE.md b/PLAYWRIGHT_SESSION_PERSISTENCE.md deleted file mode 100644 index 79aad2a..0000000 --- a/PLAYWRIGHT_SESSION_PERSISTENCE.md +++ /dev/null @@ -1,401 +0,0 @@ -# Playwright Browser Session Persistence - -## ๐ŸŽฏ Problem Solved - -**Before:** When using Playwright to navigate to sites like LinkedIn, you had to manually log in every single time. After logging in and saying "keep going", the AI would lose your session and you'd need to log in again. - -**After:** Your browser sessions are now **automatically saved**! Log in once, and your cookies/authentication persist across: -- Multiple commands in the same session -- Different conversations -- Application restarts -- Days or weeks later - -## โœจ Key Features - -โœ… **Auto-Save Sessions** - Automatically saves after navigation, clicks, and typing -โœ… **Persistent Logins** - Stay logged in to LinkedIn, Gmail, Facebook, etc. -โœ… **Multiple Sessions** - Save different sessions for different accounts/purposes -โœ… **Smart Detection** - Uses the same browser instance when already running -โœ… **Wait for Login** - New methods to wait for manual login and save -โœ… **Cross-Conversation** - Sessions persist across different AI conversations - -## ๐Ÿš€ How It Works - -### Automatic Session Saving - -The browser automatically saves your session after: -1. **Navigation** - After `NavigateTo()` completes -2. **Clicking** - After `ClickElement()` completes -3. **Typing** - After `TypeText()` completes -4. **Manual Save** - After `SaveSession()` is called - -### Session Storage - -Sessions are stored in: -``` -%APPDATA%\FlowVision\PlaywrightSessions\ -``` - -Each session includes: -- ๐Ÿช **Cookies** - Authentication cookies -- ๐Ÿ” **Local Storage** - Saved preferences -- ๐Ÿ“ฆ **Session Storage** - Temporary data -- ๐ŸŽซ **Authentication Tokens** - OAuth tokens, etc. - -## ๐Ÿ“– Usage Examples - -### Example 1: Navigate to LinkedIn (Stays Logged In) - -**First Time:** -``` -You: "Navigate to LinkedIn feed" -AI: Launches browser, goes to LinkedIn (you're logged out) -You: "I logged in, keep going" -AI: Saves your session automatically -``` - -**Next Time:** -``` -You: "Navigate to LinkedIn feed" -AI: Uses existing browser, you're already logged in! โœ“ -``` - -### Example 2: Using Multiple Commands - -``` -You: "Open LinkedIn" -AI: Navigates, auto-saves session - -You: "Take a screenshot" -AI: Uses same browser, still logged in - -You: "Click on notifications" -AI: Still using same session, auto-saves after click -``` - -### Example 3: Wait for Manual Login - -``` -You: "Navigate to LinkedIn and wait for me to log in" -AI: Opens browser, waits 30 seconds -[You manually log in] -AI: Automatically saves your login session -``` - -## ๐Ÿ› ๏ธ New Methods - -### 1. WaitForUserAndSaveSession - -Waits for you to complete manual actions (like logging in), then saves the session. - -``` -Usage: "Navigate to LinkedIn, then wait for me to log in for 60 seconds" -``` - -**Parameters:** -- `seconds` - How long to wait (default: 30) - -**Example:** -``` -You: "Open Gmail and wait 60 seconds for me to log in" -AI: Opens browser, waits 60 seconds, then saves session -``` - -### 2. WaitForElementAndSave - -Waits for a specific element to appear (indicating successful login), then saves. - -``` -Usage: "Navigate to LinkedIn, wait for the feed to load, then save session" -``` - -**Parameters:** -- `selector` - CSS selector of element to wait for -- `timeout` - Maximum seconds to wait (default: 30) - -**Example:** -``` -You: "Go to LinkedIn, wait for div.feed-shared-update-v2 to appear" -AI: Waits for feed element, then saves session automatically -``` - -### 3. EnableAutoSave / DisableAutoSave - -Control whether sessions are automatically saved. - -``` -Usage: "Enable auto-save sessions" -Usage: "Disable auto-save sessions" -``` - -**Note:** Auto-save is enabled by default! - -### 4. SaveSession (Manual) - -Explicitly save the current session. - -``` -Usage: "Save the current browser session" -``` - -Useful when: -- You've manually done something in the browser -- You want to ensure session is saved -- Auto-save is disabled - -## ๐ŸŽญ Session Management - -### Default Session - -By default, all browser operations use the "default" session. - -``` -You: "Open LinkedIn" -AI: Uses "default" session (restores previous login if exists) -``` - -### Multiple Sessions - -You can use different sessions for different purposes: - -``` -You: "Set session ID to work-account" -AI: Now using "work-account" session - -You: "Open LinkedIn" -AI: Opens with work account login (if previously saved) - -You: "Set session ID to personal-account" -You: "Open LinkedIn" -AI: Opens with personal account login (different session) -``` - -### Session Persistence Control - -``` -Enable session persistence (default): -You: "Enable session persistence" - -Disable session persistence (fresh start every time): -You: "Disable session persistence" -``` - -## ๐Ÿ’ก Smart Browser Reuse - -The system is intelligent about browser instances: - -### Scenario 1: Browser Already Running -``` -You: "Open LinkedIn" -AI: "Using existing chromium browser that is already running" -[Uses same browser, keeps session] -``` - -### Scenario 2: No Browser Running -``` -You: "Open LinkedIn" -AI: "No browser currently active, launching new browser" -[Starts new browser, restores session if exists] -``` - -### Scenario 3: Force New Browser -``` -You: "Launch a new browser, force new" -AI: Closes existing browser, launches fresh one -``` - -## ๐Ÿ” Security & Privacy - -### What's Saved -- Session cookies (authentication) -- Local storage data -- Session storage data -- Login tokens - -### What's NOT Saved -- Passwords (unless auto-filled by browser) -- Credit card information -- Form data you haven't submitted - -### Where It's Stored -``` -%APPDATA%\FlowVision\PlaywrightSessions\ -``` - -Each session is a JSON file containing your browser state. - -### Security Best Practices - -1. โœ… **Use different sessions** for different accounts -2. โœ… **Manually delete sessions** you no longer need -3. โœ… **Don't share session files** - they contain your login tokens -4. โœ… **Log out manually** from sensitive sites before closing - -## ๐Ÿ› Troubleshooting - -### "Not Staying Logged In" - -**Solution:** -1. Check auto-save is enabled: `"Is auto-save enabled?"` -2. Manually save after login: `"Save the current session"` -3. Verify session file exists: Check `%APPDATA%\FlowVision\PlaywrightSessions\` - -### "Using Wrong Account" - -**Solution:** -1. Switch sessions: `"Set session ID to my-other-account"` -2. Or delete old session: Delete file from PlaywrightSessions folder -3. Or force new browser: `"Launch browser, force new"` - -### "Session Not Restoring" - -**Solution:** -1. Check if session file exists in PlaywrightSessions folder -2. Try manual save: `"Save session"` after logging in -3. Check session ID: `"What's the current session ID?"` - -### "Browser Opens but Not Logged In" - -**Possible Causes:** -- Site cleared cookies -- Session expired -- Different session ID being used -- Site requires re-authentication - -**Solution:** -1. Log in manually -2. Save session: `"Save this session"` -3. Continue using - -## ๐Ÿ“Š Session Status - -Check your current browser and session status: - -``` -You: "What's the browser status?" -AI: Returns JSON with: -{ - "PlaywrightInitialized": "Yes", - "BrowserActive": "Yes", - "CurrentSessionId": "default", - "SessionPersistenceEnabled": "Yes" -} -``` - -Or simpler: -``` -You: "Is browser active?" -AI: "Yes, a browser is currently active..." -``` - -## ๐ŸŽฏ Use Cases - -### 1. LinkedIn Automation -``` -Day 1: -- Navigate to LinkedIn -- Log in manually -- Session auto-saved - -Day 2-N: -- Navigate to LinkedIn -- Already logged in! โœ“ -- Continue automation -``` - -### 2. Multi-Account Management -``` -Work Account: -- Set session to "work" -- Login to LinkedIn -- Do work tasks - -Personal Account: -- Set session to "personal" -- Login to LinkedIn -- Do personal tasks -``` - -### 3. Long-Running Tasks -``` -- Open LinkedIn -- Read posts -- Close app -[Next day] -- Open LinkedIn -- Still logged in! Continue where you left off -``` - -## ๐Ÿ”„ Migration from Old Behavior - -**Old (Before):** -``` -You: "Navigate to LinkedIn feed" -AI: Opens browser (not logged in) -You: "I logged in, keep going" -AI: Takes screenshot (but session lost) -[Next command] -You: "Go to LinkedIn again" -AI: Opens browser (not logged in AGAIN) โŒ -``` - -**New (After):** -``` -You: "Navigate to LinkedIn feed" -AI: Opens browser (restores previous login if exists) -[If not logged in, login once] -You: "I logged in, keep going" -AI: Automatically saves session โœ“ -[Next command or next day] -You: "Go to LinkedIn again" -AI: Opens browser (ALREADY LOGGED IN!) โœ“ -``` - -## โš™๏ธ Configuration - -### Default Settings -``` -Session ID: "default" -Session Persistence: Enabled -Auto-Save: Enabled -``` - -### Changing Settings -``` -Change session: "Set session ID to my-session" -Enable persistence: "Enable session persistence" -Enable auto-save: "Enable auto-save" -``` - -### Session Files -Located in: `%APPDATA%\FlowVision\PlaywrightSessions\` - -File format: `{sessionId}.json` - -Example: -- `default.json` - Default session -- `work-account.json` - Work account session -- `personal.json` - Personal session - -## ๐ŸŽ‰ Benefits - -1. **Save Time** - No more logging in repeatedly -2. **Better UX** - Seamless continuation of tasks -3. **Multi-Account** - Easily switch between accounts -4. **Persistent State** - Survive app restarts -5. **Privacy** - Local storage, not cloud - -## ๐Ÿ“ Summary - -With automatic session persistence: -- โœ… **Log in once**, stay logged in forever -- โœ… **Sessions auto-save** after actions -- โœ… **Multiple accounts** supported -- โœ… **Survives restarts** of the app -- โœ… **Smart browser reuse** - no unnecessary relaunches - -**No more repeated logins! Your browser remembers you!** ๐ŸŽŠ - ---- - -For more Playwright features, see the Playwright plugin documentation. diff --git a/PLAYWRIGHT_SINGLETON_FIX.md b/PLAYWRIGHT_SINGLETON_FIX.md deleted file mode 100644 index 27c2495..0000000 --- a/PLAYWRIGHT_SINGLETON_FIX.md +++ /dev/null @@ -1,274 +0,0 @@ -# Playwright Singleton Pattern Fix - -## ๐Ÿ› Problem Identified - -**Issue:** Browser was launching fresh every time, even within the same conversation. - -**Root Cause:** The `PlaywrightPlugin` was being instantiated as a new object on every AI call: -```csharp -// OLD - Creates new instance each time -tools.AddRange(PluginToolExtractor.ExtractTools(new PlaywrightPlugin())); -``` - -This meant: -- `_browser` variable was null on every call -- No memory of previous browser instance -- Sessions couldn't be maintained -- User had to log in repeatedly - -## โœ… Solution: Singleton Pattern - -**Fix:** Implemented singleton pattern to maintain ONE instance across all calls: -```csharp -// NEW - Uses same instance every time -tools.AddRange(PluginToolExtractor.ExtractTools(PlaywrightPlugin.Instance)); -``` - -Now: -- โœ… Same browser instance persists -- โœ… Memory maintained across calls -- โœ… Sessions automatically reused -- โœ… Login once, stay logged in - -## ๐Ÿ”ง Technical Implementation - -### 1. Added Singleton Pattern to PlaywrightPlugin - -```csharp -internal class PlaywrightPlugin -{ - // Singleton instance - private static PlaywrightPlugin _instance; - private static readonly object _lock = new object(); - - // Instance variables persist across calls - private IPlaywright _playwright; - private IBrowser _browser; - private IBrowserContext _context; - private IPage _page; - // ... other state variables - - // Public static property - public static PlaywrightPlugin Instance - { - get - { - if (_instance == null) - { - lock (_lock) - { - if (_instance == null) - { - _instance = new PlaywrightPlugin(); - } - } - } - return _instance; - } - } - - // Private constructor - private PlaywrightPlugin() - { - } -} -``` - -### 2. Updated All Actioners - -**Files Modified:** -- `Actioner.cs` -- `MultiAgentActioner.cs` -- `LMStudioActioner.cs` - -**Change:** -```csharp -// Before -new PlaywrightPlugin() - -// After -PlaywrightPlugin.Instance -``` - -## ๐ŸŽฏ How It Works Now - -### First Call -``` -AI receives: "Navigate to LinkedIn" -โ†“ -PlaywrightPlugin.Instance (creates if null) -โ†“ -Launches browser โ†’ Navigates โ†’ Saves session -โ†“ -Browser remains open -``` - -### Second Call (Same Conversation) -``` -AI receives: "Take a screenshot" -โ†“ -PlaywrightPlugin.Instance (returns existing instance) -โ†“ -_browser != null โ†’ Uses existing browser! โœ“ -โ†“ -Takes screenshot โ†’ Auto-saves session -``` - -### Third Call (Different Conversation, Same App Session) -``` -AI receives: "Navigate to another page" -โ†“ -PlaywrightPlugin.Instance (returns existing instance) -โ†“ -_browser != null โ†’ Uses existing browser! โœ“ -_context has saved session โ†’ Already logged in! โœ“ -โ†“ -Navigates โ†’ Auto-saves session -``` - -## ๐Ÿ“Š Before vs After Comparison - -### Before (New Instance Each Time) -``` -Call 1: new PlaywrightPlugin() โ†’ _browser = null โ†’ Launch new browser -Call 2: new PlaywrightPlugin() โ†’ _browser = null โ†’ Launch new browser AGAIN โŒ -Call 3: new PlaywrightPlugin() โ†’ _browser = null โ†’ Launch new browser AGAIN โŒ -``` - -### After (Singleton Instance) -``` -Call 1: PlaywrightPlugin.Instance โ†’ _browser = null โ†’ Launch new browser -Call 2: PlaywrightPlugin.Instance โ†’ _browser exists โ†’ Use existing! โœ“ -Call 3: PlaywrightPlugin.Instance โ†’ _browser exists โ†’ Use existing! โœ“ -``` - -## ๐ŸŽ Benefits - -| Benefit | Description | -|---------|-------------| -| **Browser Reuse** | Same browser across multiple commands | -| **Session Persistence** | Login state maintained automatically | -| **Memory Efficiency** | Only one browser instance | -| **Better Performance** | No repeated browser launches | -| **Seamless UX** | User doesn't see multiple browsers | - -## ๐Ÿ” Thread Safety - -The singleton implementation is **thread-safe** using double-check locking: - -```csharp -if (_instance == null) // First check (fast path) -{ - lock (_lock) // Acquire lock - { - if (_instance == null) // Second check (synchronized) - { - _instance = new PlaywrightPlugin(); - } - } -} -``` - -This ensures: -- โœ… Only one instance ever created -- โœ… Thread-safe initialization -- โœ… Minimal locking overhead -- โœ… No race conditions - -## ๐Ÿ“– Usage Example - -### Scenario: LinkedIn Workflow - -**Command 1:** -``` -You: "Navigate to https://www.linkedin.com/feed/" -AI: Launches browser, navigates - [Browser stays open] -``` - -**Command 2 (User manually logs in):** -``` -You: "I logged in, take a screenshot" -AI: Uses SAME browser instance โœ“ - Takes screenshot - Auto-saves session with login -``` - -**Command 3:** -``` -You: "Click on the first post" -AI: Uses SAME browser instance โœ“ - Already logged in! โœ“ - Clicks element - Auto-saves session -``` - -**Command 4 (Later or different conversation):** -``` -You: "Go to LinkedIn" -AI: Uses SAME browser instance โœ“ - Already logged in! โœ“ - Navigates -``` - -## ๐Ÿ› Troubleshooting - -### "Still launching new browser" - -**Check:** -1. Make sure app is using the built version -2. Restart the application completely -3. Check browser is from singleton: - ``` - You: "What's the browser status?" - AI: Should show "BrowserActive: Yes" - ``` - -### "Want to force a new browser" - -**Solution:** -``` -You: "Close the browser" -[Then next command will launch fresh] - -Or: - -You: "Launch browser, force new" -[Explicitly requests new browser] -``` - -### "Singleton not working across app restarts" - -**Expected Behavior:** -- Singleton persists: **Within app session** โœ“ -- Sessions persist: **Across app restarts** โœ“ -- Browser instance: **Does NOT survive app restart** (normal) - -When app restarts: -- New singleton instance created -- Browser will launch fresh -- BUT saved sessions are restored! โœ“ - -## ๐ŸŽฏ Key Points - -1. **Singleton = Same Instance** - One PlaywrightPlugin instance for entire app lifetime -2. **Auto-Save = Sessions Persist** - Login states saved across app restarts -3. **Combined Effect** - Browser reused during session, login restored on restart - -## โœจ Summary - -The singleton pattern ensures: -- โœ… **One browser instance** during app runtime -- โœ… **State maintained** across multiple AI calls -- โœ… **Sessions saved** automatically -- โœ… **Logins persist** across app restarts -- โœ… **Better performance** - no repeated launches -- โœ… **Seamless UX** - feels like natural browsing - -**Result:** Log in once, browse naturally, everything just works! ๐ŸŽ‰ - ---- - -**Build Status:** โœ… Compiled successfully -**Files Modified:** 4 (PlaywrightPlugin.cs, Actioner.cs, MultiAgentActioner.cs, LMStudioActioner.cs) -**Ready to Use:** YES diff --git a/QUICK_FIX_SUMMARY.md b/QUICK_FIX_SUMMARY.md deleted file mode 100644 index 6ceafc1..0000000 --- a/QUICK_FIX_SUMMARY.md +++ /dev/null @@ -1,220 +0,0 @@ -# โœ… GitHub Pages Build Fixed! - -## ๐Ÿ”ง What Was Wrong - -The GitHub Pages build failed with: -``` -Error: No such file or directory @ dir_chdir0 - /github/workspace/docs -``` - -**Root Cause:** Incorrect theme configuration and missing YAML front matter. - ---- - -## โœ… What I Fixed - -### 1. **Fixed `_config.yml`** - -Changed from: -```yaml -theme: jekyll-theme-cayman # โŒ Doesn't work with GitHub Actions -``` - -To: -```yaml -remote_theme: pages-themes/cayman@v0.2.0 # โœ… Works! -``` - -### 2. **Added YAML Front Matter** - -Every documentation file now has: -```yaml ---- -layout: default -title: Page Title ---- -``` - -### 3. **Used Only Whitelisted Plugins** - -```yaml -plugins: - - jekyll-remote-theme - - jekyll-seo-tag - - jekyll-sitemap -``` - ---- - -## ๐Ÿš€ Next Steps - -### 1. Commit & Push - -```bash -git add docs/ -git add GITHUB_PAGES_FIX.md -git add QUICK_FIX_SUMMARY.md -git commit -m "Fix GitHub Pages build configuration" -git push origin main -``` - -### 2. Check Build - -1. Go to **Actions** tab on GitHub -2. Wait for build to complete (2-3 minutes) -3. Should see โœ… green checkmark - -### 3. Visit Your Site - -``` -https://flowdevs-io.github.io/Recursive-Control/ -``` - ---- - -## ๐Ÿ“Š What's Ready - -โœ… **13 Documentation Files** -- index.md (Home page) -- Installation.md -- Getting-Started.md -- Multi-Agent-Architecture.md -- API-Reference.md -- FAQ.md -- Troubleshooting.md -- + 6 reference documents - -โœ… **Professional Theme** -- Cayman theme (gradient header) -- Code syntax highlighting -- Mobile responsive -- Beautiful typography - -โœ… **Full Navigation** -- All internal links work -- Previous/Next navigation -- Table of contents -- Section anchors - -โœ… **78,000+ Words** -- Complete documentation -- Step-by-step guides -- Code examples -- Troubleshooting - ---- - -## ๐ŸŽฏ Files Modified - -``` -T:\Recursive-Control\docs\ -โ”œโ”€โ”€ _config.yml # โœ… Fixed theme & plugins -โ”œโ”€โ”€ index.md # โœ… Added front matter -โ”œโ”€โ”€ Installation.md # โœ… Added front matter -โ”œโ”€โ”€ Getting-Started.md # โœ… Added front matter -โ”œโ”€โ”€ Multi-Agent-Architecture.md # โœ… Added front matter -โ”œโ”€โ”€ API-Reference.md # โœ… Added front matter -โ”œโ”€โ”€ FAQ.md # โœ… Added front matter -โ””โ”€โ”€ Troubleshooting.md # โœ… Added front matter -``` - ---- - -## โœ… Build Should Now Work! - -The configuration is now correct for GitHub Pages. The build will: - -1. โœ… Use remote theme (GitHub compatible) -2. โœ… Process all markdown with YAML front matter -3. โœ… Apply Cayman theme styling -4. โœ… Generate navigation -5. โœ… Create SEO tags -6. โœ… Build sitemap -7. โœ… Deploy to GitHub Pages - ---- - -## ๐Ÿ” How to Verify - -### After Pushing: - -**Actions Tab:** -``` -โœ… pages build and deployment - โ””โ”€ build - โœ… Building your site... - โœ… Deploying to GitHub Pages... - โ””โ”€ deploy - โœ… Published successfully -``` - -**Visit Site:** -- Homepage loads -- Navigation works -- Theme applied -- Code blocks highlighted - ---- - -## ๐Ÿ’ก Why This Fix Works - -### Before (Broken): -```yaml -# Local theme - doesn't work in GitHub Actions -theme: jekyll-theme-cayman - -# No YAML front matter -# Just starts with # Heading -``` - -### After (Working): -```yaml -# Remote theme - loads from GitHub -remote_theme: pages-themes/cayman@v0.2.0 - -# YAML front matter on every page ---- -layout: default -title: Page Title ---- -``` - -**The difference:** -- `theme:` expects local gem installation -- `remote_theme:` fetches from GitHub -- GitHub Actions doesn't have local gems, needs remote - ---- - -## ๐ŸŽ‰ Summary - -**Issue:** GitHub Pages build failed -**Cause:** Wrong theme configuration + missing front matter -**Fix:** Use `remote_theme` + add YAML front matter -**Status:** โœ… **FIXED** - Ready to build! - -**Your documentation will now build successfully and publish to GitHub Pages!** ๐Ÿš€๐Ÿ“š - ---- - -## ๐Ÿ“ž If Still Not Working - -1. **Check Error Message** - - Go to Actions tab - - Click failed build - - Read full error log - - Share in issue if needed - -2. **Verify Settings** - - Settings โ†’ Pages - - Source: "Deploy from a branch" - - Branch: `main` - - Folder: `/docs` - -3. **Common Issues** - - Wrong branch selected - - Wrong folder selected - - Permissions issue - - Wait 5 minutes (initial builds slow) - -But it **should work now**! The configuration is correct. โœ… diff --git a/README_MIGRATION.md b/README_MIGRATION.md deleted file mode 100644 index 2b128c5..0000000 --- a/README_MIGRATION.md +++ /dev/null @@ -1,276 +0,0 @@ -# Semantic Kernel โ†’ Microsoft Agent Framework Migration - -## โœ… Status: COMPLETE AND READY TO TEST - -This document provides a quick reference for the completed migration from Semantic Kernel to Microsoft Agent Framework (Microsoft.Extensions.AI). - ---- - -## ๐ŸŽฏ Quick Summary - -**What Changed:** The entire project now uses Microsoft.Extensions.AI instead of Semantic Kernel -**Build Status:** โœ… Successful -**Runtime Status:** โœ… Fixed (using `.AsIChatClient()` extension) -**Files Changed:** 17 files modified/created -**Documentation:** 4 comprehensive guides created - ---- - -## ๐Ÿ”ง Key Technical Changes - -### Before (Semantic Kernel) -```csharp -// Complex Kernel builder pattern -var builder = Kernel.CreateBuilder(); -builder.AddAzureOpenAIChatCompletion(deploymentName, endpoint, apiKey); -builder.Plugins.AddFromType(); -var kernel = builder.Build(); -var chat = kernel.GetRequiredService(); - -// Plugin with [KernelFunction] attribute -[KernelFunction, Description("Execute command")] -public async Task ExecuteCommand(string command) { } -``` - -### After (Microsoft.Extensions.AI) -```csharp -// Direct client creation -var azureClient = new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(apiKey)); -IChatClient chat = azureClient.GetChatClient(deploymentName).AsIChatClient(); - -// Add function invocation -chat = new ChatClientBuilder(chat).UseFunctionInvocation().Build(); - -// Add tools easily -var tools = PluginToolExtractor.ExtractTools(new CMDPlugin()); - -// Plugin with standard [Description] attribute -[Description("Execute command")] -public async Task ExecuteCommand(string command) { } -``` - ---- - -## ๐Ÿ“ฆ Package Changes - -### Active Packages (Using) -- โœ… **Microsoft.Extensions.AI** (9.4.0-preview.1.25207.5) -- โœ… **Microsoft.Extensions.AI.Abstractions** (9.4.0-preview.1.25207.5) -- โœ… **Microsoft.Extensions.AI.AzureAIInference** (9.4.0-preview.1.25207.5) -- โœ… **Microsoft.Extensions.AI.OpenAI** (9.4.0-preview.1.25207.5) โ† **Critical for .AsIChatClient()** -- โœ… **Azure.AI.OpenAI** (2.2.0-beta.4) -- โœ… **OpenAI** (2.2.0-beta.4) - -### Legacy Packages (Can Remove After Testing) -- โš ๏ธ Microsoft.SemanticKernel (1.47.0) -- โš ๏ธ Microsoft.SemanticKernel.Abstractions (1.47.0) -- โš ๏ธ Microsoft.SemanticKernel.Core (1.47.0) -- โš ๏ธ Microsoft.SemanticKernel.Connectors.AzureOpenAI (1.47.0) -- โš ๏ธ Microsoft.SemanticKernel.Connectors.OpenAI (1.47.0) - ---- - -## ๐Ÿšจ Important Fix: Runtime Cast Error - -### The Problem -``` -Error: Unable to cast object of type 'Azure.AI.OpenAI.Chat.AzureChatClient' -to type 'Microsoft.Extensions.AI.IChatClient'. -``` - -### The Solution -**Use `.AsIChatClient()` extension method instead of casting!** - -```csharp -// โŒ DON'T DO THIS (fails at runtime): -IChatClient chat = (IChatClient)azureClient.GetChatClient(deploymentName); - -// โœ… DO THIS (correct pattern): -IChatClient chat = azureClient.GetChatClient(deploymentName).AsIChatClient(); -``` - -**Why:** The `.AsIChatClient()` extension from `Microsoft.Extensions.AI.OpenAI` creates a proper wrapper that implements the IChatClient interface. - ---- - -## ๐Ÿ“ Files Modified - -### Core AI Classes (3) -1. **Actioner.cs** - Main single-agent executor -2. **MultiAgentActioner.cs** - Multi-agent coordinator -3. **Github_Actioner.cs** - GitHub Models integration - -### Plugins (9) -1. CMDPlugin.cs -2. KeyboardPlugin.cs -3. MousePlugin.cs -4. PlaywrightPlugin.cs -5. PowershellPlugin.cs -6. RemoteControlPlugin.cs -7. ScreenCaptureOmniParserPlugin.cs -8. ScreenCapturePlugin.cs -9. WindowSelectionPlugin.cs - -### Support Classes (2) -1. **PluginToolExtractor.cs** (NEW) - Helper to extract tools from plugins -2. **ToolDescriptionGenerator.cs** - Removed SK dependencies - -### UI & Config (3) -1. **Form1.cs** - Updated namespaces -2. **FlowVision.csproj** - Added package references, LangVersion -3. **packages.config** - Added Extensions.AI packages - ---- - -## ๐Ÿ“š Documentation Files - -1. **MIGRATION_SUMMARY.md** - Complete API migration guide with before/after patterns -2. **MIGRATION_STATUS.md** - Detailed completion report with technical solutions -3. **MIGRATION_COMPLETE.md** - Success summary and testing checklist -4. **RUNTIME_FIX.md** - Specific fix for the cast error -5. **README_MIGRATION.md** (this file) - Quick reference guide - ---- - -## ๐Ÿงช Testing Checklist - -### Before Testing -- [ ] Close any running FlowVision.exe instances -- [ ] Rebuild the solution completely -- [ ] Verify no build errors (only pre-existing warnings are OK) - -### Functional Tests -- [ ] Test Actioner (single-agent execution) -- [ ] Test MultiAgentActioner (multi-agent workflow) -- [ ] Test Github_Actioner (GitHub Models) -- [ ] Test each plugin: - - [ ] CMDPlugin - - [ ] PowerShellPlugin - - [ ] KeyboardPlugin - - [ ] MousePlugin - - [ ] ScreenCapturePlugin - - [ ] WindowSelectionPlugin - - [ ] PlaywrightPlugin - - [ ] RemoteControlPlugin - -### Integration Tests -- [ ] Verify tool calling works -- [ ] Verify auto-invocation works -- [ ] Verify streaming responses work -- [ ] Verify chat history persists correctly -- [ ] Verify configuration loading works - ---- - -## ๐ŸŽ New Features/Improvements - -### 1. PluginToolExtractor Utility -A new helper class that automatically extracts tools from plugin instances: - -```csharp -// Simple, clean usage: -var tools = new List(); -tools.AddRange(PluginToolExtractor.ExtractTools(new CMDPlugin())); -``` - -### 2. Cleaner Plugin Architecture -Plugins now use standard .NET attributes: -- No more `[KernelFunction]` dependency -- Just `[Description]` attribute -- More portable and maintainable - -### 3. Modern Async Patterns -- Using `await foreach` with `GetStreamingResponseAsync` -- C# latest language features enabled -- Cleaner async/await code - -### 4. Simplified Client Creation -- Direct client instantiation -- No complex Kernel builders -- Easier to understand and maintain - ---- - -## ๐Ÿ’ก Benefits Achieved - -1. **Simpler Codebase** - No more Kernel complexity -2. **Standard .NET** - Using standard attributes and patterns -3. **Better Maintainability** - Cleaner, more modern code -4. **Future-Proof** - Using Microsoft's recommended AI abstractions -5. **Provider Agnostic** - Easy to swap AI providers -6. **Lighter Dependencies** - Fewer packages needed - ---- - -## ๐Ÿš€ How to Rebuild and Test - -### 1. Close Running App -Make sure FlowVision.exe is not running - -### 2. Rebuild Solution -```powershell -cd T:\Recursive-Control -& "C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin\MSBuild.exe" FlowVision.sln /t:Rebuild /p:Configuration=Debug -``` - -### 3. Run Application -```powershell -.\FlowVision\bin\Debug\FlowVision.exe -``` - -### 4. Test Functionality -- Configure API keys in settings -- Test a simple command (e.g., "What is 2+2?") -- Test a tool-calling command (e.g., "Open notepad") -- Verify streaming responses work - ---- - -## ๐Ÿ› Troubleshooting - -### Issue: "Unable to cast to IChatClient" -**Solution:** Make sure you're using `.AsIChatClient()` not casting. See RUNTIME_FIX.md - -### Issue: Build errors about Microsoft.Extensions.AI.OpenAI -**Solution:** The package reference was added. Try cleaning and rebuilding. - -### Issue: Tool calling doesn't work -**Solution:** Verify that `UseFunctionInvocation()` is being called in the ChatClientBuilder - -### Issue: Plugins not found -**Solution:** Check that PluginToolExtractor is properly extracting methods from plugin instances - ---- - -## ๐Ÿ“ž Support & References - -### Documentation -- Read MIGRATION_SUMMARY.md for detailed API patterns -- Read RUNTIME_FIX.md for the cast error solution -- Read MIGRATION_STATUS.md for technical details - -### External Resources -- [Microsoft.Extensions.AI Documentation](https://learn.microsoft.com/dotnet/ai/) -- [Microsoft Agent Framework GitHub](https://github.com/microsoft/agent-framework) -- [OpenAI SDK for .NET](https://github.com/openai/openai-dotnet) -- [Azure OpenAI Documentation](https://learn.microsoft.com/azure/ai-services/openai/) - ---- - -## โœจ Summary - -The migration from Semantic Kernel to Microsoft Agent Framework is **COMPLETE**! - -The application: -- โœ… Builds successfully -- โœ… Uses modern Microsoft.Extensions.AI APIs -- โœ… Has cleaner, more maintainable code -- โœ… Is ready for testing - -**Next step:** Test the application to verify all functionality works as expected! - ---- - -*Migration completed: October 2, 2025* -*Build status: โœ… SUCCESS* -*Ready for testing: โœ… YES* diff --git a/README_OCR_INTEGRATION.md b/README_OCR_INTEGRATION.md deleted file mode 100644 index ea8eb87..0000000 --- a/README_OCR_INTEGRATION.md +++ /dev/null @@ -1,148 +0,0 @@ -# OCR Integration - README - -## ๐ŸŽ‰ SUCCESS! Tesseract OCR 5.2.0 is now integrated! - -**Date**: October 2, 2025 -**Status**: โœ… COMPLETE AND OPERATIONAL - ---- - -## What Happened? - -The ONNX OmniParser can now **extract and read text** from detected UI elements using Tesseract OCR! - -### Before -``` -"Element 171" -"Element 172" -"Element 173" -``` -โŒ **No meaning** - AI had no idea what these were - -### After -``` -"Play Video at (150,200) [size: 120x40]" -"Subscribe Button at (300,250) [size: 200x60]" -"YouTube Logo at (450,300) [size: 180x50]" -``` -โœ… **Rich meaning** - AI understands exactly what each element is! - ---- - -## Quick Start - -### 1. Verify Installation -```powershell -.\test_ocr_simple.ps1 -``` - -Expected: โœ… All prerequisites satisfied! - -### 2. Launch Application -``` -FlowVision\bin\Debug\FlowVision.exe -``` - -### 3. Check Logs -Look for this message: -``` -"โœ“ Tesseract OCR initialized successfully. Text extraction is now enabled." -``` - -### 4. Test It Out -1. Open OmniParser screen capture tool -2. Capture a screenshot with text -3. Watch OCR extract text in real-time! - ---- - -## Documentation - -Pick what you need: - -### ๐Ÿš€ Quick Start -- **[OCR_QUICK_REFERENCE.md](OCR_QUICK_REFERENCE.md)** - One page, all essentials - -### ๐Ÿ“‹ Overview -- **[OCR_INTEGRATION_COMPLETE.md](OCR_INTEGRATION_COMPLETE.md)** - What changed and why - -### ๐Ÿ”ง Technical -- **[OCR_TEXT_EXTRACTION_STATUS.md](OCR_TEXT_EXTRACTION_STATUS.md)** - Complete technical docs -- **[TEST_OCR.md](TEST_OCR.md)** - Implementation details - -### ๐Ÿ“Š Reports -- **[TASK_COMPLETE_OCR_INTEGRATION.md](TASK_COMPLETE_OCR_INTEGRATION.md)** - Full task report -- **[OCR_INTEGRATION_CHECKLIST.md](OCR_INTEGRATION_CHECKLIST.md)** - Verification checklist - -### ๐Ÿ› ๏ธ Tools -- **[test_ocr_simple.ps1](test_ocr_simple.ps1)** - Prerequisites checker - ---- - -## Key Benefits - -โœ… **AI can READ** - Extracts actual text from UI elements -โœ… **Better accuracy** - 40% โ†’ 90% automation success rate -โœ… **Natural commands** - "Click Save button" works reliably -โœ… **Verification** - AI can confirm actions by reading results - ---- - -## What Changed? - -### Modified (3 files) -1. `FlowVision/FlowVision.csproj` - Added Tesseract -2. `FlowVision/lib/Classes/OcrHelper.cs` - Full implementation -3. `OCR_TEXT_EXTRACTION_STATUS.md` - Updated status - -### Added (~11 MB dependencies) -- Tesseract OCR engine (native DLLs) -- English language model -- Build automation - ---- - -## Build Status - -โœ… **Compilation**: SUCCESS (0 errors) -โœ… **Dependencies**: ALL DEPLOYED -โœ… **OCR**: OPERATIONAL -โœ… **Ready**: TO USE - ---- - -## Troubleshooting - -### OCR not working? -```powershell -# Check prerequisites -.\test_ocr_simple.ps1 - -# Look for missing files -``` - -### Need help? -Check the documentation files above, especially: -- OCR_QUICK_REFERENCE.md for quick answers -- OCR_TEXT_EXTRACTION_STATUS.md for technical details - ---- - -## Summary - -**Task**: Enable OCR text extraction from UI elements -**Result**: โœ… COMPLETE -**Technology**: Tesseract 5.2.0 + ONNX YOLO -**Impact**: AI can now see AND read the screen! - ---- - -## Next Steps - -๐Ÿš€ **Launch FlowVision and start using OCR today!** - -The AI now has semantic understanding of UI elements - dramatically improving automation accuracy and user experience! - ---- - -**Questions? Check the documentation files listed above!** diff --git a/README_UPDATE_DOCS.md b/README_UPDATE_DOCS.md deleted file mode 100644 index 9ff8013..0000000 --- a/README_UPDATE_DOCS.md +++ /dev/null @@ -1,120 +0,0 @@ -# Update Main README.md with Documentation Links - -Add this section to your main README.md file to link to all documentation: - -```markdown -## ๐Ÿ“š Documentation - -### Getting Started -- **[Installation Guide](docs/Installation.md)** - Complete setup instructions -- **[Getting Started Tutorial](docs/Getting-Started.md)** - Your first tasks -- **[Multi-Agent Architecture](docs/Multi-Agent-Architecture.md)** - How the 3-agent system works - -### Configuration -- **[AI Provider Setup](docs/Installation.md#configure-ai-provider)** - Azure, LM Studio, GitHub Models -- **[Plugin Configuration](docs/Installation.md#configure-plugins)** - Enable/disable features -- **[Multi-Agent Mode](docs/Multi-Agent-Architecture.md#configuration)** - Complex task handling - -### Reference -- **[FAQ](docs/FAQ.md)** - Frequently asked questions -- **[Troubleshooting](docs/Troubleshooting.md)** - Common issues and solutions -- **[API Reference](docs/API-Reference.md)** - Developer documentation - -### Blog & Updates -- **[Version 2.0 Release Post](docs/Blog-Post-v2.0.md)** - Major upgrade announcement -- **[System Prompts Reference](docs/System-Prompts-Reference.md)** - Complete prompt library -- **[UI Features](docs/UI-Features.md)** - New interface improvements -- **[UI Redesign](docs/UI-Redesign.md)** - Modern menu structure - ---- - -## ๐ŸŒ Online Documentation - -**GitHub Pages** (Recommended): https://flowdevs-io.github.io/Recursive-Control/ - -*Enable GitHub Pages in Settings โ†’ Pages โ†’ Source: `/docs` folder* - ---- -``` - -## Alternative: Simple TOC - -If you want a minimal approach: - -```markdown -## ๐Ÿ“– Documentation - -๐Ÿ“ฅ [Installation](docs/Installation.md) โ€ข -๐Ÿš€ [Getting Started](docs/Getting-Started.md) โ€ข -๐Ÿค– [Multi-Agent System](docs/Multi-Agent-Architecture.md) โ€ข -๐Ÿ”ง [API Reference](docs/API-Reference.md) - -**More:** [FAQ](docs/FAQ.md) | [Troubleshooting](docs/Troubleshooting.md) | [v2.0 Release Notes](docs/Blog-Post-v2.0.md) -``` - -## Alternative: Detailed TOC - -If you want full visibility: - -```markdown -## ๐Ÿ“š Complete Documentation - -### ๐Ÿš€ Quick Start -1. [Installation Guide](docs/Installation.md) - Download, install, configure -2. [First-Time Setup](docs/Installation.md#initial-setup) - AI provider and plugins -3. [Your First Task](docs/Getting-Started.md#your-first-task) - Test the system - -### ๐Ÿ“– Core Guides -- **[Getting Started](docs/Getting-Started.md)** - Tutorials and examples - - Opening applications - - Window management - - Keyboard control - - Taking screenshots - - Browser automation - -- **[Multi-Agent Architecture](docs/Multi-Agent-Architecture.md)** - How it works - - Coordinator agent (routing) - - Planner agent (breakdown) - - Actioner agent (execution) - - When to use multi-agent mode - -### โš™๏ธ Configuration -- **AI Providers**: Configure [Azure OpenAI](docs/Installation.md#option-a-azure-openai), [LM Studio](docs/Installation.md#option-b-lm-studio), or [GitHub Models](docs/Installation.md#option-c-github-models) -- **Plugins**: Enable features in [Plugin Configuration](docs/Installation.md#step-2-configure-plugins) -- **Multi-Agent Mode**: Toggle in [Setup menu](docs/Multi-Agent-Architecture.md#enabledisable-multi-agent-mode) - -### ๐Ÿ”ง Developer Resources -- **[API Reference](docs/API-Reference.md)** - Plugin API and extension guide -- **[System Prompts](docs/System-Prompts-Reference.md)** - Complete prompt library -- **[Plugin Development](docs/API-Reference.md)** - Create custom tools - -### ๐Ÿ†˜ Support -- **[FAQ](docs/FAQ.md)** - Common questions -- **[Troubleshooting](docs/Troubleshooting.md)** - Problem solving -- **[Discord Community](https://discord.gg/mQWsWeHsVU)** - Get help -- **[Report Issue](https://github.com/flowdevs-io/Recursive-Control/issues)** - Bug reports - -### ๐Ÿ“ฐ Latest Updates -- **[Version 2.0 Release](docs/Blog-Post-v2.0.md)** - From Good to Great - - Complete system prompt rewrite (800+ lines) - - Window-targeted keyboard/mouse control - - ONNX auto-initialization - - Multi-agent improvements (25 steps) - - Chat export in 4 formats - - Modern emoji-enhanced UI - -- **[UI Improvements](docs/UI-Features.md)** - New interface features - - Chat export system - - Execution visualizer - - Activity monitor - -- **[UI Redesign](docs/UI-Redesign.md)** - Modern menu structure - - All 4 AI agents accessible - - Per-agent model configuration - - Multi-agent mode toggle - - Emoji visual language -``` - ---- - -Choose the style that fits your README best! diff --git a/READY_TO_GO.md b/READY_TO_GO.md index ef53b9b..e69de29 100644 --- a/READY_TO_GO.md +++ b/READY_TO_GO.md @@ -1,232 +0,0 @@ -# ๐Ÿš€ READY TO GO - Final Steps - -## โœ… What's Been Completed - -I've successfully simplified your OmniParser implementation following KISS principles: - -- โœ… Created `SimpleOmniParser.cs` - single, focused class -- โœ… Simplified `ScreenCaptureOmniParserPlugin.cs` - no more complex server logic -- โœ… Created setup automation scripts -- โœ… Created documentation -- โœ… Created Python conversion script - -**Result**: 70% less code, no servers, no complexity, should no longer freeze! - -## โš ๏ธ ONE THING LEFT: Get the ONNX Model - -The official OmniParser uses PyTorch format. You need ONNX for .NET. - -### Quick Option: Download My Pre-Converted Model - -I'll convert it for you if you need. For now, here are your options: - -### Option 1: I Have Python Installed โœ… - -```powershell -# 1. Install dependencies -pip install torch ultralytics huggingface-hub - -# 2. Download the PyTorch model -huggingface-cli download microsoft/OmniParser-v2.0 icon_detect/model.pt --local-dir weights - -# 3. Run the conversion script I created -python convert_omniparser_to_onnx.py - -# Done! The ONNX model will be at FlowVision/models/icon_detect.onnx -``` - -### Option 2: I Don't Have Python โŒ - -**Temporary Solution**: Use safetensors model directly - -The model is also available as `model.safetensors`. While not ideal, we can load it: - -```powershell -# Download safetensors version (works with some .NET libraries) -$url = "https://huggingface.co/microsoft/OmniParser/resolve/main/icon_detect/model.safetensors" -$output = ".\FlowVision\models\icon_detect.safetensors" - -Invoke-WebRequest -Uri $url -OutFile $output -``` - -Then I can update SimpleOmniParser to also support safetensors. - -### Option 3: Find Pre-converted ONNX - -Someone may have already converted it. Check: -- GitHub issues/discussions for OmniParser -- Community model zoos -- Alternative repos - -### Option 4: I'll Do It For You ๐Ÿค - -If you provide me with access to the PyTorch model, I can convert it and give you the ONNX file directly. - -## Once You Have the ONNX Model - -### Step 1: Place the Model - -```powershell -# Put it here: -FlowVision/models/icon_detect.onnx -``` - -### Step 2: Embed in Executable (Recommended) - -1. Open FlowVision project in Visual Studio -2. Right-click `models/icon_detect.onnx` -3. Properties โ†’ **Build Action: Embedded Resource** -4. Save - -### Step 3: Build - -```powershell -# In Visual Studio: Build โ†’ Build Solution -# Or via command line: -msbuild FlowVision.sln /p:Configuration=Release -``` - -### Step 4: Test - -```powershell -# Run the test script -.\test_simple_omniparser.ps1 - -# Should show: -# [โœ“] Model found -# [โœ“] All checks passed! -``` - -### Step 5: Run FlowVision - -```powershell -.\FlowVision\bin\Release\FlowVision.exe -``` - -Try capturing a screen - should see: -``` -[2025-10-02 22:50:23] TASK START: OmniParser -[2025-10-02 22:50:23] Info: SimpleOmniParser, Initialize, โœ“ Model loaded successfully -[2025-10-02 22:50:23] Info: Detected X UI elements -[2025-10-02 22:50:23] TASK COMPLETE: OmniParser -``` - -**NO MORE**: Server startup, delays, freezing! ๐ŸŽ‰ - -## What Changed in Your Code - -### Before (Complex) -```csharp -// Multiple mode detection -if (_useOnnxMode) { - if (_onnxEngine == null) ConfigureMode(true); - if (_onnxEngine != null) { - var result = _onnxEngine.ParseImageBase64(base64); - return ConvertOnnxResultToParsedContent(result); - } -} -// Fall back to HTTP server -await LocalOmniParserManager.EnsureServerRunningAsync(); -// ...more complexity -``` - -### After (Simple) -```csharp -// That's it! -var elements = SimpleOmniParser.Instance.ParseScreenshot(screenshot); -return ConvertToLegacyFormat(elements); -``` - -## Expected Behavior - -### First Screen Capture -``` -[22:50:23.105] Plugin: ScreenCaptureOmniParserPlugin, Method: CaptureWholeScreen -[22:50:23.150] Info: SimpleOmniParser, Initialize, Loading ONNX model... -[22:50:23.650] Info: SimpleOmniParser, Initialize, โœ“ Model loaded successfully -[22:50:23.710] Info: SimpleOmniParser, PostProcess, Detected 161 UI elements -[22:50:23.722] TASK COMPLETE: OmniParser - -Total time: ~600ms (includes model loading) -``` - -### Subsequent Captures -``` -[22:50:25.105] Plugin: ScreenCaptureOmniParserPlugin, Method: CaptureWholeScreen -[22:50:25.305] Info: SimpleOmniParser, PostProcess, Detected 145 UI elements -[22:50:25.310] TASK COMPLETE: OmniParser - -Total time: ~200ms (model already loaded) -``` - -**No server messages, no delays, no freezing!** โœจ - -## Troubleshooting - -### If It Still Freezes - -1. **Check the logs** - what line is it stuck on? -2. **Verify model size** - should be 6-50MB -3. **Check memory** - ONNX Runtime needs ~150MB -4. **Test smaller screenshot** - try 640x480 first - -### If Model Won't Load - -1. **Check file exists**: `FlowVision/models/icon_detect.onnx` -2. **Check file size**: Should be > 1MB -3. **Try external file first**: Don't embed until it works -4. **Check logs**: What's the exact error? - -### If Detection Is Wrong - -1. **Model version**: Make sure it's the icon_detect model -2. **Input size**: Should auto-resize to 640x640 -3. **Confidence**: Default threshold is 0.05 (5%) - -## Files You Can Remove (Optional Cleanup) - -Once everything works, you can delete these legacy files: - -``` -FlowVision/lib/Classes/ - - OnnxOmniParserEngine.cs - - LocalOmniParserManager.cs - - OmniParserClient.cs - - OmniParserConfig.cs - -FlowVision/ - - OmniParserForm.cs - - OmniParserForm.Designer.cs - - OmniParserForm.resx - -Root/ - - download_omniparser_model.ps1 (replaced) -``` - -But keep them for now as backup! - -## Next Steps - -1. **Get the ONNX model** (choose an option above) -2. **Place at** `FlowVision/models/icon_detect.onnx` -3. **Build** the project -4. **Test** with `.\test_simple_omniparser.ps1` -5. **Run** FlowVision and try screen capture -6. **Enjoy** the speed and simplicity! ๐Ÿš€ - -## Need Help? - -I'm here! Just ask: -- "How do I convert the model?" -- "Can you explain the safetensors option?" -- "How do I enable GPU acceleration?" -- "Can you help debug if it's not working?" - ---- - -**The implementation is done - just need the ONNX model and you're ready to rock!** ๐ŸŽธ - -**Before**: Complex, slow, freezes -**After**: Simple, fast, reliable - -**This is KISS in action!** ๐Ÿ˜Š diff --git a/RUNTIME_FIX.md b/RUNTIME_FIX.md deleted file mode 100644 index 7a0e849..0000000 --- a/RUNTIME_FIX.md +++ /dev/null @@ -1,103 +0,0 @@ -# Runtime Cast Error Fix - -## Problem -When running the migrated application, you encountered: -``` -Error: Unable to cast object of type 'Azure.AI.OpenAI.Chat.AzureChatClient' to type 'Microsoft.Extensions.AI.IChatClient'. -``` - -## Root Cause -The `AzureOpenAIClient.GetChatClient()` method returns an `Azure.AI.OpenAI.Chat.AzureChatClient` (or `OpenAI.Chat.ChatClient`) which doesn't directly implement `Microsoft.Extensions.AI.IChatClient`. - -We were trying to use an explicit cast `(IChatClient)` which fails at runtime because there's no direct inheritance relationship. - -## Solution -Use the `.AsIChatClient()` extension method provided by the `Microsoft.Extensions.AI.OpenAI` package instead of casting. - -### Changes Made - -#### 1. Added Package Reference -**File:** `FlowVision.csproj` - -Added the missing reference to Microsoft.Extensions.AI.OpenAI: -```xml - - ..\packages\Microsoft.Extensions.AI.OpenAI.9.4.0-preview.1.25207.5\lib\net462\Microsoft.Extensions.AI.OpenAI.dll - -``` - -#### 2. Updated Actioner.cs -**Before:** -```csharp -var azureClient = new AzureOpenAIClient(new Uri(config.EndpointURL), new AzureKeyCredential(config.APIKey)); -IChatClient baseChatClient = (IChatClient)azureClient.GetChatClient(config.DeploymentName); -``` - -**After:** -```csharp -var azureClient = new AzureOpenAIClient(new Uri(config.EndpointURL), new AzureKeyCredential(config.APIKey)); -IChatClient baseChatClient = azureClient.GetChatClient(config.DeploymentName).AsIChatClient(); -``` - -#### 3. Updated MultiAgentActioner.cs -**Before:** -```csharp -coordinatorChat = (IChatClient)coordinatorAzureClient.GetChatClient(coordinatorConfig.DeploymentName); -plannerChat = (IChatClient)plannerAzureClient.GetChatClient(plannerConfig.DeploymentName); -IChatClient actionerChatBase = (IChatClient)actionerAzureClient.GetChatClient(actionerConfig.DeploymentName); -``` - -**After:** -```csharp -coordinatorChat = coordinatorAzureClient.GetChatClient(coordinatorConfig.DeploymentName).AsIChatClient(); -plannerChat = plannerAzureClient.GetChatClient(plannerConfig.DeploymentName).AsIChatClient(); -IChatClient actionerChatBase = actionerAzureClient.GetChatClient(actionerConfig.DeploymentName).AsIChatClient(); -``` - -## How AsIChatClient() Works - -The `AsIChatClient()` extension method is provided by `Microsoft.Extensions.AI.OpenAI` and creates a wrapper around the OpenAI SDK's ChatClient that implements the `IChatClient` interface. - -This is the **correct and recommended way** to convert OpenAI SDK chat clients to the Microsoft.Extensions.AI abstraction. - -## Package Details - -- **Package:** Microsoft.Extensions.AI.OpenAI -- **Version:** 9.4.0-preview.1.25207.5 -- **Purpose:** Provides integration between OpenAI SDK and Microsoft.Extensions.AI -- **Key Extension:** `.AsIChatClient()` for OpenAI.Chat.ChatClient and Azure.AI.OpenAI.Chat.AzureChatClient - -## Verification - -โœ… **Build Status:** Successfully compiles -โœ… **Runtime Status:** Ready for testing with `.AsIChatClient()` - -## Testing Steps - -To verify the fix works: - -1. **Close the running application** (if any) -2. **Rebuild the solution:** - ```powershell - cd T:\Recursive-Control - & "C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Current\Bin\MSBuild.exe" FlowVision.sln /t:Rebuild /p:Configuration=Debug - ``` -3. **Run the application** -4. **Test an AI action** to verify the chat client works correctly - -## Additional Notes - -- The package was already in `packages.config` but wasn't referenced in the `.csproj` -- This is a common issue when manually managing package references in .NET Framework projects -- The `.AsIChatClient()` extension method is the **official recommended pattern** from Microsoft - -## Related Files - -- โœ… FlowVision/lib/Classes/ai/Actioner.cs -- โœ… FlowVision/lib/Classes/ai/MultiAgentActioner.cs -- โœ… FlowVision/FlowVision.csproj -- โœ… FlowVision/packages.config (already had the package) - -## Summary - -The runtime cast error is now **FIXED**. The application should run correctly after rebuilding. The fix uses the proper Microsoft-recommended pattern for integrating OpenAI SDK with Microsoft.Extensions.AI. diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index 79eb52f..0000000 --- a/SECURITY.md +++ /dev/null @@ -1,29 +0,0 @@ -# Reporting Security Issues - -If you believe you have ran into a security concern with the release of Recursive Control, please report it through coordinated disclosure. - -**Please do not make a report through any other method other than what is outlined below** - -Open a new [Github security advisory](https://github.com/flowdevs-io/recursive-control/security/advisories/new). - -## Required Information -Type of vulnerability -(e.g., buffer overflow, SQL injection, cross-site scripting, privilege escalation, etc.) - -## Relevant file paths and affected components -Include the full path(s) to any source file(s) directly related to the issue. - -## Location in source history -Specify where the issue appears (tag, branch, commit hash, or a direct URL to the relevant code). - -## Environment or configuration prerequisites -Mention any specific runtime environments, build flags, or configurations required to reproduce the issue. - -## Reproduction steps -Provide clear, step-by-step instructions so we can replicate the vulnerability. - -## Proof-of-concept (PoC) or exploit code -If available, include a minimal example demonstrating the issue. - -## Impact assessment -Explain the potential consequences, including how an attacker might exploit the vulnerability and what systems or data could be at risk. diff --git a/SYSTEM_PROMPTS_UPGRADE.md b/SYSTEM_PROMPTS_UPGRADE.md deleted file mode 100644 index 41387d4..0000000 --- a/SYSTEM_PROMPTS_UPGRADE.md +++ /dev/null @@ -1,473 +0,0 @@ -# System Prompts Upgrade - Computer Control Alignment - -## Date: October 2, 2025 - -## Executive Summary - -As an AI coding agent that regularly interacts with computers, I've upgraded Recursive Control's system prompts and multi-agent architecture to be significantly better aligned with **actual computer control workflows**. - -### Key Philosophy Shift - -**Before**: Prompts were generic, lacked computer-control specifics -**After**: Prompts are laser-focused on the observe โ†’ act โ†’ verify cycle - ---- - -## Major Improvements - -### 1. โœ… Window Handle Emphasis -**Problem**: AI was using global keyboard commands that went to random windows -**Solution**: Prompts now emphasize window-targeted methods throughout - -``` -OLD: "Send keys using SendKey()" -NEW: "ALWAYS use SendKeyToWindow(windowHandle, keys) with specific window handles" -``` - -### 2. โœ… Observation-First Approach -**Problem**: AI would act blindly without seeing current state -**Solution**: Prompts mandate starting with screenshots - -``` -Standard Workflow: -1. CaptureWholeScreen() - See what's there -2. ListWindowHandles() - Get window information -3. Plan based on observations -4. Execute with verification -``` - -### 3. โœ… Iterative Verification -**Problem**: AI would execute 10 steps blindly and fail -**Solution**: Prompts enforce step-by-step verification - -``` -Do โ†’ Verify โ†’ Adjust โ†’ Continue -Not: Plan 10 steps โ†’ Execute all โ†’ Hope it worked -``` - -### 4. โœ… Increased Iteration Limit -**Problem**: Complex tasks failed at 10 steps -**Solution**: Increased to 25 steps with better progress tracking - -``` -OLD: maxIterations = 10 -NEW: maxIterations = 25 with "Step X/25" progress indicators -``` - -### 5. โœ… Better Error Handling Guidance -**Problem**: AI didn't know how to recover from failures -**Solution**: Prompts include specific error recovery patterns - -``` -Window Not Found: -1. ListWindowHandles() again -2. Check if window closed -3. If needed, launch application -4. Get new handle and retry -``` - ---- - -## New System Prompts - -### Actioner Prompt (Single Agent Mode) - -**Focus**: Direct computer control with full tool access - -**Key Elements**: -- Complete tool catalog with window-targeted methods -- Operating principles (observation-first, window handles, verification) -- UI element format explanation (bbox coordinates) -- Workflow patterns for common tasks -- Best practices (DO/DON'T lists) - -**Length**: ~400 lines (comprehensive but focused) - -**Tone**: Direct, practical, action-oriented - -### Coordinator Prompt (Multi-Agent Mode) - -**Focus**: User interface and task routing - -**Key Elements**: -- Decision tree for task complexity assessment -- Simple vs complex task differentiation -- Communication style guidelines -- Example interactions - -**Length**: ~150 lines (concise, clear) - -**Tone**: Friendly but professional - -### Planner Prompt (Multi-Agent Mode) - -**Focus**: Sequential breakdown and adaptive planning - -**Key Elements**: -- Planning principles (observation-first, one action per step) -- Step format requirements -- Workflow pattern (receive โ†’ output first step โ†’ wait โ†’ adapt) -- Common task patterns (opening apps, browsing, UI interaction) -- Completion signal format - -**Length**: ~250 lines (structured, methodical) - -**Tone**: Analytical, step-by-step - ---- - -## Prompt Comparison - -### Old Actioner Prompt Issues: -``` -โŒ "Use available tools to accomplish the requested action" - โ†’ Too vague, no specifics - -โŒ No mention of window handles - โ†’ Led to SendKey() going to wrong windows - -โŒ No workflow guidance - โ†’ AI didn't know to take screenshots first - -โŒ No UI element format explanation - โ†’ AI confused about bbox coordinates -``` - -### New Actioner Prompt Strengths: -``` -โœ… "ALWAYS use SendKeyToWindow(windowHandle, keys)" - โ†’ Clear, specific instruction - -โœ… Complete tool catalog with window-targeted methods - โ†’ AI knows exactly what's available - -โœ… Standard workflow: Observe โ†’ Plan โ†’ Act โ†’ Verify - โ†’ Clear execution pattern - -โœ… UI element format: "Element #1 at (150,200) [size: 120x40]" - โ†’ AI understands the format - -โœ… BBox format: [left, top, right, bottom] with usage example - โ†’ AI can use coordinates correctly -``` - ---- - -## Multi-Agent Improvements - -### Iteration Loop Enhancements - -**1. Increased Steps**: -```python -OLD: maxIterations = 10 # Too few for complex tasks -NEW: maxIterations = 25 # Handles multi-step workflows -``` - -**2. Better Progress Tracking**: -``` -OLD: "Iteration 5 of 10" -NEW: "Step 5/25" with clearer context in prompts -``` - -**3. Enhanced Planner Feedback**: -``` -OLD: "Here is the result: {result}" -NEW: "Step 5 Result: {result} - - Evaluate: - 1. Did this step succeed? - 2. Is overall task complete? - 3. If not, what's the next single step?" -``` - -**4. Clearer Actioner Instructions**: -``` -OLD: "Execute the following step: {plan}" -NEW: "Execute this step: {plan} - - Remember to: - 1. Use window handles for keyboard/mouse - 2. Take screenshots to verify state - 3. Report exactly what you did and observed - 4. If something fails, explain what went wrong" -``` - -**5. Better Completion Detection**: -```python -OLD: if (plan.Contains("TASK COMPLETED")) -NEW: if (plan.IndexOf("TASK COMPLETED", StringComparison.OrdinalIgnoreCase) >= 0 || - plan.IndexOf("Task completed", StringComparison.OrdinalIgnoreCase) >= 0) -``` - ---- - -## Practical Examples - -### Example 1: Opening YouTube in Chrome - -**Old Approach** (Often Failed): -``` -1. SendKey("^t") # Might go to wrong window! -2. SendKey("youtube.com") # Might type in wrong place! -3. SendKey("{ENTER}") # Who knows what this hit! -``` - -**New Approach** (Reliable): -``` -1. CaptureWholeScreen() - See current state -2. ListWindowHandles() - Find Chrome (handle: 12345678) -3. ForegroundSelect("12345678") - Bring Chrome forward -4. SendKeyToWindow("12345678", "^t") - New tab in Chrome -5. SendKeyToWindow("12345678", "youtube.com") - Type URL -6. EnterKeyToWindow("12345678") - Navigate -7. Wait 2000ms -8. CaptureScreen("12345678") - Verify YouTube loaded -``` - -### Example 2: Finding UI Element and Clicking - -**Old Approach** (Vague): -``` -1. "Click the search button" # Where? Which window? -``` - -**New Approach** (Specific): -``` -1. CaptureScreen("12345678") - Get UI elements -2. Analyze elements: - - Element #5 at (300,250) [size: 200x60] - - This looks like the search box (large, top-center) -3. ClickOnWindow("12345678", [300, 250, 500, 310], true, 1) -4. CaptureScreen("12345678") - Verify search box focused -``` - ---- - -## Files Modified - -### 1. ToolConfig.cs -- Updated `ActionerSystemPrompt` (400+ lines, comprehensive) -- Updated `PlannerSystemPrompt` (250+ lines, structured) -- Updated `CoordinatorSystemPrompt` (150+ lines, focused) - -### 2. MultiAgentActioner.cs -- Increased `maxIterations` from 10 to 25 -- Enhanced planner feedback prompts -- Added clearer actioner instructions -- Improved completion detection (case-insensitive) -- Better progress logging - -### 3. Documentation Created -- `COMPUTER_USE_SYSTEM_PROMPTS.md` - Complete prompt library -- `SYSTEM_PROMPTS_UPGRADE.md` (this file) - Upgrade summary - ---- - -## Benefits - -### For Users - -โœ… **More Reliable**: Tasks complete successfully more often -โœ… **Better Feedback**: Clear progress indicators ("Step 5/25") -โœ… **Handles Complexity**: Can tackle 25-step workflows -โœ… **Fewer Errors**: Window-targeted actions prevent mistakes -โœ… **Self-Correcting**: AI verifies and adjusts approach - -### For AI - -โœ… **Clear Guidance**: Knows exactly what to do and how -โœ… **Better Tools**: Window-targeted methods emphasized -โœ… **Error Recovery**: Knows how to handle failures -โœ… **Structured Workflow**: Observe โ†’ Act โ†’ Verify pattern -โœ… **Context Awareness**: Screenshots provide visual feedback - -### For Developers - -โœ… **Maintainable**: Prompts are well-structured and documented -โœ… **Extensible**: Easy to add new patterns and guidance -โœ… **Debuggable**: Better logging of steps and progress -โœ… **Testable**: Clear workflows make testing easier - ---- - -## Migration Guide - -### For Existing Configurations - -**No breaking changes!** The prompts are stored in `ToolConfig` which: -- Auto-creates with new defaults for new users -- Preserves existing configs for current users -- Can be edited via UI or config files - -### To Use New Prompts - -**Option 1: Fresh Install** -- New users automatically get new prompts - -**Option 2: Manual Update** -1. Open Tool Configuration UI -2. View the new prompts in config -3. Save to update - -**Option 3: Delete and Recreate** -1. Delete `%APPDATA%\FlowVision\Config\toolsconfig.json` -2. Restart application -3. New defaults will be created - ---- - -## Testing Recommendations - -### Test Scenarios - -1. **Simple Task** (Single Agent Mode): - ``` - "Open Chrome and navigate to YouTube" - ``` - Expected: Should complete in 5-7 steps - -2. **Complex Task** (Multi-Agent Mode): - ``` - "Find the weather in Tokyo and create a text file with the information" - ``` - Expected: Should complete in 10-15 steps - -3. **Window Targeting**: - ``` - "Open Notepad and type 'Hello World'" - ``` - Expected: Should use window handles, not global SendKey - -4. **Error Recovery**: - ``` - "Open a browser to nonexistent.local" - ``` - Expected: Should detect error and explain what went wrong - -5. **Multi-Step Browser**: - ``` - "Search YouTube for Python tutorials and tell me the top 3 results" - ``` - Expected: Should verify each step with screenshots - ---- - -## Performance Expectations - -### Step Counts - -| Task Complexity | Expected Steps | Old Max | New Max | -|----------------|----------------|---------|---------| -| Simple | 3-5 steps | โœ… (within 10) | โœ… (within 25) | -| Medium | 8-12 steps | โš ๏ธ (might timeout) | โœ… (within 25) | -| Complex | 15-20 steps | โŒ (exceeds limit) | โœ… (within 25) | -| Very Complex | 20-25 steps | โŒ (exceeds limit) | โœ… (at limit) | - -### Success Rates (Estimated) - -| Task Type | Old Prompts | New Prompts | -|-----------|-------------|-------------| -| Browser Navigation | 70% | 95% | -| Window Management | 60% | 90% | -| Keyboard Input | 50% | 95% | -| Multi-Step Tasks | 40% | 85% | -| Error Recovery | 30% | 75% | - ---- - -## Future Enhancements - -### Potential Improvements - -1. **Context Persistence**: - - Remember previous screenshots - - Track window handles across sessions - - Cache common application handles - -2. **Smarter Defaults**: - - Learn common window โ†’ handle mappings - - Predict likely next steps - - Suggest shortcuts for common tasks - -3. **Vision Enhancement**: - - Better OCR integration for text labels - - Semantic understanding of UI elements - - Auto-labeling of common controls - -4. **Adaptive Complexity**: - - Start simple, add multi-agent only if needed - - Auto-switch between modes based on task - - Dynamic iteration limits based on progress - -5. **User Preferences**: - - Verbose vs concise progress updates - - Fast vs careful execution mode - - Auto-confirm vs ask before critical actions - ---- - -## Conclusion - -### What Was Accomplished - -1. โœ… **Rewrote all three system prompts** from scratch -2. โœ… **Increased iteration limit** from 10 to 25 -3. โœ… **Enhanced multi-agent feedback loop** -4. โœ… **Emphasized window-targeted methods** -5. โœ… **Added observation-first workflow** -6. โœ… **Included error recovery patterns** -7. โœ… **Created comprehensive documentation** - -### Impact - -**Before**: Generic AI agent trying to use computer tools -**After**: Specialized computer control agent with clear workflows - -The prompts now reflect **how an AI should actually interact with a Windows computer**: observe state, target specific windows, verify actions, adapt approach. - -### Build Status - -``` -โœ… Compilation: Success (0 errors, 7 warnings) -โœ… All prompts updated in ToolConfig.cs -โœ… Multi-agent loop enhanced in MultiAgentActioner.cs -โœ… Backward compatible with existing configs -โœ… Ready for production use -``` - ---- - -## Quick Start - -### For AI Using This System - -1. **Start with observation**: Always `CaptureWholeScreen()` first -2. **Use window handles**: Never use `SendKey()` without window handle -3. **Verify critical steps**: Take screenshots after important actions -4. **Work iteratively**: Don't plan 10 steps ahead, do 1 and verify -5. **Check browser state**: Always `IsBrowserActive()` before launching - -### For Users - -1. **Try simple tasks first**: "Open Chrome" or "Take a screenshot" -2. **Enable multi-agent for complexity**: Complex planning benefits from 3 agents -3. **Watch the progress**: UI shows "Step X/25" to track execution -4. **Be patient**: Quality execution takes time, especially with verification -5. **Provide feedback**: If something fails, the AI will explain what went wrong - ---- - -## Credits - -Prompts designed based on real experience as a computer-control AI agent. Patterns tested and refined through actual usage. The key insight: **Computer control requires the observe โ†’ act โ†’ verify cycle, not blind plan execution**. - ---- - -## Support - -For questions, issues, or suggestions: -- GitHub Issues: Report problems -- Discussions: Share experiences -- Discord: Real-time community support - -**Remember**: These prompts are designed for computer control. They work best when the AI has access to screenshots, can target specific windows, and can verify results iteratively. diff --git a/SYSTEM_PROMPT_IMPROVEMENTS.md b/SYSTEM_PROMPT_IMPROVEMENTS.md deleted file mode 100644 index 4dc7a97..0000000 --- a/SYSTEM_PROMPT_IMPROVEMENTS.md +++ /dev/null @@ -1,178 +0,0 @@ -# System Prompt Improvements Summary - -## ๐ŸŽฏ Problem Solved - -The agents were having difficulty knowing how to use the tools to control the computer effectively. The previous system prompts lacked: -- Specific tool call formats and examples -- Common mistake warnings -- Step-by-step workflows -- Emphasis on mandatory practices (like using window handles) - -## โœ… Changes Made - -### 1. Added "Reset to Default" Buttons - -Added reset buttons to the UI for all three agent prompts: -- **Actioner Tab**: "๐Ÿ”„ Reset to Default" button -- **Planner Tab**: "๐Ÿ”„ Reset to Default" button -- **Coordinator Tab**: "๐Ÿ”„ Reset to Default" button - -Each button: -- Shows confirmation dialog before resetting -- Reminds user to click "Save" after reset -- Loads default prompt from static methods - -### 2. Completely Rewrote Actioner System Prompt - -**New Features:** -- ๐Ÿ“‹ **YOUR MISSION section** - Makes it clear the agent DOES things, not just advises -- ๐Ÿ”ง **AVAILABLE TOOLS** - Complete tool reference with exact syntax -- ๐Ÿ“‹ **MANDATORY WORKFLOW** - Step-by-step process: Observe โ†’ Plan โ†’ Execute โ†’ Verify -- โœ… **EXAMPLES OF CORRECT USAGE** - 3 detailed examples: - - Opening Notepad and typing - - Clicking a button using bbox - - Browser search automation -- โŒ **COMMON MISTAKES** - Shows wrong vs right approaches -- ๐ŸŽ“ **TOOL CALL FORMAT** - Exact format requirements -- ๐Ÿ’ก **PRO TIPS** - Best practices checklist - -**Key Improvements:** -- Emphasizes observation FIRST (CaptureWholeScreen before any action) -- Makes window handles MANDATORY for all keyboard/mouse operations -- Provides exact tool call syntax with examples -- Uses emojis for visual organization -- Shows step-by-step workflows for common tasks -- Warns about common pitfalls - -**Length:** ~300 lines (vs ~80 lines previously) - -### 3. Improved Planner System Prompt - -**New Features:** -- Clear role explanation: "Output ONE step at a time" -- Iterative approach emphasized -- Good vs Bad step examples -- Common patterns library for: - - Opening applications - - Browser navigation - - UI interaction - - Typing text - - File operations -- Critical reminders about window handles -- Example task breakdown showing one-step-at-a-time approach - -**Key Improvements:** -- Emphasizes outputting single steps and waiting for results -- Provides template patterns for common tasks -- Shows complete example of breaking down a task -- Makes it clear to adapt based on actual results - -**Length:** ~180 lines (vs ~70 lines previously) - -### 4. Enhanced Coordinator System Prompt - -**New Features:** -- Decision tree diagram -- Clearer task routing guidelines -- Communication style examples (good vs bad) -- Tone guidance for user-friendly responses - -**Key Improvements:** -- Clearer distinction between simple/complex tasks -- Better examples of routing decisions -- Emphasis on user-friendly language vs technical jargon - -**Length:** ~140 lines (vs ~60 lines previously) - -### 5. Added Static Methods in ToolConfig - -New methods to retrieve default prompts: -```csharp -ToolConfig.GetDefaultActionerPrompt() -ToolConfig.GetDefaultPlannerPrompt() -ToolConfig.GetDefaultCoordinatorPrompt() -``` - -These are used both for: -- Initial configuration (when creating new config) -- Reset buttons (to restore defaults) - -## ๐Ÿ“ Files Modified - -1. **`FlowVision/lib/Classes/ToolConfig.cs`** - - Changed property initializers to use static methods - - Added 3 new static methods with comprehensive default prompts - - ~400 lines added - -2. **`FlowVision/ToolConfigForm.cs`** - - Added 3 reset button event handlers - - Each shows confirmation and success messages - - ~60 lines added - -3. **`FlowVision/ToolConfigForm.Designer.cs`** - - Added 3 button declarations - - Added buttons to group box controls - - Added button property definitions - - ~50 lines added - -## ๐ŸŽ“ Key Improvements for Tool Usage - -### Before: -โŒ Agents didn't know exact tool syntax -โŒ Missing examples of correct usage -โŒ No emphasis on observation-first approach -โŒ Unclear about window handle requirements -โŒ No common mistake warnings - -### After: -โœ… Exact tool call format with examples -โœ… Step-by-step workflows for common tasks -โœ… MANDATORY observation before action -โœ… Clear emphasis on window handles -โœ… Common mistakes section with wrong vs right -โœ… Pro tips and best practices -โœ… Visual organization with emojis -โœ… Iterative approach explained clearly - -## ๐Ÿ’ก Expected Impact - -Agents should now: -1. **Always start with observation** (CaptureWholeScreen first) -2. **Use correct tool syntax** (with all required parameters) -3. **Include window handles** for all keyboard/mouse operations -4. **Follow proper workflows** (observe โ†’ plan โ†’ execute โ†’ verify) -5. **Avoid common mistakes** (outlined in prompts) -6. **Work iteratively** (one step at a time, adapt based on results) - -## ๐Ÿ”— Pull Request - -Branch: `improve-system-prompts` -Create PR: https://github.com/flowdevs-io/Recursive-Control/pull/new/improve-system-prompts - -## ๐Ÿงช Testing Recommendations - -After merging: -1. Reset all agent prompts to defaults using the new buttons -2. Test common tasks: - - Opening applications - - Clicking UI elements - - Typing text in windows - - Browser automation -3. Verify agents now: - - Call CaptureWholeScreen first - - Use correct tool syntax - - Include window handles - - Work step-by-step - -## ๐Ÿ“ Notes - -- All changes are backward compatible -- Existing custom prompts are not affected until user clicks Reset -- Default prompts are significantly more detailed and instructive -- Build verified successful with MSBuild -- No breaking changes to API or configuration structure - ---- - -**Status**: โœ… Ready for review and merge -**Branch**: improve-system-prompts diff --git a/TASK_COMPLETE_OCR_INTEGRATION.md b/TASK_COMPLETE_OCR_INTEGRATION.md deleted file mode 100644 index d053d67..0000000 --- a/TASK_COMPLETE_OCR_INTEGRATION.md +++ /dev/null @@ -1,434 +0,0 @@ -# โœ… TASK COMPLETE: OCR Text Extraction Integration - -**Date**: October 2, 2025 -**Status**: โœ… COMPLETE AND OPERATIONAL -**Technology**: Tesseract 5.2.0 + ONNX YOLO - ---- - -## Executive Summary - -Successfully integrated **Tesseract OCR 5.2.0** into the ONNX OmniParser system, enabling **semantic text extraction** from detected UI elements. The AI can now read and understand what UI elements say, not just where they are located. - ---- - -## The Problem (Before) - -Your logs showed: -``` -[2025-10-02 22:50:08] Info: OcrHelper, Initialize, OCR is currently disabled -[2025-10-02 22:50:08] Info: OnnxOmniParser, ExtractTextFromDetections, OCR not available -``` - -UI elements were labeled as: -``` -"Element 171" -"Element 172" -"Element 173" -``` - -โŒ **No semantic meaning** - AI couldn't understand what these elements were for. - ---- - -## The Solution (After) - -OCR is now active: -``` -[2025-10-02 22:50:08] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully -[2025-10-02 22:50:08] Info: OnnxOmniParser, ExtractTextFromDetections, Extracting text from 145 elements -[2025-10-02 22:50:12] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: 85 elements with text -``` - -UI elements are now labeled as: -``` -"Play Video at (150,200) [size: 120x40]" -"Subscribe Button at (300,250) [size: 200x60]" -"YouTube Logo at (450,300) [size: 180x50]" -``` - -โœ… **Rich semantic meaning** - AI understands both content and context. - ---- - -## What Was Done - -### 1. โœ… Tesseract Package Integration -- Added Tesseract 5.2.0 NuGet package reference -- Configured build system to deploy native libraries -- Downloaded English language model (3.92 MB) - -### 2. โœ… OCR Implementation -**File**: `FlowVision/lib/Classes/OcrHelper.cs` -- Replaced 67-line placeholder with 189-line production implementation -- Added TesseractEngine initialization with error handling -- Implemented thread-safe OCR processing -- Added automatic tessdata path detection -- Configured for optimal UI text recognition - -### 3. โœ… Build Configuration -**File**: `FlowVision/FlowVision.csproj` -- Added Tesseract reference with `True` -- Imported Tesseract.targets for automatic setup -- Created custom MSBuild target to copy native DLLs -- Ensured all dependencies deploy with application - -### 4. โœ… Native Dependencies Deployed -- `tesseract50.dll` (2.66 MB) - Core OCR engine -- `leptonica-1.82.0.dll` (3.98 MB) - Image processing library -- `Tesseract.dll` (0.13 MB) - .NET wrapper -- `eng.traineddata` (3.92 MB) - English language model - -### 5. โœ… Documentation Created -- `OCR_INTEGRATION_COMPLETE.md` - Overview and summary -- `OCR_TEXT_EXTRACTION_STATUS.md` - Complete technical details -- `OCR_QUICK_REFERENCE.md` - One-page reference card -- `TEST_OCR.md` - Implementation documentation -- `test_ocr_simple.ps1` - Prerequisites verification script - ---- - -## Technical Architecture - -### Processing Flow - -``` -User triggers screenshot capture - โ†“ -ONNX YOLO detects UI elements (145 found) - โ†“ -For each detected bounding box: - โ”‚ - โ”œโ”€ Validate region bounds - โ”œโ”€ Crop image to region - โ”œโ”€ Convert to Tesseract Pix format - โ”œโ”€ Run OCR text extraction - โ””โ”€ Clean and validate extracted text - โ†“ -Generate enhanced labels: - โ”œโ”€ If text found: "Button Text at (x,y) [size: WxH]" - โ””โ”€ If no text: "UI Element #N at (x,y) [size: WxH]" - โ†“ -Return results to AI with rich semantic context -``` - -### Key Components - -1. **OcrHelper.cs** - OCR abstraction layer - - Static class with singleton TesseractEngine - - Thread-safe with lock-based synchronization - - Automatic initialization and error handling - -2. **OnnxOmniParserEngine.cs** - Integration point - - Calls OcrHelper for each detected region - - Populates UIElementDetection.Caption with text - - Already implemented (no changes needed) - -3. **ScreenCaptureOmniParserPlugin.cs** - Label generation - - Combines OCR text with position/size info - - Falls back to descriptive labels if no text - - Already implemented (no changes needed) - ---- - -## Minimal Changes Approach โœ… - -Following the principle of **surgical, minimal modifications**: - -### What Changed (Minimal) -1. โœ… `FlowVision.csproj` - Added 3 lines for Tesseract reference + 1 build target -2. โœ… `OcrHelper.cs` - Replaced placeholder with production code -3. โœ… `OCR_TEXT_EXTRACTION_STATUS.md` - Updated status to COMPLETE - -### What Didn't Change (Infrastructure Ready) -- โœ… `OnnxOmniParserEngine.cs` - OCR integration already implemented -- โœ… `ScreenCaptureOmniParserPlugin.cs` - Label generation already implemented -- โœ… `UIElementDetection` class - Caption property already available -- โœ… All other plugins and components - Unaffected - -**Result**: Maximum impact with minimum code changes! ๐ŸŽฏ - ---- - -## Build & Deployment Status - -### Build Results -``` -โœ… Compilation: SUCCESSFUL -โœ… Errors: 0 -โœ… Warnings: 11 (pre-existing, unrelated to OCR) -โœ… Output: FlowVision.exe (3.97 MB) -``` - -### Deployment Verification -``` -โœ… FlowVision.exe (3.97 MB) -โœ… Tesseract.dll (0.13 MB) -โœ… tesseract50.dll (2.66 MB) -โœ… leptonica-1.82.0.dll (3.98 MB) -โœ… tessdata/eng.traineddata (3.92 MB) -``` - -**Total additional size**: ~11 MB (OCR dependencies) - ---- - -## Configuration Details - -### Tesseract Settings - -**Engine Mode**: Default (combines legacy Tesseract + LSTM neural network) - -**Language**: English (eng.traineddata) - -**Character Whitelist**: UI text optimized -``` -ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .-_:@/\()[]{}!?&+=#$% -``` - -**Processing Options**: -- `preserve_interword_spaces = 1` - Maintains word spacing -- Async processing on background threads -- Thread-safe with single engine instance -- Skip regions smaller than 10x10 pixels -- Automatic error recovery and fallback - ---- - -## Performance Metrics - -### Typical Analysis Times -- **YOLO Detection**: ~400-500ms -- **OCR Processing**: ~2-4 seconds (for 145 elements) -- **Total Analysis**: ~4-5 seconds -- **Text Success Rate**: 60-80% of elements - -### Optimization Features -โœ… Single TesseractEngine instance (no repeated initialization) -โœ… Thread-safe locking (concurrent access protected) -โœ… Small region filtering (< 10x10 pixels skipped) -โœ… Async processing (non-blocking) -โœ… Graceful error handling (no crashes on OCR failure) - ---- - -## Testing & Verification - -### Prerequisites Check -```powershell -.\test_ocr_simple.ps1 -``` - -Expected output: -``` -โœ“ All prerequisites satisfied! -โœ“ FlowVision.exe -โœ“ Tesseract.dll -โœ“ tesseract50.dll -โœ“ leptonica-1.82.0.dll -โœ“ tessdata folder -โœ“ eng.traineddata -``` - -### Runtime Verification - -**Startup Log** (look for this): -``` -[timestamp] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully. Text extraction is now enabled. -``` - -**During Analysis** (look for this): -``` -[timestamp] Info: OnnxOmniParser, ParseImage, Processing image 4480x1440 -[timestamp] Info: OnnxOmniParser, ParseImage, Detected 145 UI elements -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, Extracting text from 145 elements -[timestamp] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: 85 elements with text -``` - ---- - -## Impact & Benefits - -### For the AI Agent -1. โœ… **Semantic Understanding** - Knows what UI elements say -2. โœ… **Target Accuracy** - Can find "Subscribe" button specifically -3. โœ… **Content Verification** - Can read and confirm action results -4. โœ… **Context Awareness** - Understands UI meaning and purpose - -### For End Users -1. โœ… **Better Automation** - AI interacts with correctly identified elements -2. โœ… **Higher Accuracy** - Fewer mistakes due to better understanding -3. โœ… **Natural Commands** - "Click the Save button" works reliably -4. โœ… **Result Verification** - AI confirms actions by reading text - -### Examples - -**Before OCR**: -- User: "Click the subscribe button" -- AI: "I see Element 172 at position (300, 250), is that what you want?" -- Success rate: ~40% (positional guessing) - -**After OCR**: -- User: "Click the subscribe button" -- AI: "Found 'Subscribe Button' at (300, 250), clicking now" -- Success rate: ~90% (semantic matching) - ---- - -## Error Handling - -### Initialization Errors -Gracefully handled with fallback: - -**Missing tessdata folder**: -``` -[timestamp] Error: OcrHelper, Initialize, tessdata directory not found at: [path] -โ†’ Result: OCR disabled, falls back to position-based labels -``` - -**Missing language file**: -``` -[timestamp] Error: OcrHelper, Initialize, English language data not found at: [path] -โ†’ Result: OCR disabled, falls back to position-based labels -``` - -**Engine creation failure**: -``` -[timestamp] Error: OcrHelper, Initialize, Failed to initialize Tesseract: [details] -โ†’ Result: OCR disabled, falls back to position-based labels -``` - -### Runtime Errors -Never crash the application: - -**OCR processing failure**: -``` -โ†’ Result: Log error, return empty string, continue with next element -``` - -**Invalid region**: -``` -โ†’ Result: Validate and adjust bounds, or skip if too small -``` - ---- - -## Files Changed - -### Modified (3 files) -1. `FlowVision/FlowVision.csproj` - Tesseract integration -2. `FlowVision/lib/Classes/OcrHelper.cs` - OCR implementation -3. `OCR_TEXT_EXTRACTION_STATUS.md` - Status update - -### Created (4 files) -4. `OCR_INTEGRATION_COMPLETE.md` - Quick summary -5. `OCR_QUICK_REFERENCE.md` - One-page reference -6. `TEST_OCR.md` - Technical details -7. `test_ocr_simple.ps1` - Verification script - -### Binary/Data (not in git) -- `FlowVision/bin/Debug/tessdata/eng.traineddata` -- `FlowVision/bin/Release/tessdata/eng.traineddata` -- `FlowVision/bin/Debug/tesseract50.dll` -- `FlowVision/bin/Debug/leptonica-1.82.0.dll` - ---- - -## Future Enhancements (Optional) - -These are **not required** but could be added later: - -### Potential Improvements -- ๐Ÿ”„ Additional languages (fra, spa, deu traineddata files) -- ๐Ÿ”„ Confidence filtering (only use high-confidence results) -- ๐Ÿ”„ Parallel OCR processing (multiple threads) -- ๐Ÿ”„ Result caching (reuse OCR for unchanged screens) -- ๐Ÿ”„ Fine-tuning for specific UI frameworks - -### Not Necessary -Current implementation is **production-ready** and fully functional. - ---- - -## Troubleshooting Guide - -### Issue: OCR not initializing - -**Check**: -1. tessdata folder exists in application directory -2. eng.traineddata file present (3.92 MB) -3. Native DLLs deployed (tesseract50.dll, leptonica) -4. Check application logs for initialization errors - -**Fix**: Run `.\test_ocr_simple.ps1` to verify all files present - -### Issue: No text extracted - -**Reasons** (all normal): -- UI elements don't contain text (icons, dividers, etc.) -- Text too small (< 10x10 pixels) -- Non-standard fonts or symbols -- Poor image quality - -**Result**: System falls back to position-based labels (graceful) - -### Issue: Build failures - -**Check**: -1. Tesseract reference in .csproj -2. Tesseract.targets imported -3. Native DLL copy target present - -**Fix**: Review FlowVision.csproj changes in this document - ---- - -## Documentation Index - -๐Ÿ“„ **OCR_INTEGRATION_COMPLETE.md** - Quick overview -๐Ÿ“„ **OCR_TEXT_EXTRACTION_STATUS.md** - Complete technical documentation -๐Ÿ“„ **OCR_QUICK_REFERENCE.md** - One-page reference card -๐Ÿ“„ **TEST_OCR.md** - Implementation details -๐Ÿ“„ **test_ocr_simple.ps1** - Prerequisites verification -๐Ÿ“„ **TASK_COMPLETE_OCR_INTEGRATION.md** - This document - ---- - -## Success Criteria โœ… - -All objectives achieved: - -โœ… **OCR Integration** - Tesseract 5.2.0 fully integrated -โœ… **Text Extraction** - Working and extracting text from UI elements -โœ… **Semantic Labels** - AI receives meaningful element descriptions -โœ… **Build Success** - 0 errors, clean compilation -โœ… **Dependencies** - All native libs and language data deployed -โœ… **Documentation** - Comprehensive docs created -โœ… **Verification** - Test script and validation complete -โœ… **No Breaking Changes** - Existing functionality preserved -โœ… **Graceful Degradation** - Falls back if OCR unavailable - ---- - -## Conclusion - -### ๐ŸŽ‰ Mission Accomplished! - -The ONNX OmniParser now has **full OCR capabilities** powered by Tesseract 5.2.0. - -**Before**: Generic element labels with no semantic meaning -**After**: Rich semantic labels with actual UI text content - -**Impact**: Dramatically improved AI understanding and automation accuracy! - -### ๐Ÿš€ Ready to Use - -Launch `FlowVision.exe` and capture a screenshot to see OCR in action! - ---- - -**Task**: OCR Text Extraction Integration -**Status**: โœ… COMPLETE -**Result**: OPERATIONAL -**Version**: 1.0 -**Date**: October 2, 2025 diff --git a/TEST_OCR.md b/TEST_OCR.md deleted file mode 100644 index f759b4c..0000000 --- a/TEST_OCR.md +++ /dev/null @@ -1,207 +0,0 @@ -# OCR Integration Test - October 2, 2025 - -## Changes Made - -### 1. Added Tesseract Reference to Project -- **File**: `FlowVision/FlowVision.csproj` -- **Changes**: - - Added Tesseract reference: `` - - Added Tesseract.targets import - - Added custom build target to copy native DLLs - -### 2. Implemented Full OCR Support -- **File**: `FlowVision/lib/Classes/OcrHelper.cs` -- **Changes**: - - Replaced placeholder with full Tesseract implementation - - Added initialization code for TesseractEngine - - Implemented `ExtractTextAsync()` method - - Implemented `ExtractTextFromRegionAsync()` method - - Added thread-safe locking for Tesseract engine usage - - Configured Tesseract for UI text recognition with custom character whitelist - -### 3. Downloaded Language Data -- **Location**: `FlowVision/bin/Debug/tessdata/eng.traineddata` (3.92 MB) -- **Location**: `FlowVision/bin/Release/tessdata/eng.traineddata` (3.92 MB) - -### 4. Deployed Native Libraries -- **Tesseract Native**: `tesseract50.dll` (2.66 MB) -- **Leptonica Native**: `leptonica-1.82.0.dll` (3.98 MB) -- Both copied to Debug and Release output directories - -## How It Works - -### Initialization Flow - -``` -Application Start - โ†“ -OcrHelper Static Constructor - โ†“ -Initialize() Method - โ†“ -Check for tessdata directory - โ†“ -Check for eng.traineddata file - โ†“ -Create TesseractEngine - โ†“ -Configure for UI text recognition - โ†“ -Set IsAvailable = true - โ†“ -Log success message -``` - -### OCR Processing Flow - -``` -Screenshot Captured - โ†“ -ONNX YOLO Detection (finds UI elements) - โ†“ -For each detected element: - โ†“ - ExtractTextFromRegionAsync() - โ†“ - Validate region bounds - โ†“ - Crop image to region - โ†“ - Convert to Tesseract Pix format - โ†“ - Run OCR (page.GetText()) - โ†“ - Return trimmed text - โ†“ -Add text to detection.Caption - โ†“ -Generate enhanced label with text + position -``` - -## Expected Behavior - -### Before OCR (Previous Behavior) -``` -[2025-10-02 22:50:08] Info: OcrHelper, Initialize, OCR is currently disabled. -[2025-10-02 22:50:08] Info: OnnxOmniParser, ExtractTextFromDetections, OCR not available -[2025-10-02 22:50:08] Info: Found 145 UI elements - -Labels: "UI Element #1 at (150,200) [size: 120x40]" -``` - -### After OCR (New Behavior) -``` -[2025-10-02 22:50:08] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully -[2025-10-02 22:50:08] Info: OnnxOmniParser, ExtractTextFromDetections, Extracting text from 145 elements -[2025-10-02 22:50:12] Info: OnnxOmniParser, ExtractTextFromDetections, OCR complete: 85 elements with text -[2025-10-02 22:50:12] Info: Found 145 UI elements - -Labels: "Play Video at (150,200) [size: 120x40]" - "Subscribe Button at (300,250) [size: 200x60]" -``` - -## Technical Details - -### Tesseract Configuration - -**Engine Mode**: Default (combines legacy and LSTM engines) - -**Character Whitelist**: -``` -ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .-_:@/\()[]{}!?&+=#$% -``` -This ensures only common UI text characters are recognized. - -**Other Settings**: -- `preserve_interword_spaces = 1` - Keeps spaces between words - -### Performance Optimizations - -1. **Thread Safety**: Single Tesseract engine instance with lock-based synchronization -2. **Region Validation**: Skips very small regions (< 10x10 pixels) unlikely to contain text -3. **Async Processing**: OCR runs on background thread pool via `Task.Run()` -4. **Empty Result Handling**: Returns empty string for regions without meaningful text - -### Error Handling - -- **Missing tessdata**: Logs error, sets IsAvailable = false -- **Missing language file**: Logs error, sets IsAvailable = false -- **OCR processing error**: Logs error, returns empty string for that region -- **Region crop error**: Logs error, returns empty string - -## Build Status - -``` -โœ… Build successful with 0 errors, 11 warnings -โœ… All native dependencies deployed -โœ… Language data files in place -โœ… OCR infrastructure complete -``` - -## Testing Instructions - -### Manual Test -1. Run FlowVision.exe -2. Open OmniParser screen capture tool -3. Capture a screenshot with visible UI elements containing text -4. Check logs for: - - "โœ“ Tesseract OCR initialized successfully" - - "Extracting text from X elements" - - "OCR complete: Y elements with text" -5. Verify element labels contain actual text instead of generic descriptions - -### Expected Results - -**Without OCR**: -- Labels like "UI Element #5 at (300,250) [size: 200x60]" - -**With OCR**: -- Labels like "Subscribe Button at (300,250) [size: 200x60]" - -## Files Modified - -1. โœ… `FlowVision/FlowVision.csproj` - Added Tesseract reference and build targets -2. โœ… `FlowVision/lib/Classes/OcrHelper.cs` - Full Tesseract implementation -3. โœ… `FlowVision/bin/Debug/tessdata/eng.traineddata` - Language data -4. โœ… `FlowVision/bin/Release/tessdata/eng.traineddata` - Language data -5. โœ… Native DLLs copied to output directories - -## What Was Changed - -### Minimal Changes Approach - -Following the principle of **minimal modifications**, I: - -1. โœ… **Only added Tesseract support** - No other code changes -2. โœ… **Used existing infrastructure** - OcrHelper.cs was already prepared -3. โœ… **No breaking changes** - Existing functionality unchanged -4. โœ… **Graceful degradation** - If OCR fails, falls back to position-based labels - -### Infrastructure Already in Place - -The following was already implemented (no changes needed): -- โœ… `OnnxOmniParserEngine.ExtractTextFromDetections()` method -- โœ… `UIElementDetection.Caption` property -- โœ… Label generation logic in ScreenCaptureOmniParserPlugin -- โœ… Error logging and status reporting - -## Verification - -To verify OCR is working, look for this log message on startup: -``` -[timestamp] Info: OcrHelper, Initialize, โœ“ Tesseract OCR initialized successfully. Text extraction is now enabled. -``` - -If you see this message, OCR is active and will extract text from detected UI elements. - -## Summary - -โœ… **OCR is now fully functional** -- Tesseract 5.2.0 integrated -- Native libraries deployed -- Language data installed -- Thread-safe implementation -- Optimized for UI text recognition -- Graceful error handling - -The system will now extract actual text from UI elements instead of using generic placeholders, making the AI much more effective at understanding and interacting with screen content. diff --git a/TODAYS_COMPLETE_FIXES.md b/TODAYS_COMPLETE_FIXES.md deleted file mode 100644 index 94769aa..0000000 --- a/TODAYS_COMPLETE_FIXES.md +++ /dev/null @@ -1,485 +0,0 @@ -# Complete Fixes Applied - October 2, 2025 - -## Summary - -Today we've completely transformed Recursive Control from a basic computer control system to a **production-ready, best-in-class AI agent platform**. Here's everything that was fixed and improved. - ---- - -## ๐ŸŽฏ Issues Fixed (6 Total) - -### 1. โœ… Tool Calls Compilation Error -**Problem**: `SetChatHistory` method was `internal`, test project couldn't access it -**File**: `MultiAgentActioner.cs` -**Fix**: Changed from `internal` to `public` -**Impact**: Tests now compile successfully - -### 2. โœ… ONNX Runtime Auto-Initialization -**Problem**: YOLO model not loaded at startup, causing 15-30 second delays -**File**: `ScreenCaptureOmniParserPlugin.cs` -**Fix**: Added automatic initialization in constructor -**Impact**: Model always ready, instant screenshot processing - -### 3. โœ… Keyboard Shortcuts to Wrong Tab/Window -**Problem**: `SendKey()` went to whatever window had focus (unreliable) -**File**: `KeyboardPlugin.cs` -**Fix**: Added window-targeted methods: -- `SendKeyToWindow(windowHandle, keys)` -- `EnterKeyToWindow(windowHandle)` -- `CtrlKeyToWindow(windowHandle, letter)` -**Impact**: Keyboard commands now reliably target specific windows - -### 4. โœ… Enhanced UI Element Labels -**Problem**: Generic labels like "Element 171" (not useful) -**File**: `ScreenCaptureOmniParserPlugin.cs` -**Fix**: Labels now include position and size: -``` -"UI Element #1 at (150,200) [size: 120x40]" -``` -**Impact**: AI can identify elements by location and size - -### 5. โœ… System Prompts Completely Rewritten -**Problem**: Prompts were generic, not aligned with computer control workflows -**Files**: `ToolConfig.cs` -**Fix**: Rewrote all 3 prompts (800+ lines) with: -- Window handle emphasis -- Observe โ†’ Act โ†’ Verify workflow -- Error recovery patterns -- Practical examples -**Impact**: AI now follows best practices for computer control - -### 6. โœ… MarkdownHelper NullReferenceException -**Problem**: Crash when `SelectionFont` was null -**File**: `MarkdownHelper.cs` -**Fix**: Used null-conditional operators (`?.` and `??`) with safe defaults -**Impact**: No more random crashes when formatting markdown - ---- - -## ๐Ÿ“Š Statistics - -### Code Changes -- **6 files modified** -- **5 new documentation files created** -- **1,200+ lines of code improved** -- **800+ lines of new prompts** - -### Build Status -``` -โœ… Main Project: 0 errors, 0 warnings -โœ… Test Project: 0 errors (compilation fixed) -โœ… Full Solution: 0 errors -โœ… All features: Working -``` - -### Quality Improvements -- **Reliability**: 40-70% โ†’ 85-95% (estimated) -- **Iteration limit**: 10 โ†’ 25 steps -- **Window targeting**: 0% โ†’ 100% of keyboard/mouse operations -- **Crash resistance**: Multiple null reference fixes - ---- - -## ๐Ÿ”ง Major Enhancements - -### Multi-Agent System -**Before**: Simple 10-step loop with vague prompts -**After**: 25-step iterative system with clear guidance - -**Improvements**: -- Increased max iterations (10 โ†’ 25) -- Better progress tracking ("Step X/25") -- Enhanced feedback prompts -- Clearer completion detection -- Better error recovery guidance - -### Computer Control Alignment -**Before**: Generic AI assistant -**After**: Specialized computer control agent - -**Key Principles Now Enforced**: -1. **Observe First**: Always take screenshots before acting -2. **Window Handles**: Target specific windows, not global -3. **Verify Always**: Check results after important actions -4. **Iterate**: Work step-by-step, not blindly -5. **Adapt**: Adjust based on what you actually see - -### Keyboard Plugin -**Before**: Global commands (unreliable) -**After**: Window-targeted commands (reliable) - -**New Methods**: -```csharp -// Old way (unreliable) -SendKey("^t") // Goes to random window! - -// New way (reliable) -SendKeyToWindow(chromeHandle, "^t") // Goes to Chrome specifically -``` - ---- - -## ๐Ÿ“ Files Modified - -### Core System Files -1. **ToolConfig.cs** - - Rewrote ActionerSystemPrompt (400+ lines) - - Rewrote PlannerSystemPrompt (250+ lines) - - Rewrote CoordinatorSystemPrompt (150+ lines) - -2. **MultiAgentActioner.cs** - - Increased maxIterations (10 โ†’ 25) - - Enhanced feedback loop - - Better completion detection - - Changed `SetChatHistory` to public - -3. **KeyboardPlugin.cs** - - Added `SendKeyToWindow()` - - Added `EnterKeyToWindow()` - - Added `CtrlKeyToWindow()` - - Added `BringWindowToForegroundWithFocus()` helper - -4. **ScreenCaptureOmniParserPlugin.cs** - - Added ONNX auto-initialization - - Enhanced element label generation - - Improved position/size reporting - -5. **OnnxOmniParserEngine.cs** - - Added OCR infrastructure (prepared) - - Enhanced text extraction (ready for future) - -6. **OcrHelper.cs** (NEW) - - OCR abstraction layer - - Ready for Tesseract or Windows OCR - -7. **MarkdownHelper.cs** - - Fixed NullReferenceException in 4 methods - - Added null-safe font handling - - Sensible defaults for missing fonts - -8. **FlowVision.csproj** - - Added OcrHelper.cs to compilation - - Added Windows Runtime references - -### Documentation Created -1. **FIXES_APPLIED.md** - Initial fixes summary -2. **OCR_TEXT_EXTRACTION_STATUS.md** - OCR infrastructure -3. **KEYBOARD_FOCUS_FIX.md** - Keyboard improvements -4. **COMPUTER_USE_SYSTEM_PROMPTS.md** - Complete prompt library -5. **SYSTEM_PROMPTS_UPGRADE.md** - Upgrade guide -6. **MARKDOWN_NULL_REFERENCE_FIX.md** - Null reference fix -7. **TODAYS_COMPLETE_FIXES.md** (this file) - Complete summary - ---- - -## ๐ŸŽ New Features - -### Window-Targeted Keyboard Control -```csharp -// Get window handle -var windows = windowSelection.ListWindowHandles(); -string chromeHandle = "12345678"; - -// Send keys to specific window -await keyboard.SendKeyToWindow(chromeHandle, "^t"); // Ctrl+T to Chrome -await keyboard.EnterKeyToWindow(chromeHandle); // Enter to Chrome -await keyboard.CtrlKeyToWindow(chromeHandle, "w"); // Ctrl+W to Chrome -``` - -### Enhanced UI Element Detection -``` -Before: "Element 171" -After: "UI Element #1 at (150,200) [size: 120x40]" -``` - -### ONNX Auto-Initialization -```csharp -// Old: Model loads on first use (15-30 second delay) -// New: Model loads at startup (instant processing) - -var plugin = new ScreenCaptureOmniParserPlugin(); -// YOLO model already loaded and ready! โœ… -``` - -### Improved System Prompts -- **Actioner**: 400+ lines of computer control guidance -- **Planner**: 250+ lines of sequential planning -- **Coordinator**: 150+ lines of user interaction - ---- - -## ๐Ÿ“ˆ Performance Improvements - -### Task Success Rates (Estimated) - -| Task Type | Before | After | Improvement | -|-----------|--------|-------|-------------| -| Browser Navigation | 70% | 95% | +25% | -| Window Management | 60% | 90% | +30% | -| Keyboard Input | 50% | 95% | +45% | -| Multi-Step Tasks | 40% | 85% | +45% | -| Error Recovery | 30% | 75% | +45% | - -### Iteration Capacity - -| Complexity | Steps Needed | Before (Max 10) | After (Max 25) | -|------------|--------------|-----------------|----------------| -| Simple | 3-5 | โœ… Success | โœ… Success | -| Medium | 8-12 | โš ๏ธ Borderline | โœ… Success | -| Complex | 15-20 | โŒ Fails | โœ… Success | -| Very Complex | 20-25 | โŒ Fails | โœ… Success | - -### Reliability Improvements -- **Keyboard commands**: 50% โ†’ 95% success (window targeting) -- **Screenshot processing**: 15-30s delay โ†’ instant (auto-init) -- **UI crashes**: Random โ†’ None (null-safe markdown) -- **Multi-step tasks**: 10 step limit โ†’ 25 step limit - ---- - -## ๐ŸŽฏ Real-World Examples - -### Example 1: Opening YouTube in Chrome - -**Before** (50% success): -``` -1. SendKey("^t") โŒ Might go to Terminal -2. Type URL โŒ Typed in wrong window -3. Press Enter โŒ Random results -``` - -**After** (95% success): -``` -1. CaptureWholeScreen() - See current state -2. ListWindowHandles() - Find Chrome (12345678) -3. ForegroundSelect("12345678") - Bring Chrome forward -4. SendKeyToWindow("12345678", "^t") - New tab in Chrome -5. SendKeyToWindow("12345678", "youtube.com") - Type in Chrome -6. EnterKeyToWindow("12345678") - Navigate in Chrome -7. Wait 2000ms - Allow page load -8. CaptureScreen("12345678") - Verify YouTube loaded โœ… -``` - -### Example 2: Multi-Step Browser Task - -**Task**: "Search YouTube for Python tutorials and tell me the top 3 results" - -**Steps** (would fail before at 10, now succeeds): -``` -1. Check browser active -2. If not, launch browser -3. Navigate to YouTube -4. Wait for load -5. Screenshot to see page -6. Find search box (by position) -7. Click search box -8. Type "Python tutorials" -9. Press Enter -10. Wait for results -11. Screenshot to see results -12. Analyze top 3 elements -13. Extract information -14. Format response -15. Return to user -โœ… Completes in 15 steps (would fail with 10 limit) -``` - ---- - -## ๐Ÿ” Backward Compatibility - -### No Breaking Changes -- โœ… Old keyboard methods still work (SendKey, CtrlKey, EnterKey) -- โœ… Existing configs preserved -- โœ… New prompts only for fresh installs (or manual update) -- โœ… All existing code continues to function - -### Migration Path -**For Users**: -1. Update application -2. Optionally delete config to get new prompts -3. Or manually update via UI - -**For Code**: -- Old methods available for backward compatibility -- New methods recommended for new code - ---- - -## ๐Ÿงช Testing Recommendations - -### Critical Paths to Test - -1. **Window-Targeted Keyboard**: - ``` - "Open Notepad and type 'Hello World'" - Expected: Uses window handle, types in Notepad - ``` - -2. **Multi-Step Workflow**: - ``` - "Search YouTube for Python and report top 3" - Expected: Completes in 12-15 steps successfully - ``` - -3. **Screenshot Processing**: - ``` - "Take a screenshot of Chrome" - Expected: Instant processing, no delay - ``` - -4. **Markdown Formatting**: - ``` - Send message with `code`, **bold**, *italic* - Expected: No crashes, proper formatting - ``` - -5. **Error Recovery**: - ``` - "Open a website that doesn't exist" - Expected: Detects error, explains what happened - ``` - ---- - -## ๐Ÿ“š Documentation - -### Complete Documentation Set -1. **User Guides**: - - FIXES_APPLIED.md - What was fixed - - KEYBOARD_FOCUS_FIX.md - Keyboard improvements - - OCR_TEXT_EXTRACTION_STATUS.md - Future OCR plans - -2. **Developer Guides**: - - COMPUTER_USE_SYSTEM_PROMPTS.md - Prompt library - - SYSTEM_PROMPTS_UPGRADE.md - Upgrade details - - MARKDOWN_NULL_REFERENCE_FIX.md - Null reference fix - -3. **Complete Summary**: - - TODAYS_COMPLETE_FIXES.md (this file) - ---- - -## ๐Ÿš€ Future Enhancements - -### Ready for Next Steps -1. **OCR Integration**: - - Infrastructure complete - - Ready for Tesseract or Windows OCR - - Will add text labels to UI elements - -2. **Vision Improvements**: - - YOLO model always loaded - - Ready for Florence2 integration - - Can add semantic understanding - -3. **Advanced Features**: - - Context persistence across sessions - - Smart window handle caching - - Predictive next-step suggestions - ---- - -## โœ… Quality Checklist - -### Code Quality -- โœ… All null references fixed -- โœ… Defensive programming applied -- โœ… Sensible defaults everywhere -- โœ… Error handling improved -- โœ… Logging enhanced - -### System Quality -- โœ… 0 compilation errors -- โœ… 0 critical warnings -- โœ… Backward compatible -- โœ… Production ready -- โœ… Fully documented - -### User Experience -- โœ… More reliable execution -- โœ… Better progress feedback -- โœ… Clear error messages -- โœ… Faster processing -- โœ… No random crashes - ---- - -## ๐ŸŽ‰ Bottom Line - -### What Was Accomplished Today - -**6 major fixes** + **7 comprehensive improvements** + **7 documentation files** = **Complete system overhaul** - -### Key Achievements - -1. โœ… **Fixed all reported issues** -2. โœ… **Rewrote system prompts from scratch** -3. โœ… **Added window-targeted keyboard control** -4. โœ… **Implemented ONNX auto-initialization** -5. โœ… **Enhanced UI element detection** -6. โœ… **Improved multi-agent system** -7. โœ… **Fixed markdown rendering crashes** - -### Impact - -**Before**: Basic computer control system with reliability issues -**After**: Production-ready AI agent platform aligned with best practices - -### Reliability Improvement - -``` -Overall Success Rate: -Before: ~50% (would often fail or go to wrong windows) -After: ~90% (reliable, targeted, verified execution) -``` - ---- - -## ๐Ÿ™ Special Notes - -This wasn't just bug fixing - this was a **complete system alignment** based on real experience as an AI coding agent. The changes reflect **how AI should actually interact with computers**: - -1. **See before acting** (screenshots) -2. **Target specifically** (window handles) -3. **Verify results** (iterative checking) -4. **Adapt approach** (based on observations) -5. **Explain clearly** (user feedback) - -Every change was made with the goal of making Recursive Control the **best computer control platform for AI agents**. - ---- - -## ๐Ÿ“ž Support - -For questions about these changes: -- **GitHub Issues**: Bug reports -- **Discussions**: Questions and ideas -- **Discord**: Community support -- **Documentation**: All 7 new docs explain everything - ---- - -## ๐ŸŽฏ Quick Start with New Features - -### For Users -``` -1. Update to latest version -2. Try: "Open Chrome and navigate to YouTube" -3. Watch it work reliably with window targeting! -4. Try complex tasks (now handles 25 steps) -5. Notice faster screenshot processing (ONNX ready) -``` - -### For Developers -``` -1. Check new prompts in ToolConfig.cs -2. Use window-targeted keyboard methods -3. Take advantage of 25-step capacity -4. Read COMPUTER_USE_SYSTEM_PROMPTS.md for patterns -5. Build on the enhanced multi-agent system -``` - ---- - -**All changes committed, tested, and production-ready!** ๐Ÿš€ diff --git a/UI_DOCS_LINK_UPDATE.md b/UI_DOCS_LINK_UPDATE.md deleted file mode 100644 index bbb1924..0000000 --- a/UI_DOCS_LINK_UPDATE.md +++ /dev/null @@ -1,57 +0,0 @@ -# UI Documentation Link Updated - -## Changes Made - -Updated the documentation link in the FlowVision UI to point to the new GitHub Pages documentation site. - -### File Modified -- **`FlowVision/Form1.cs`** - Updated `documentationToolStripMenuItem_Click` method - -### Changes -**Before:** -- URL: `https://github.com/flowdevs-io/Recursive-Control/wiki` -- Fallback message: `Visit: https://github.com/flowdevs-io/Recursive-Control/wiki` - -**After:** -- URL: `https://flowdevs-io.github.io/Recursive-Control` -- Fallback message: `Visit: https://flowdevs-io.github.io/Recursive-Control` - -## Location in UI - -Users can access documentation via: -- **Menu**: Help โ†’ Documentation -- This opens the comprehensive GitHub Pages documentation site - -## What Users Will See - -When clicking the documentation menu item, users will now be taken to: -https://flowdevs-io.github.io/Recursive-Control - -This provides: -- Complete installation guide -- Getting started tutorial -- API reference for developers -- Multi-agent architecture explanation -- Troubleshooting guide -- FAQ section - -## Branch & Pull Request - -- **Branch**: `update-docs-link` -- **Base**: `master` -- **Files Changed**: 1 file, 2 lines modified - -## Next Steps - -1. Create pull request: https://github.com/flowdevs-io/Recursive-Control/pull/new/update-docs-link -2. Review and merge the PR -3. Users will get the new documentation link in the next build - -## Note - -The "About" dialog (Help โ†’ About) still shows the main repository link (`github.com/flowdevs-io/Recursive-Control`) which is correct - that points to the project home, while the documentation link now specifically points to the documentation site. - ---- - -**Status**: โœ… Changes committed and pushed to `update-docs-link` branch -**Ready for**: Pull request and merge diff --git a/UI_IMPROVEMENTS.md b/UI_IMPROVEMENTS.md deleted file mode 100644 index 0527873..0000000 --- a/UI_IMPROVEMENTS.md +++ /dev/null @@ -1,609 +0,0 @@ -# Novel UI Improvements for Recursive Control - -## Date: October 2, 2025 - -## Overview - -We've added **interactive, user-friendly UI enhancements** that make Recursive Control more powerful, transparent, and easier to troubleshoot. These improvements focus on giving users visibility into what's happening and making the system more engaging. - ---- - -## ๐ŸŽ **New Features** - -### 1. **Chat Export System** ๐Ÿ“ค - -Export your conversations in multiple formats for debugging, sharing, or documentation. - -#### Features: -- **Export to JSON**: Machine-readable format with timestamps -- **Export to Markdown**: Human-readable format for documentation -- **Debug Export**: Includes chat + plugin usage logs for troubleshooting -- **Copy to Clipboard**: Quick copy for pasting elsewhere - -#### Access: -``` -File Menu โ†’ Export Chat โ†’ [Choose Format] -``` - -#### Formats: - -**JSON Export**: -```json -{ - "ExportTime": "2025-10-02 21:30:45", - "MessageCount": 15, - "Messages": [ - { - "Timestamp": "2025-10-02T21:25:10", - "Author": "You", - "Content": "Open Chrome" - }, - { - "Timestamp": "2025-10-02T21:25:12", - "Author": "AI", - "Content": "Chrome has been opened successfully" - } - ] -} -``` - -**Markdown Export**: -```markdown -# Chat Export - 2025-10-02 21:30:45 - -**Total Messages:** 15 - ---- - -## You -*2025-10-02T21:25:10* - -Open Chrome - ---- - -## AI -*2025-10-02T21:25:12* - -Chrome has been opened successfully - ---- -``` - -**Debug Export** (with Tool Calls): -```markdown -# Debugging Chat Export -**Export Time:** 2025-10-02 21:30:45 -**Total Messages:** 15 - -## Chat Messages - -### You - 2025-10-02T21:25:10 -``` -Open Chrome -``` - -### AI - 2025-10-02T21:25:12 -``` -Chrome has been opened successfully -``` - ---- - -## Plugin Usage Log - -``` -[21:25:10] WindowSelectionPlugin.ListWindowHandles -[21:25:11] ExecuteCommand: chrome.exe -[21:25:12] WindowSelectionPlugin.ForegroundSelect (12345678) -``` -``` - -#### Use Cases: -- **Debugging**: Export with tool calls to diagnose issues -- **Documentation**: Share workflows in markdown -- **Analysis**: Parse JSON exports programmatically -- **Support**: Send debug logs to support team -- **Training**: Create tutorials from actual interactions - ---- - -### 2. **Execution Visualizer** ๐ŸŽฏ - -Real-time visual display of step-by-step execution progress. - -#### Features: -- **Step-by-step display**: See each action as it happens -- **Status icons**: โณ Pending, โš™๏ธ In Progress, โœ… Completed, โŒ Failed -- **Progress bar**: Overall completion percentage -- **Color-coded steps**: Visual feedback for status -- **Auto-scroll**: Follows current step automatically - -#### Visual Layout: -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Execution Progress โ”‚ -โ”‚ Status: Step 3/10: Clicking element โ”‚ -โ”‚ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ 30% โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ #1 โœ… Take screenshot โ”‚ -โ”‚ #2 โœ… Find window handle โ”‚ -โ”‚ #3 โš™๏ธ Click element (in progress) โ”‚ -โ”‚ #4 โณ Verify action โ”‚ -โ”‚ #5 โณ Continue workflow โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -#### Color Scheme: -- **White/Gray**: Pending (not started) -- **Light Blue**: In Progress (currently executing) -- **Light Green**: Completed (success) -- **Light Red**: Failed (error occurred) -- **Light Gray**: Skipped (intentionally skipped) - -#### Benefits: -- **Transparency**: See exactly what the AI is doing -- **Confidence**: Visual feedback builds trust -- **Debugging**: Identify where failures occur -- **Learning**: Understand AI's problem-solving approach -- **Engagement**: Interactive feel vs black box - ---- - -### 3. **Activity Monitor** ๐Ÿ“Š - -Real-time system status and activity logging. - -#### Features: -- **Status Indicators**: AI, ONNX, Browser states -- **Activity Log**: Color-coded event stream -- **Export Capability**: Save logs for analysis -- **Auto-scroll**: Always shows latest activity -- **Level Filtering**: Debug, Info, Success, Warning, Error - -#### Visual Layout: -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ ๐Ÿค– AI: Processing (Blue) โ”‚ -โ”‚ ๐Ÿ‘๏ธ ONNX: Ready (Green) โ”‚ -โ”‚ ๐ŸŒ Browser: Active - Chrome (Green)โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ [21:30:45] โ„น๏ธ System: Started task โ”‚ -โ”‚ [21:30:46] โœ… ONNX: Screenshot OK โ”‚ -โ”‚ [21:30:47] โ„น๏ธ Planner: Step 1/10 โ”‚ -โ”‚ [21:30:48] โš ๏ธ Warning: Slow resp. โ”‚ -โ”‚ [21:30:49] โœ… Success: Task done โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -#### Icon Legend: -- ๐Ÿ” **Debug**: Detailed diagnostic info -- โ„น๏ธ **Info**: General information -- โœ… **Success**: Positive outcome -- โš ๏ธ **Warning**: Potential issue -- โŒ **Error**: Failure or problem - -#### Benefits: -- **Awareness**: Know system state at a glance -- **Monitoring**: Watch AI activity in real-time -- **Diagnostics**: Track down performance issues -- **Documentation**: Export for issue reports -- **Transparency**: No hidden operations - ---- - -## ๐ŸŽจ **UI Philosophy** - -### Interactive & Transparent -Users should **see** what's happening, not guess. Every action should have visual feedback. - -### Informative, Not Overwhelming -Show important information clearly, hide details until needed. Progressive disclosure. - -### Engaging Experience -Computer control should feel **interactive** and **responsive**, not robotic. - -### Debugging-Friendly -When things go wrong, users should have the tools to understand why. - ---- - -## ๐Ÿ“‹ **Implementation Details** - -### ChatExporter Class - -**Location**: `FlowVision/lib/Classes/ChatExporter.cs` - -**Methods**: -```csharp -// Export to JSON format -ChatExporter.ExportToJson(chatHistory); - -// Export to Markdown format -ChatExporter.ExportToMarkdown(chatHistory); - -// Export with plugin logs for debugging -ChatExporter.ExportWithToolCalls(chatHistory); - -// Quick copy to clipboard -ChatExporter.CopyToClipboard(chatHistory); -``` - -**Features**: -- Save file dialog with format-appropriate defaults -- Automatic filename with timestamp -- Error handling with user feedback -- Includes plugin usage logs in debug export - ---- - -### ExecutionVisualizer Component - -**Location**: `FlowVision/lib/Classes/UI/ExecutionVisualizer.cs` - -**Usage**: -```csharp -var visualizer = new ExecutionVisualizer(); - -// Start execution -visualizer.StartExecution(totalSteps: 10); - -// Add steps -visualizer.AddStep("Take screenshot"); -visualizer.AddStep("Click button"); - -// Update step status -visualizer.UpdateStep(0, StepStatus.InProgress); -visualizer.UpdateStep(0, StepStatus.Completed, "Screenshot captured"); - -// Complete -visualizer.CompleteExecution(success: true); -``` - -**Features**: -- Fluent API for easy integration -- Real-time visual updates -- Auto-scrolling to current step -- Color-coded status indicators -- Progress bar for overall completion - ---- - -### ActivityMonitor Component - -**Location**: `FlowVision/lib/Classes/UI/ActivityMonitor.cs` - -**Usage**: -```csharp -var monitor = new ActivityMonitor(); - -// Update system status -monitor.UpdateAIStatus("Processing", Color.Blue); -monitor.UpdateONNXStatus("Ready", Color.Green); -monitor.UpdateBrowserStatus("Active - Chrome", Color.Green); - -// Log activities -monitor.LogActivity("System", "Task started", ActivityLevel.Info); -monitor.LogActivity("ONNX", "Screenshot captured", ActivityLevel.Success); -monitor.LogActivity("Planner", "Step 1/10", ActivityLevel.Info); -monitor.LogActivity("Network", "Slow response", ActivityLevel.Warning); -monitor.LogActivity("Task", "Completed successfully", ActivityLevel.Success); - -// Export log -monitor.ExportLog(); -``` - -**Features**: -- Thread-safe updates -- Color-coded by severity -- Icon-based visual language -- Timestamp for each entry -- Export capability - ---- - -## ๐Ÿš€ **Usage Examples** - -### Example 1: Debugging a Failed Task - -**Scenario**: User reports "AI clicked wrong button" - -**Steps**: -1. File โ†’ Export Chat โ†’ Export Debug Log -2. Open exported file -3. See exact sequence of actions -4. Find tool calls that executed -5. Identify incorrect window handle or coordinates -6. Fix and retest - -**Export Shows**: -``` -### AI - 21:30:47 -``` -Clicking element at coordinates [300, 250] -``` - -## Plugin Usage Log -``` -[21:30:47] MousePlugin.ClickOnWindow(12345678, [300, 250, 500, 310], true, 1) -[21:30:47] Result: Clicked successfully -``` -``` - -**Analysis**: Wrong window handle! Should have been 87654321 (different Chrome window). - ---- - -### Example 2: Monitoring Complex Workflow - -**Scenario**: 15-step automation task - -**Execution Visualizer Shows**: -``` -โœ… Step 1/15: Screenshot captured -โœ… Step 2/15: Window found (Chrome) -โœ… Step 3/15: Brought to foreground -โš™๏ธ Step 4/15: Typing search query (IN PROGRESS) -โณ Step 5/15: Press Enter (PENDING) -โณ Step 6/15: Wait for results (PENDING) -... -``` - -**Activity Monitor Shows**: -``` -[21:30:45] โ„น๏ธ System: Starting 15-step workflow -[21:30:46] โœ… ONNX: Screenshot captured (640x480) -[21:30:47] โ„น๏ธ Planner: Step 4/15 - Type query -[21:30:48] โš™๏ธ Keyboard: SendKeyToWindow(12345678, "Python tutorials") -``` - -**Benefits**: -- User sees progress in real-time -- Confidence that system is working -- Can identify if step is taking too long -- Visual confirmation of each action - ---- - -### Example 3: Sharing Workflow - -**Scenario**: User wants to document their automation - -**Steps**: -1. Complete automation task -2. File โ†’ Export Chat โ†’ Export to Markdown -3. Share markdown file -4. Others can see exact conversation and results - -**Result**: Clean, readable documentation of the workflow. - ---- - -## ๐Ÿ’ก **Novel Features** - -### What Makes These Improvements Unique? - -#### 1. Debug Export with Tool Calls -**Novel**: Most chat apps only export conversations. We export the **actual tool calls** that were executed, making debugging trivial. - -**Impact**: Support teams can see exactly what the AI did, not just what it said. - -#### 2. Real-Time Execution Visualization -**Novel**: Not just a "loading" spinnerโ€”users see **each step** with status, icon, and color. - -**Impact**: Builds trust and understanding. Users learn how the AI solves problems. - -#### 3. Activity Monitor Integration -**Novel**: System status + activity log in one place with color-coded severity. - -**Impact**: Power users can monitor system health, casual users see reassuring status indicators. - -#### 4. Multi-Format Export -**Novel**: One feature, four export formats (JSON, Markdown, Debug, Clipboard) for different use cases. - -**Impact**: Flexibility for developers (JSON), documentation writers (Markdown), support (Debug), and quick sharing (Clipboard). - ---- - -## ๐ŸŽฏ **Future Enhancements** - -### Potential Additions - -**1. Element Highlighting**: -- Overlay on screenshots showing where AI will click -- Visual confirmation before execution -- Red outline = target, Green = success - -**2. Timeline View**: -- Horizontal timeline of all steps -- Click to see details of each step -- Duration visualization - -**3. Interactive Step Editing**: -- Pause execution -- Modify next step -- Resume with changes - -**4. Voice Feedback**: -- Optional audio cues for step completion -- "Step 5 complete" announcement -- Accessibility feature - -**5. Analytics Dashboard**: -- Success rate over time -- Most used features -- Average steps per task -- Performance metrics - -**6. Collaboration Features**: -- Share workflows with team -- Import exported workflows -- Template library - ---- - -## ๐Ÿ“Š **Metrics** - -### Before UI Improvements: -- **Visibility**: Low (black box behavior) -- **Debugging**: Hard (no logs, no exports) -- **Engagement**: Passive (waiting for results) -- **Trust**: Uncertain (can't see what's happening) - -### After UI Improvements: -- **Visibility**: High (see every step) -- **Debugging**: Easy (export with tool calls) -- **Engagement**: Active (watch progress real-time) -- **Trust**: Strong (transparency builds confidence) - ---- - -## ๐Ÿ”ง **Developer Guide** - -### Adding to Your UI - -**Execution Visualizer**: -```csharp -// In your form -private ExecutionVisualizer visualizer; - -void InitializeVisualizer() -{ - visualizer = new ExecutionVisualizer - { - Dock = DockStyle.Right, - Width = 400 - }; - this.Controls.Add(visualizer); -} - -// During execution -visualizer.StartExecution(steps.Count); -foreach (var step in steps) -{ - visualizer.AddStep(step.Description); -} -``` - -**Activity Monitor**: -```csharp -// In your form -private ActivityMonitor monitor; - -void InitializeMonitor() -{ - monitor = new ActivityMonitor - { - Dock = DockStyle.Right, - Width = 300 - }; - this.Controls.Add(monitor); -} - -// Log activities -monitor.LogActivity("AI", "Task started", ActivityLevel.Info); -``` - ---- - -## โœ… **Testing Checklist** - -### Chat Export -- [ ] JSON export creates valid JSON file -- [ ] Markdown export is readable -- [ ] Debug export includes plugin logs -- [ ] Clipboard copy works -- [ ] Timestamps are correct -- [ ] Large chats export without errors - -### Execution Visualizer -- [ ] Steps appear in correct order -- [ ] Status updates work (Pending โ†’ InProgress โ†’ Completed) -- [ ] Progress bar updates correctly -- [ ] Auto-scroll follows current step -- [ ] Colors change based on status -- [ ] Failed steps show in red - -### Activity Monitor -- [ ] Status indicators update correctly -- [ ] Activity log shows timestamped entries -- [ ] Color coding works for all levels -- [ ] Export log creates valid file -- [ ] Thread-safe (no UI freezing) -- [ ] Icons display correctly - ---- - -## ๐Ÿ“ **User Documentation** - -### Quick Start: Exporting Chat - -1. Click **File** menu -2. Select **Export Chat** -3. Choose format: - - **JSON**: For developers/programmers - - **Markdown**: For documentation - - **Debug Log**: For troubleshooting - - **Clipboard**: For quick sharing -4. Select save location -5. Done! File is saved - -### Quick Start: Monitoring Execution - -1. Enable Multi-Agent Mode (for step-by-step execution) -2. Start a task -3. Watch the execution visualizer on the right -4. See each step complete with checkmarks -5. Progress bar shows overall completion - -### Quick Start: Activity Monitoring - -1. Open Activity Monitor panel -2. Watch real-time status updates -3. See color-coded activity log -4. Export log if needed for troubleshooting - ---- - -## ๐ŸŽ‰ **Impact Summary** - -### What We Achieved: - -1. **Transparency**: Users can see exactly what's happening -2. **Debugability**: Easy to export and analyze -3. **Engagement**: Interactive, visual feedback -4. **Trust**: Builds confidence through visibility -5. **Professionalism**: Polished, modern UI experience - -### User Benefits: - -- โœ… Never wonder "is it working?" -- โœ… Debug issues yourself before asking for help -- โœ… Share workflows easily -- โœ… Learn how AI solves problems -- โœ… Feel in control, not helpless - -### Developer Benefits: - -- โœ… Easy to diagnose user issues -- โœ… Export format works with existing tools -- โœ… Clean component architecture -- โœ… Extensible for future features -- โœ… Well-documented APIs - ---- - -## ๐Ÿš€ **Build Status** - -``` -โœ… All UI components compile successfully -โœ… Chat export integrated into File menu -โœ… Execution visualizer ready to use -โœ… Activity monitor ready to use -โœ… No breaking changes -โœ… Backward compatible -``` - ---- - -**These UI improvements transform Recursive Control from a functional tool into an engaging, transparent, and user-friendly platform. The focus on visibility, debugging, and interactivity makes it a joy to use!** ๐ŸŽจโœจ diff --git a/WIKI_GENERATED.md b/WIKI_GENERATED.md deleted file mode 100644 index cda4d39..0000000 --- a/WIKI_GENERATED.md +++ /dev/null @@ -1,600 +0,0 @@ -# Wiki Generation Complete! ๐Ÿ“š - -## Overview - -I've generated a **comprehensive GitHub Wiki** for Recursive Control with 12 complete documentation files ready to publish. - ---- - -## ๐Ÿ“š Wiki Structure - -### Core Documentation (Ready to Publish) - -``` -wiki/ -โ”œโ”€โ”€ README.md # Wiki maintenance guide -โ”œโ”€โ”€ Home.md # Main landing page โญ -โ”œโ”€โ”€ Installation.md # Complete setup guide -โ”œโ”€โ”€ Getting-Started.md # First tasks tutorial -โ”œโ”€โ”€ Multi-Agent-Architecture.md # Technical deep dive -โ”œโ”€โ”€ FAQ.md # Common questions -โ”œโ”€โ”€ Troubleshooting.md # Problem solving -โ””โ”€โ”€ API-Reference.md # Developer docs -``` - -### Additional Reference Material - -``` -wiki/ -โ”œโ”€โ”€ Blog-Post-v2.0.md # Version 2.0 announcement -โ”œโ”€โ”€ System-Prompts-Reference.md # Complete prompt library -โ”œโ”€โ”€ UI-Features.md # UI improvements guide -โ””โ”€โ”€ UI-Redesign.md # Modern UI documentation -``` - -**Total: 12 comprehensive markdown files** - ---- - -## ๐Ÿ“– Page Descriptions - -### 1. **Home.md** - Wiki Landing Page - -**Content:** -- Welcome message with badges -- Quick start links (Installation โ†’ Setup โ†’ First Task) -- Core documentation navigation -- Feature highlights -- Multi-agent system overview -- Key features explanation -- Documentation structure -- Learning paths (Beginner โ†’ Intermediate โ†’ Advanced) -- Recent updates (Version 2.0) -- Community & support links - -**Word Count:** ~3,500 words -**Status:** โœ… Production ready - ---- - -### 2. **Installation.md** - Complete Setup Guide - -**Content:** -- System requirements -- Quick installation (download release) -- Build from source instructions -- Initial setup wizard -- AI provider configuration (Azure/LM Studio/GitHub) -- Plugin configuration -- Configuration file locations -- ONNX model setup -- Troubleshooting installation -- Update procedures -- Uninstallation guide -- Verification checklist - -**Word Count:** ~4,000 words -**Status:** โœ… Production ready -**Features:** Step-by-step, code examples, troubleshooting - ---- - -### 3. **Getting-Started.md** - First Tasks Tutorial - -**Content:** -- Your first task (Open Notepad) -- Task 2: List open windows -- Task 3: Type text -- Common tasks by category - - Desktop automation - - Window management - - Keyboard control - - Screenshots - - Web browsing - - Command execution -- Understanding AI responses -- Best practices (be specific, use window names) -- Multi-agent mode introduction -- Task examples by category -- Monitoring execution -- Export & debugging -- Common issues -- Next steps - -**Word Count:** ~5,000 words -**Status:** โœ… Production ready -**Features:** Examples, best practices, troubleshooting - ---- - -### 4. **Multi-Agent-Architecture.md** - Technical Deep Dive - -**Content:** -- Single vs Multi-Agent comparison -- The three agents explained: - - ๐ŸŽฏ Coordinator (routing & user interface) - - ๐Ÿ“‹ Planner (sequential breakdown) - - โšก Actioner (execution) -- Execution flow diagrams -- Step-by-step example with 12-step workflow -- Per-agent model configuration -- Enable/disable multi-agent mode -- Performance characteristics -- Success rate comparison -- When to use each mode -- Best practices for users & developers -- Debugging multi-agent execution -- Advanced configuration -- Custom iteration limits -- Custom agent prompts - -**Word Count:** ~6,500 words -**Status:** โœ… Production ready -**Features:** Diagrams, examples, advanced topics - ---- - -### 5. **FAQ.md** - Frequently Asked Questions - -**Topics to Cover:** -- General questions -- Setup & configuration -- Features & capabilities -- Troubleshooting -- Performance -- Privacy & security -- Pricing & models -- Development & customization - -**Status:** ๐Ÿ“ Template ready (needs specific Q&A) - ---- - -### 6. **Troubleshooting.md** - Problem Solving Guide - -**Topics to Cover:** -- Installation issues -- Configuration problems -- Runtime errors -- Performance issues -- Plugin problems -- AI provider issues -- Network/connectivity -- Permission issues -- Log file analysis -- Debug export usage - -**Status:** ๐Ÿ“ Template ready (needs specific solutions) - ---- - -### 7. **API-Reference.md** - Developer Documentation - -**Topics to Cover:** -- Plugin architecture -- Creating custom plugins -- Tool function API -- Configuration objects -- Event system -- Extension points -- Code examples -- Best practices -- Testing plugins - -**Status:** ๐Ÿ“ Template ready (needs API details) - ---- - -### 8-12. **Reference Documentation** - -- **Blog-Post-v2.0.md**: Complete version 2.0 announcement (14,000 words) -- **System-Prompts-Reference.md**: All system prompts with examples (15,500 words) -- **UI-Features.md**: UI improvements documentation (15,600 words) -- **UI-Redesign.md**: Modern UI redesign guide (13,800 words) - -**Total Reference Words:** ~59,000 words - ---- - -## ๐Ÿ“Š Statistics - -### Content Metrics - -| Metric | Value | -|--------|-------| -| **Total Files** | 12 markdown files | -| **Core Pages** | 7 (Home, Install, Start, Architecture, FAQ, Troubleshoot, API) | -| **Reference Pages** | 4 (Blog, Prompts, UI Features, UI Redesign) | -| **Total Words** | ~70,000+ words | -| **Code Examples** | 100+ snippets | -| **Diagrams** | Mermaid flow charts | -| **Images** | Badges, icons, emoji | - -### Completeness - -| Page | Words | Status | Ready | -|------|-------|--------|-------| -| Home | 3,500 | Complete | โœ… | -| Installation | 4,000 | Complete | โœ… | -| Getting Started | 5,000 | Complete | โœ… | -| Multi-Agent | 6,500 | Complete | โœ… | -| FAQ | TBD | Template | ๐Ÿ“ | -| Troubleshooting | TBD | Template | ๐Ÿ“ | -| API Reference | TBD | Template | ๐Ÿ“ | -| **Core Total** | **19,000+** | **75%** | **๐ŸŸข** | -| **Reference** | **59,000** | **100%** | **โœ…** | -| **Grand Total** | **78,000+** | **85%** | **๐ŸŸข** | - ---- - -## ๐Ÿš€ Publishing to GitHub Wiki - -### Method 1: Web Interface (Manual) - -1. **Go to Repository** - ``` - https://github.com/flowdevs-io/Recursive-Control/wiki - ``` - -2. **Create Each Page** - - Click "New Page" - - Set page title (e.g., "Installation") - - Copy content from `wiki/Installation.md` - - Click "Save Page" - -3. **Repeat for All Pages** - - Home (set as wiki homepage) - - Installation - - Getting-Started - - Multi-Agent-Architecture - - FAQ - - Troubleshooting - - API-Reference - ---- - -### Method 2: Git Clone (Automated) โญ - -```bash -# Clone the wiki repository -git clone https://github.com/flowdevs-io/Recursive-Control.wiki.git - -# Copy all markdown files -cp wiki/*.md Recursive-Control.wiki/ - -# Commit and push -cd Recursive-Control.wiki -git add . -git commit -m "Complete wiki documentation - 12 pages, 70K+ words" -git push origin master -``` - -**This is the recommended method** - much faster and preserves formatting. - ---- - -## โœจ Wiki Features - -### Navigation - -Every page includes: -- โœ… Back to Home link -- โœ… Previous/Next page links -- โœ… Table of contents -- โœ… Section anchors - -Example: -```markdown -

- โ† Back to Home | - Next: Getting Started โ†’ -

-``` - ---- - -### Visual Elements - -**Emoji Icons:** -- ๐Ÿš€ Quick Start -- ๐Ÿค– AI Agents -- ๐Ÿ”ง Configuration -- ๐Ÿ“Š Monitoring -- ๐Ÿ“ค Export -- ๐ŸŽฏ Execution -- โ„น๏ธ Information - -**Badges:** -```markdown -Version -Platform -License -``` - -**Code Blocks:** -- Syntax highlighting -- Language specified -- Copy-paste ready - -**Diagrams:** -```mermaid -graph TD - User[User Input] --> Coord[Coordinator] - Coord --> Plan[Planner] - Plan --> Act[Actioner] -``` - ---- - -### Interactive Elements - -**Checklists:** -```markdown -- [ ] Application launches -- [ ] AI provider configured -- [ ] Test connection succeeds -- [ ] Basic command works -``` - -**Expandable Sections:** -```markdown -### Step 1: Download -
-Click to expand -Detailed instructions here... -
-``` - -**Tables:** -```markdown -| Feature | Single Agent | Multi-Agent | -|---------|--------------|-------------| -| Speed | Fast | Moderate | -| Complexity | Simple | Complex | -``` - ---- - -## ๐ŸŽฏ Content Highlights - -### Home Page Features - -- **Quick Start Path**: Install โ†’ Setup โ†’ First Task in 3 clicks -- **Feature Overview**: Key capabilities explained -- **Multi-Agent Diagram**: Visual workflow -- **Learning Paths**: Beginner to Advanced -- **Recent Updates**: Version 2.0 highlights -- **Community Links**: Discord, GitHub, LinkedIn - -### Installation Guide Features - -- **Two Methods**: Release download + Build from source -- **Three AI Providers**: Azure, LM Studio, GitHub Models -- **Plugin Configuration**: Enable/disable features -- **Verification Steps**: Ensure everything works -- **Troubleshooting**: Common installation issues -- **Update/Uninstall**: Complete lifecycle - -### Getting Started Features - -- **Three First Tasks**: Progressive learning -- **Common Tasks**: By category -- **Best Practices**: Do's and don'ts -- **Multi-Agent Intro**: When to use -- **Export Guide**: Debug your workflows -- **Next Steps**: Continue learning - -### Architecture Deep Dive - -- **Complete Explanation**: All 3 agents -- **Execution Flow**: Step-by-step with example -- **Configuration**: Per-agent model selection -- **Performance**: Success rates, timing -- **When to Use**: Decision guide -- **Debugging**: How to troubleshoot - ---- - -## ๐Ÿ“ฑ Mobile Friendly - -All pages are: -- โœ… Responsive markdown -- โœ… No fixed widths -- โœ… Mobile-readable code blocks -- โœ… Collapsible sections -- โœ… Touch-friendly navigation - ---- - -## ๐Ÿ” SEO & Discovery - -### Keywords Included - -- AI computer control -- Windows automation -- Multi-agent system -- Natural language -- Desktop automation -- Browser automation -- PowerShell automation -- Command execution -- Screen capture -- ONNX inference - -### Internal Linking - -Every page links to: -- Home page -- Related topics -- Previous/next in sequence -- Reference materials - -### External Links - -- GitHub repository -- Discord community -- Issue tracker -- LinkedIn page -- Documentation sites - ---- - -## ๐ŸŽ“ Learning Paths - -### Beginner Path (In Wiki) - -``` -Home โ†’ Installation โ†’ Getting Started โ†’ First Task Complete! -Time: ~30 minutes -``` - -### Intermediate Path - -``` -Multi-Agent Mode โ†’ Browser Automation โ†’ Plugin Configuration -Time: ~2 hours -``` - -### Advanced Path - -``` -Multi-Agent Architecture โ†’ System Prompts โ†’ Plugin Development -Time: ~4 hours -``` - ---- - -## ๐Ÿ“ Content To-Do - -### Pages Needing Completion - -**FAQ.md** - Template ready, needs: -- [ ] 50+ Q&A pairs -- [ ] Categorized by topic -- [ ] Link to detailed pages -- [ ] Est. time: 2-3 hours - -**Troubleshooting.md** - Template ready, needs: -- [ ] Common error messages -- [ ] Step-by-step solutions -- [ ] Debug techniques -- [ ] Log analysis guide -- [ ] Est. time: 2-3 hours - -**API-Reference.md** - Template ready, needs: -- [ ] Plugin API documentation -- [ ] Function signatures -- [ ] Parameter descriptions -- [ ] Return values -- [ ] Code examples -- [ ] Est. time: 4-5 hours - -### Additional Pages (Future) - -- [ ] Browser-Automation.md (detailed Playwright guide) -- [ ] Plugin-Development.md (create custom tools) -- [ ] Performance-Tuning.md (optimize for your use case) -- [ ] Use-Cases.md (real-world examples) -- [ ] Contributing.md (how to contribute) -- [ ] Release-Notes.md (version history) - ---- - -## โœ… What's Ready Now - -### Publish Today - -These 4 pages are **100% complete** and ready to publish: - -1. โœ… **Home.md** - Complete landing page -2. โœ… **Installation.md** - Full setup guide -3. โœ… **Getting-Started.md** - Tutorial with examples -4. โœ… **Multi-Agent-Architecture.md** - Technical deep dive - -**Plus 4 reference pages:** -- โœ… Blog-Post-v2.0.md -- โœ… System-Prompts-Reference.md -- โœ… UI-Features.md -- โœ… UI-Redesign.md - -**Total Ready:** 8 pages, ~78,000 words - ---- - -## ๐Ÿš€ Quick Publish Commands - -```bash -# Navigate to repository -cd Recursive-Control - -# Clone wiki -git clone https://github.com/flowdevs-io/Recursive-Control.wiki.git - -# Copy ready pages -cp wiki/Home.md Recursive-Control.wiki/ -cp wiki/Installation.md Recursive-Control.wiki/ -cp wiki/Getting-Started.md Recursive-Control.wiki/ -cp wiki/Multi-Agent-Architecture.md Recursive-Control.wiki/ - -# Copy reference -cp wiki/Blog-Post-v2.0.md Recursive-Control.wiki/ -cp wiki/System-Prompts-Reference.md Recursive-Control.wiki/ -cp wiki/UI-Features.md Recursive-Control.wiki/ -cp wiki/UI-Redesign.md Recursive-Control.wiki/ - -# Publish -cd Recursive-Control.wiki -git add . -git commit -m "Initial wiki: 8 pages, core documentation complete" -git push origin master -``` - ---- - -## ๐Ÿ“Š Impact - -### Before (No Wiki) -- โŒ Users had to read source code -- โŒ No centralized documentation -- โŒ Hard to discover features -- โŒ Steep learning curve -- โŒ Support burden high - -### After (With Wiki) -- โœ… Comprehensive documentation -- โœ… Step-by-step guides -- โœ… Feature discovery easy -- โœ… Gentle learning curve -- โœ… Self-service support -- โœ… Professional appearance -- โœ… Community resource - ---- - -## ๐ŸŽ‰ Summary - -**What Was Generated:** -- ๐Ÿ“š 12 markdown files -- ๐Ÿ“ 78,000+ words -- ๐ŸŽฏ 4 core pages (100% complete) -- ๐Ÿ“– 4 reference pages (100% complete) -- ๐Ÿ“‹ 3 templates (FAQ, Troubleshooting, API) -- ๐ŸŽจ Emoji visual language throughout -- ๐Ÿ”— Complete internal navigation -- ๐Ÿ“ฑ Mobile-friendly formatting -- ๐ŸŽ“ Learning paths defined -- โœ… Production ready - -**Publishing Status:** -- โœ… 8 pages ready to publish today -- ๐Ÿ“ 3 templates ready for content -- ๐ŸŽฏ Core documentation complete (85%) -- ๐Ÿš€ Can go live immediately - -**Next Steps:** -1. Publish 8 ready pages to GitHub Wiki -2. Complete FAQ, Troubleshooting, API (6-8 hours) -3. Add future enhancement pages -4. Promote wiki to community - -**The wiki is comprehensive, professional, and ready to serve your community!** ๐Ÿ“šโœจ diff --git a/agent-framework-README.md b/agent-framework-README.md deleted file mode 100644 index dfdf896..0000000 --- a/agent-framework-README.md +++ /dev/null @@ -1,166 +0,0 @@ -![Microsoft Agent Framework](docs/assets/readme-banner.png) - -# Welcome to Microsoft Agent Framework! - -[![Microsoft Azure AI Foundry Discord](https://dcbadge.limes.pink/api/server/b5zjErwbQM?style=flat)](https://discord.gg/b5zjErwbQM) -[![MS Learn Documentation](https://img.shields.io/badge/MS%20Learn-Documentation-blue)](https://learn.microsoft.com/en-us/agent-framework/) -[![PyPI](https://img.shields.io/pypi/v/agent-framework)](https://pypi.org/project/agent-framework/) -[![NuGet](https://img.shields.io/nuget/v/Microsoft.Agents.AI)](https://www.nuget.org/profiles/MicrosoftAgentFramework/) - -Welcome to Microsoft's comprehensive multi-language framework for building, orchestrating, and deploying AI agents with support for both .NET and Python implementations. This framework provides everything from simple chat agents to complex multi-agent workflows with graph-based orchestration. - -

- - Watch the full Agent Framework introduction (30 min) - -

-

- - Watch the full Agent Framework introduction (30 min) - -

- -## ๐Ÿ“‹ Getting Started - -### ๐Ÿ“ฆ Installation - -Python - -```bash -pip install agent-framework --pre -# This will install all sub-packages, see `python/packages` for individual packages. -# It may take a minute on first install on Windows. -``` - -.NET - -```bash -dotnet add package Microsoft.Agents.AI -``` - -### ๐Ÿ“š Documentation - -- **[Overview](https://learn.microsoft.com/agent-framework/overview/agent-framework-overview)** - High level overview of the framework -- **[Quick Start](https://learn.microsoft.com/agent-framework/tutorials/quick-start)** - Get started with a simple agent -- **[Tutorials](https://learn.microsoft.com/agent-framework/tutorials/overview)** - Step by step tutorials -- **[User Guide](https://learn.microsoft.com/en-us/agent-framework/user-guide/overview)** - In-depth user guide for building agents and workflows -- **[Migration from Semantic Kernel](https://learn.microsoft.com/en-us/agent-framework/migration-guide/from-semantic-kernel)** - Guide to migrate from Semantic Kernel -- **[Migration from AutoGen](https://learn.microsoft.com/en-us/agent-framework/migration-guide/from-autogen)** - Guide to migrate from AutoGen - -### โœจ **Highlights** - -- **Graph-based Workflows**: Connect agents and deterministic functions using data flows with streaming, checkpointing, human-in-the-loop, and time-travel capabilities - - [Python workflows](./python/samples/getting_started/workflows/) | [.NET workflows](./dotnet/samples/GettingStarted/Workflows/) -- **AF Labs**: Experimental packages for cutting-edge features including benchmarking, reinforcement learning, and research initiatives - - [Labs directory](./python/packages/lab/) -- **DevUI**: Interactive developer UI for agent development, testing, and debugging workflows - - [DevUI package](./python/packages/devui/) - -

- - See the DevUI in action - -

-

- - See the DevUI in action (1 min) - -

- -- **Python and C#/.NET Support**: Full framework support for both Python and C#/.NET implementations with consistent APIs - - [Python packages](./python/packages/) | [.NET source](./dotnet/src/) -- **Observability**: Built-in OpenTelemetry integration for distributed tracing, monitoring, and debugging - - [Python observability](./python/samples/getting_started/observability/) | [.NET telemetry](./dotnet/samples/GettingStarted/AgentOpenTelemetry/) -- **Multiple Agent Provider Support**: Support for various LLM providers with more being added continuously - - [Python examples](./python/samples/getting_started/agents/) | [.NET examples](./dotnet/samples/GettingStarted/AgentProviders/) -- **Middleware**: Flexible middleware system for request/response processing, exception handling, and custom pipelines - - [Python middleware](./python/samples/getting_started/middleware/) | [.NET middleware](./dotnet/samples/GettingStarted/Agents/Agent_Step14_Middleware/) - -### ๐Ÿ’ฌ **We want your feedback!** - -- For bugs, please file a [GitHub issue](https://github.com/microsoft/agent-framework/issues). - -## Quickstart - -### Basic Agent - Python - -Create a simple Azure Responses Agent that writes a haiku about the Microsoft Agent Framework - -```python -# pip install agent-framework --pre -# Use `az login` to authenticate with Azure CLI -import os -import asyncio -from agent_framework.azure import AzureOpenAIResponsesClient -from azure.identity import AzureCliCredential - - -async def main(): - # Initialize a chat agent with Azure OpenAI Responses - # the endpoint, deployment name, and api version can be set via environment variables - # or they can be passed in directly to the AzureOpenAIResponsesClient constructor - agent = AzureOpenAIResponsesClient( - # endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], - # deployment_name=os.environ["AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME"], - # api_version=os.environ["AZURE_OPENAI_API_VERSION"], - # api_key=os.environ["AZURE_OPENAI_API_KEY"], # Optional if using AzureCliCredential - credential=AzureCliCredential(), # Optional, if using api_key - ).create_agent( - name="HaikuBot", - instructions="You are an upbeat assistant that writes beautifully.", - ) - - print(await agent.run("Write a haiku about Microsoft Agent Framework.")) - -if __name__ == "__main__": - asyncio.run(main()) -``` - -### Basic Agent - .NET - -```c# -// dotnet add package Microsoft.Agents.AI.OpenAI --prerelease -// dotnet add package Azure.AI.OpenAI -// dotnet add package Azure.Identity -// Use `az login` to authenticate with Azure CLI -using System; -using Azure.AI.OpenAI; -using Azure.Identity; -using Microsoft.Agents.AI; -using OpenAI; - -var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT")!; -var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME")!; - -var agent = new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()) - .GetOpenAIResponseClient(deploymentName) - .CreateAIAgent(name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully."); - -Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework.")); -``` - -## More Examples & Samples - -### Python - -- [Getting Started with Agents](./python/samples/getting_started/agents): basic agent creation and tool usage -- [Chat Client Examples](./python/samples/getting_started/chat_client): direct chat client usage patterns -- [Getting Started with Workflows](./python/samples/getting_started/workflows): basic workflow creation and integration with agents - -### .NET - -- [Getting Started with Agents](./dotnet/samples/GettingStarted/Agents): basic agent creation and tool usage -- [Agent Provider Samples](./dotnet/samples/GettingStarted/AgentProviders): samples showing different agent providers -- [Workflow Samples](./dotnet/samples/GettingStarted/Workflows): advanced multi-agent patterns and workflow orchestration - -## Contributor Resources - -- [Contributing Guide](./CONTRIBUTING.md) -- [Python Development Guide](./python/DEV_SETUP.md) -- [Design Documents](./docs/design) -- [Architectural Decision Records](./docs/decisions) - -## Important Notes - -If you use the Microsoft Agent Framework to build applications that operate with third-party servers or agents, you do so at your own risk. We recommend reviewing all data being shared with third-party servers or agents and being cognizant of third-party practices for retention and location of data. It is your responsibility to manage whether your data will flow outside of your organization's Azure compliance and geographic boundaries and any related implications. diff --git a/AI_PROVIDER_COMPARISON.md b/docs/guides/AI-Provider-Comparison.md similarity index 59% rename from AI_PROVIDER_COMPARISON.md rename to docs/guides/AI-Provider-Comparison.md index ed988de..70f2c45 100644 --- a/AI_PROVIDER_COMPARISON.md +++ b/docs/guides/AI-Provider-Comparison.md @@ -4,10 +4,10 @@ FlowVision now supports **three AI provider options**. Choose the one that fits your needs! -## ๐Ÿ“Š Comparison Table +## ๐Ÿ“Š Quick Comparison -| Feature | Azure OpenAI | LM Studio (Local) | Azure Foundry / GitHub Models | -|---------|--------------|-------------------|-------------------------------| +| Feature | Azure OpenAI | LM Studio (Local) | Azure Foundry | +|---------|--------------|-------------------|---------------| | **Cost** | ๐Ÿ’ฐ Pay per use | โœ… **FREE** | ๐Ÿ’ฐ Pay per use | | **Privacy** | โš ๏ธ Cloud-based | โœ… **100% Local** | โš ๏ธ Cloud-based | | **Internet Required** | โœ… Yes | โœ… **No** (after setup) | โœ… Yes | @@ -18,7 +18,6 @@ FlowVision now supports **three AI provider options**. Choose the one that fits | **API Limits** | โš ๏ธ Rate limits apply | โœ… **No limits** | โš ๏ธ Rate limits apply | | **Model Choice** | Limited to Azure | โœ… **Any LM Studio model** | Limited to provider | | **Offline Mode** | โŒ No | โœ… **YES** | โŒ No | -| **Data Ownership** | โš ๏ธ Shared with Microsoft | โœ… **100% Yours** | โš ๏ธ Shared with provider | ## ๐ŸŽฏ Use Case Recommendations @@ -28,9 +27,8 @@ FlowVision now supports **three AI provider options**. Choose the one that fits - โœ… **Offline operation** (no internet needed) - โœ… **Sensitive data** handling - โœ… **Unlimited usage** without rate limits -- โœ… **Full control** over the model -**Best for**: Privacy-conscious users, offline work, testing, learning, unlimited usage +**Best for**: Privacy-conscious users, offline work, testing, learning ### Choose **Azure OpenAI** if you need: - โœ… **Best performance** without local hardware @@ -39,9 +37,9 @@ FlowVision now supports **three AI provider options**. Choose the one that fits - โœ… **No setup hassle** - works immediately - โœ… **Latest models** (GPT-4, etc.) -**Best for**: Production use, enterprise apps, guaranteed uptime, latest models +**Best for**: Production use, enterprise apps, guaranteed uptime -### Choose **Azure Foundry / GitHub Models** if you need: +### Choose **Azure Foundry** if you need: - โœ… **Free tier** for testing - โœ… **Variety of models** to choose from - โœ… **GitHub integration** @@ -58,19 +56,10 @@ FlowVision now supports **three AI provider options**. Choose the one that fits - **Hardware**: Any PC ### LM Studio -- **RAM**: - - Minimum: 16GB - - Recommended: 32GB+ - - For large models: 64GB+ -- **Storage**: - - Model size: 2GB - 50GB - - Installation: 500MB -- **GPU**: - - Optional but recommended - - NVIDIA (CUDA), AMD (ROCm), Apple Silicon (Metal) -- **Internet**: - - Required for initial download - - Not needed after setup +- **RAM**: 16GB minimum, 32GB+ recommended +- **Storage**: 2GB - 50GB for models +- **GPU**: Optional but recommended for speed +- **Internet**: Required for initial download only ### Azure Foundry - **RAM**: N/A (cloud service) @@ -78,7 +67,7 @@ FlowVision now supports **three AI provider options**. Choose the one that fits - **Internet**: Required - **Hardware**: Any PC -## ๐Ÿ’ฐ Cost Comparison (Example) +## ๐Ÿ’ฐ Cost Comparison ### Scenario: 1,000 requests/day (30,000/month) @@ -88,15 +77,6 @@ FlowVision now supports **three AI provider options**. Choose the one that fits | **Azure OpenAI** | $30-150/month | Depends on model and usage | | **GitHub Models** | Free tier โ†’ Paid | Limited free tier | -### Break-even Analysis - -If you make more than **~100 requests per day**, LM Studio typically pays for itself through: -- No API costs -- No rate limits -- Unlimited usage - -**Initial investment**: $0 (software is free, use existing hardware) - ## โšก Performance Comparison ### Response Time (Average) @@ -108,15 +88,6 @@ If you make more than **~100 requests per day**, LM Studio typically pays for it | **LM Studio (32GB+GPU)** | 1-3s | 3-6s | 5-15s | | **Azure Foundry** | 1-3s | 3-5s | 5-12s | -### Quality Rating (Subjective) - -| Provider | Model | Quality | Tool Calling | -|----------|-------|---------|--------------| -| **Azure OpenAI** | GPT-4 | โญโญโญโญโญ | โญโญโญโญโญ | -| **LM Studio** | Hermes-2-Pro-7B | โญโญโญโญ | โญโญโญโญ | -| **LM Studio** | Mixtral-8x7B | โญโญโญโญโญ | โญโญโญโญ | -| **Azure Foundry** | Various | โญโญโญโญ | โญโญโญโญ | - ## ๐Ÿ” Privacy & Security ### Data Flow @@ -125,56 +96,26 @@ If you make more than **~100 requests per day**, LM Studio typically pays for it ``` Your Data โ†’ Azure Cloud โ†’ Processed โ†’ Returned ``` -- Data sent to Microsoft servers -- Subject to Microsoft privacy policy -- Encrypted in transit and at rest -- May be used for service improvement **LM Studio:** ``` Your Data โ†’ Your Computer โ†’ Processed โ†’ Stays Local ``` -- Data never leaves your machine -- Complete control over data -- No external dependencies -- Perfect for sensitive information **Azure Foundry:** ``` Your Data โ†’ Azure/GitHub Cloud โ†’ Processed โ†’ Returned ``` -- Similar to Azure OpenAI -- May have different retention policies -- Check provider terms of service ## ๐ŸŽฎ Switching Between Providers -### In FlowVision: +FlowVision automatically switches based on your configuration: ``` -Priority Order (Top to Bottom): +Priority Order: 1. LM Studio (if enabled) - โ””โ”€ Check: LMStudioConfig.Enabled = true -2. Multi-Agent Mode (if enabled) - โ””โ”€ Uses Azure OpenAI with multiple agents +2. Multi-Agent Mode (if enabled) 3. Azure OpenAI (default) - โ””โ”€ Always available as fallback -``` - -### Quick Switch Commands: - -**Enable LM Studio:** -```csharp -var config = LMStudioConfig.LoadConfig(); -config.Enabled = true; -config.SaveConfig(); -``` - -**Disable LM Studio (use Azure):** -```csharp -var config = LMStudioConfig.LoadConfig(); -config.Enabled = false; -config.SaveConfig(); ``` **No restart required!** Changes take effect immediately. @@ -196,23 +137,6 @@ config.SaveConfig(); **Primary**: LM Studio (free, learn AI concepts) **Alternative**: Azure Foundry (free tier) -### For **Open Source Projects**: -**Primary**: LM Studio (no API costs for contributors) -**Secondary**: GitHub Models (integration benefits) - -## ๐ŸŽฏ Bottom Line - -| Priority | Factor | Best Choice | -|----------|--------|-------------| -| 1 | **Privacy** | ๐Ÿ† LM Studio | -| 2 | **Cost** | ๐Ÿ† LM Studio | -| 3 | **Quality** | ๐Ÿ† Azure OpenAI | -| 4 | **Speed** | ๐Ÿ† Azure OpenAI | -| 5 | **Offline** | ๐Ÿ† LM Studio | -| 6 | **Ease of Use** | ๐Ÿ† Azure OpenAI | -| 7 | **Unlimited Use** | ๐Ÿ† LM Studio | -| 8 | **Latest Models** | ๐Ÿ† Azure OpenAI | - ## ๐Ÿ’ก Pro Tip: Hybrid Approach **Best of Both Worlds:** @@ -235,7 +159,7 @@ config.SaveConfig(); ## ๐Ÿš€ Getting Started ### LM Studio Setup: **5 minutes** -See: [LMSTUDIO_QUICKSTART.md](LMSTUDIO_QUICKSTART.md) +See: [LM Studio Quickstart](LM-Studio-Quickstart.md) ### Azure OpenAI Setup: **10 minutes** Already configured in FlowVision @@ -244,4 +168,4 @@ Already configured in FlowVision --- -**Choose what works best for YOUR needs!** ๐ŸŽฏ +**Choose what works best for YOUR needs!** ๐ŸŽฏ \ No newline at end of file diff --git a/LMSTUDIO_QUICKSTART.md b/docs/guides/LM-Studio-Quickstart.md similarity index 96% rename from LMSTUDIO_QUICKSTART.md rename to docs/guides/LM-Studio-Quickstart.md index 1911682..1e22908 100644 --- a/LMSTUDIO_QUICKSTART.md +++ b/docs/guides/LM-Studio-Quickstart.md @@ -173,11 +173,11 @@ You're now running AI locally! - Try different models - Experiment with settings - Test tool calling features -- Read full documentation: LMSTUDIO_INTEGRATION.md +- Read full documentation: [LM Studio Integration Guide](LM-Studio-Integration.md) ## ๐Ÿ†˜ Need Help? -1. Check full guide: `LMSTUDIO_INTEGRATION.md` +1. Check full guide: [LM Studio Integration Guide](LM-Studio-Integration.md) 2. LM Studio docs: https://lmstudio.ai/docs 3. LM Studio Discord: https://discord.gg/lmstudio 4. GitHub Issues: (your repo) @@ -186,4 +186,4 @@ You're now running AI locally! **Enjoy your private, local AI!** ๐ŸŽŠ -No more API costs! No more cloud dependency! Complete control! ๐Ÿš€ +No more API costs! No more cloud dependency! Complete control! ๐Ÿš€ \ No newline at end of file diff --git a/OMNIPARSER_QUICK_START.md b/docs/guides/OmniParser-Setup.md similarity index 99% rename from OMNIPARSER_QUICK_START.md rename to docs/guides/OmniParser-Setup.md index 3e9ac41..6830493 100644 --- a/OMNIPARSER_QUICK_START.md +++ b/docs/guides/OmniParser-Setup.md @@ -439,6 +439,5 @@ All fast after first one! --- -**Full Documentation:** See `LOCAL_OMNIPARSER_INTEGRATION.md` **Status:** โœ… Ready to use -**Configuration:** โŒ None needed (optional customization available) +**Configuration:** โŒ None needed (optional customization available) \ No newline at end of file diff --git a/download_and_convert_all.py b/download_and_convert_all.py index 377b455..e69de29 100644 --- a/download_and_convert_all.py +++ b/download_and_convert_all.py @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -""" -OmniParser Complete Model Downloader and Converter -Downloads both icon_detect (YOLO) and icon_caption_florence models -Converts them to ONNX format for .NET use -""" - -import sys -import shutil -from pathlib import Path - -def check_dependencies(): - """Check if required packages are installed""" - print("=" * 60) - print(" OmniParser Complete Setup") - print("=" * 60) - print() - - missing = [] - - try: - import torch - except ImportError: - missing.append("torch") - - try: - from ultralytics import YOLO - except ImportError: - missing.append("ultralytics") - - try: - from transformers import AutoProcessor, AutoModelForCausalLM - except ImportError: - missing.append("transformers") - - if missing: - print("[โœ—] Missing dependencies:", ", ".join(missing)) - print() - print("Install with:") - print(f" pip install {' '.join(missing)}") - sys.exit(1) - - print("[โœ“] All dependencies installed") - print() - -def download_models(): - """Download both models from HuggingFace""" - print("โ”" * 60) - print("Step 1: Downloading Models from HuggingFace") - print("โ”" * 60) - print() - - try: - from huggingface_hub import hf_hub_download - except ImportError: - print("[โœ—] huggingface_hub not installed") - print(" Install with: pip install huggingface-hub") - sys.exit(1) - - weights_dir = Path("weights") - - # Download icon_detect (YOLO) - print("[1/2] Downloading icon_detect (YOLO)...") - detect_files = ["model.pt", "model.yaml", "train_args.yaml"] - detect_dir = weights_dir / "icon_detect" - detect_dir.mkdir(parents=True, exist_ok=True) - - for file in detect_files: - try: - print(f" Downloading {file}...") - hf_hub_download( - repo_id="microsoft/OmniParser-v2.0", - filename=f"icon_detect/{file}", - local_dir=str(weights_dir) - ) - except Exception as e: - print(f" [!] Could not download {file}: {e}") - - print("[โœ“] icon_detect downloaded") - print() - - # Download icon_caption_florence - print("[2/2] Downloading icon_caption_florence (Florence-2)...") - caption_files = ["config.json", "generation_config.json", "model.safetensors", - "preprocessor_config.json", "tokenizer.json", "tokenizer_config.json"] - caption_dir = weights_dir / "icon_caption_florence" - caption_dir.mkdir(parents=True, exist_ok=True) - - for file in caption_files: - try: - print(f" Downloading {file}...") - hf_hub_download( - repo_id="microsoft/OmniParser-v2.0", - filename=f"icon_caption/{file}", - local_dir=str(weights_dir / "icon_caption_florence") - ) - except Exception as e: - print(f" [!] Could not download {file}: {e}") - - print("[โœ“] icon_caption_florence downloaded") - print() - -def convert_detection_model(): - """Convert YOLO detection model to ONNX""" - print("โ”" * 60) - print("Step 2: Converting Detection Model (YOLO โ†’ ONNX)") - print("โ”" * 60) - print() - - from ultralytics import YOLO - - pt_path = Path("weights/icon_detect/model.pt") - if not pt_path.exists(): - print(f"[โœ—] Model not found at {pt_path}") - return False - - print(f"[+] Loading YOLO model from: {pt_path}") - model = YOLO(str(pt_path)) - - print("[+] Exporting to ONNX format...") - print(" Settings: opset=12, simplify=True, dynamic=False") - - try: - model.export( - format='onnx', - simplify=True, - opset=12, - dynamic=False, - imgsz=640 - ) - - # Move to FlowVision models directory - generated_onnx = pt_path.with_suffix('.onnx') - output_dir = Path("FlowVision/models") - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / "icon_detect.onnx" - - shutil.move(str(generated_onnx), str(output_path)) - - import os - file_size = os.path.getsize(output_path) / (1024 * 1024) - - print() - print("[โœ“] Detection model converted successfully!") - print(f" Output: {output_path}") - print(f" Size: {file_size:.2f} MB") - print() - return True - - except Exception as e: - print(f"[โœ—] Conversion failed: {e}") - return False - -def convert_caption_model(): - """Convert Florence caption model to ONNX""" - print("โ”" * 60) - print("Step 3: Converting Caption Model (Florence-2 โ†’ ONNX)") - print("โ”" * 60) - print() - - caption_dir = Path("weights/icon_caption_florence") - if not caption_dir.exists(): - print(f"[โœ—] Caption model not found at {caption_dir}") - return False - - print("[!] Note: Florence-2 ONNX conversion is complex") - print(" For KISS approach, we'll keep the model in PyTorch format") - print(" and load it via Python if needed, or skip captions entirely.") - print() - - # Check if we can load the model - try: - from transformers import AutoProcessor, AutoModelForCausalLM - import torch - - print("[+] Loading Florence-2 model...") - model = AutoModelForCausalLM.from_pretrained( - str(caption_dir), - trust_remote_code=True, - torch_dtype=torch.float32 - ) - processor = AutoProcessor.from_pretrained( - str(caption_dir), - trust_remote_code=True - ) - - print("[โœ“] Florence-2 model loaded successfully") - print(f" Location: {caption_dir}") - print() - print("[!] For .NET integration, we have options:") - print(" 1. Use Python bridge for captions (hybrid approach)") - print(" 2. Skip captions and use detection-only (KISS)") - print(" 3. Use ONNX Runtime with manual conversion (complex)") - print() - print(" Recommendation: Option 2 (detection-only) for simplicity") - print() - - return True - - except Exception as e: - print(f"[โœ—] Could not load Florence model: {e}") - print() - return False - -def main(): - """Main setup flow""" - check_dependencies() - - # Step 1: Download - try: - download_models() - except Exception as e: - print(f"[โœ—] Download failed: {e}") - print(" You can try manual download:") - print(" huggingface-cli download microsoft/OmniParser-v2.0 --local-dir weights") - sys.exit(1) - - # Step 2: Convert detection - if not convert_detection_model(): - print("[โœ—] Detection model conversion failed") - sys.exit(1) - - # Step 3: Handle caption model - caption_success = convert_caption_model() - - # Summary - print("=" * 60) - print(" Setup Complete!") - print("=" * 60) - print() - print("โœ“ Detection Model: Ready (ONNX)") - print(" โ””โ”€ FlowVision/models/icon_detect.onnx") - print() - - if caption_success: - print("โœ“ Caption Model: Available (PyTorch)") - print(" โ””โ”€ weights/icon_caption_florence/") - print() - print(" [!] Caption model is optional for KISS implementation") - else: - print("โ—‹ Caption Model: Not configured") - print(" โ””โ”€ Detection-only mode (recommended for simplicity)") - - print() - print("Next steps:") - print(" 1. Build FlowVision project in Visual Studio") - print(" 2. Set icon_detect.onnx as Embedded Resource") - print(" 3. Test screen capture functionality") - print() - print("The detection model alone provides bounding boxes,") - print("which is sufficient for most AI agent use cases!") - print() - -if __name__ == "__main__": - main() diff --git a/download_omniparser_model.ps1 b/download_omniparser_model.ps1 index b466b9f..e69de29 100644 --- a/download_omniparser_model.ps1 +++ b/download_omniparser_model.ps1 @@ -1,113 +0,0 @@ -# OmniParser Model Downloader -# Downloads icon_detect model from HuggingFace and sets up for FlowVision - -param( - [string]$OutputPath = ".\FlowVision\models", - [switch]$Embedded = $false -) - -$ErrorActionPreference = "Stop" - -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host " OmniParser Model Downloader - KISS Edition" -ForegroundColor Cyan -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host "" - -# Model URL from HuggingFace (PyTorch format - we'll need to convert to ONNX) -$modelUrl = "https://huggingface.co/microsoft/OmniParser-v2.0/resolve/main/icon_detect/model.pt" -$modelName = "model.pt" -$modelNameOnnx = "icon_detect.onnx" - -# Create output directory -if (-not (Test-Path $OutputPath)) { - Write-Host "[+] Creating directory: $OutputPath" -ForegroundColor Green - New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null -} - -$outputFile = Join-Path $OutputPath $modelName - -# Check if model already exists -if (Test-Path $outputFile) { - $response = Read-Host "Model already exists at $outputFile. Overwrite? (y/N)" - if ($response -ne 'y' -and $response -ne 'Y') { - Write-Host "[!] Download cancelled." -ForegroundColor Yellow - exit 0 - } -} - -# Download model -Write-Host "" -Write-Host "[+] Downloading OmniParser model from HuggingFace..." -ForegroundColor Green -Write-Host " URL: $modelUrl" -ForegroundColor Gray -Write-Host " Destination: $outputFile" -ForegroundColor Gray -Write-Host "" -Write-Host " This may take a few minutes (~50MB)..." -ForegroundColor Yellow -Write-Host "" - -try { - # Use WebClient for progress display - $webClient = New-Object System.Net.WebClient - - # Register progress event - Register-ObjectEvent -InputObject $webClient -EventName DownloadProgressChanged -SourceIdentifier WebClient.DownloadProgressChanged -Action { - $percent = $EventArgs.ProgressPercentage - Write-Progress -Activity "Downloading model..." -Status "$percent% Complete" -PercentComplete $percent - } | Out-Null - - # Download - $webClient.DownloadFile($modelUrl, $outputFile) - - # Unregister event - Unregister-Event -SourceIdentifier WebClient.DownloadProgressChanged - Write-Progress -Activity "Downloading model..." -Completed - - $webClient.Dispose() - - Write-Host "[โœ“] Download complete!" -ForegroundColor Green - Write-Host "" -} -catch { - Write-Host "[โœ—] Download failed: $($_.Exception.Message)" -ForegroundColor Red - exit 1 -} - -# Verify file -if (Test-Path $outputFile) { - $fileSize = (Get-Item $outputFile).Length / 1MB - Write-Host "[โœ“] Model file verified" -ForegroundColor Green - Write-Host " Size: $([math]::Round($fileSize, 2)) MB" -ForegroundColor Gray - Write-Host " Path: $outputFile" -ForegroundColor Gray -} -else { - Write-Host "[โœ—] Model file not found after download!" -ForegroundColor Red - exit 1 -} - -Write-Host "" -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host " Setup Complete!" -ForegroundColor Cyan -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host "" - -if ($Embedded) { - Write-Host "Next steps (Embedded Resource Mode):" -ForegroundColor Yellow - Write-Host " 1. Open FlowVision project in Visual Studio" - Write-Host " 2. Right-click '$modelName' in models folder" - Write-Host " 3. Properties โ†’ Build Action โ†’ Embedded Resource" - Write-Host " 4. Rebuild project" - Write-Host "" - Write-Host "The model will be compiled into FlowVision.exe" -ForegroundColor Green -} -else { - Write-Host "Next steps (External File Mode):" -ForegroundColor Yellow - Write-Host " 1. Build FlowVision project" - Write-Host " 2. Copy models folder to output directory:" - Write-Host " .\FlowVision\bin\Debug\models\" - Write-Host " .\FlowVision\bin\Release\models\" - Write-Host "" - Write-Host "Or run with -Embedded flag to set up embedded mode" -ForegroundColor Green -} - -Write-Host "" -Write-Host "Model is ready to use!" -ForegroundColor Cyan -Write-Host "" diff --git a/setup_omniparser_complete.ps1 b/setup_omniparser_complete.ps1 index 7d596b8..e69de29 100644 --- a/setup_omniparser_complete.ps1 +++ b/setup_omniparser_complete.ps1 @@ -1,235 +0,0 @@ -# OmniParser Model Setup - Complete Solution -# Downloads PyTorch model and converts to ONNX for .NET use - -param( - [string]$OutputPath = ".\FlowVision\models", - [switch]$SkipConversion = $false -) - -$ErrorActionPreference = "Stop" - -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host " OmniParser Complete Setup - KISS Edition" -ForegroundColor Cyan -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host "" - -# Check if we have a pre-converted ONNX model available -$onnxModel = Join-Path $OutputPath "icon_detect.onnx" -if (Test-Path $onnxModel) { - $response = Read-Host "Found existing ONNX model. Use it? (Y/n)" - if ($response -eq '' -or $response -eq 'y' -or $response -eq 'Y') { - Write-Host "[โœ“] Using existing ONNX model" -ForegroundColor Green - $fileSize = (Get-Item $onnxModel).Length / 1MB - Write-Host " Size: $([math]::Round($fileSize, 2)) MB" -ForegroundColor Gray - Write-Host " Path: $onnxModel" -ForegroundColor Gray - Write-Host "" - Write-Host "[โœ“] Setup complete! Model is ready to use." -ForegroundColor Cyan - exit 0 - } -} - -Write-Host "" -Write-Host "๐Ÿ“ฆ IMPORTANT: Model Format Information" -ForegroundColor Yellow -Write-Host "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -ForegroundColor Yellow -Write-Host "" -Write-Host "The OmniParser model is available in PyTorch format (.pt)" -ForegroundColor White -Write-Host "For .NET/ONNX Runtime, we need to convert it to ONNX format." -ForegroundColor White -Write-Host "" -Write-Host "Options:" -ForegroundColor Cyan -Write-Host " 1. Download pre-converted ONNX model (recommended)" -ForegroundColor Green -Write-Host " 2. Download PyTorch and convert manually" -ForegroundColor Yellow -Write-Host "" - -# Option 1: Try to download pre-converted ONNX -Write-Host "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -ForegroundColor Cyan -Write-Host "Option 1: Checking for pre-converted ONNX model..." -ForegroundColor Cyan -Write-Host "" - -# Create output directory -if (-not (Test-Path $OutputPath)) { - New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null -} - -# Try common ONNX model locations -$onnxUrls = @( - "https://huggingface.co/microsoft/OmniParser/resolve/main/icon_detect/model.onnx", - "https://huggingface.co/microsoft/OmniParser-v2.0/resolve/main/icon_detect/model.onnx" -) - -$onnxDownloaded = $false -foreach ($url in $onnxUrls) { - Write-Host "[*] Trying: $url" -ForegroundColor Gray - try { - $webClient = New-Object System.Net.WebClient - $webClient.DownloadFile($url, $onnxModel) - $webClient.Dispose() - - if (Test-Path $onnxModel) { - Write-Host "[โœ“] Successfully downloaded ONNX model!" -ForegroundColor Green - $onnxDownloaded = $true - break - } - } - catch { - Write-Host "[โœ—] Not available at this location" -ForegroundColor DarkGray - } -} - -if ($onnxDownloaded) { - $fileSize = (Get-Item $onnxModel).Length / 1MB - Write-Host "" - Write-Host "[โœ“] Model ready!" -ForegroundColor Green - Write-Host " Format: ONNX" -ForegroundColor Gray - Write-Host " Size: $([math]::Round($fileSize, 2)) MB" -ForegroundColor Gray - Write-Host " Path: $onnxModel" -ForegroundColor Gray - Write-Host "" - Write-Host "===============================================" -ForegroundColor Cyan - Write-Host " Setup Complete!" -ForegroundColor Cyan - Write-Host "===============================================" -ForegroundColor Cyan - exit 0 -} - -# Option 2: Manual conversion required -Write-Host "" -Write-Host "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -ForegroundColor Yellow -Write-Host "Option 2: Manual Conversion Required" -ForegroundColor Yellow -Write-Host "" -Write-Host "No pre-converted ONNX model found." -ForegroundColor Yellow -Write-Host "" -Write-Host "To convert the PyTorch model to ONNX:" -ForegroundColor White -Write-Host "" -Write-Host "1. Install Python dependencies:" -ForegroundColor Cyan -Write-Host " pip install torch onnx ultralytics" -ForegroundColor Gray -Write-Host "" -Write-Host "2. Download the PyTorch model:" -ForegroundColor Cyan -Write-Host " huggingface-cli download microsoft/OmniParser-v2.0 icon_detect/model.pt --local-dir weights" -ForegroundColor Gray -Write-Host "" -Write-Host "3. Convert to ONNX using Python:" -ForegroundColor Cyan -Write-Host "" -Write-Host " import torch" -ForegroundColor Gray -Write-Host " from ultralytics import YOLO" -ForegroundColor Gray -Write-Host "" -Write-Host " # Load PyTorch model" -ForegroundColor Gray -Write-Host " model = YOLO('weights/icon_detect/model.pt')" -ForegroundColor Gray -Write-Host " # Export to ONNX" -ForegroundColor Gray -Write-Host " model.export(format='onnx', simplify=True)" -ForegroundColor Gray -Write-Host "" -Write-Host "4. Copy the resulting icon_detect.onnx to:" -ForegroundColor Cyan -Write-Host " $OutputPath\icon_detect.onnx" -ForegroundColor Gray -Write-Host "" -Write-Host "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -ForegroundColor Yellow -Write-Host "" - -# Offer to create a conversion script -$createScript = Read-Host "Would you like me to create a Python conversion script? (Y/n)" -if ($createScript -eq '' -or $createScript -eq 'y' -or $createScript -eq 'Y') { - $scriptContent = @" -#!/usr/bin/env python3 -""" -OmniParser PyTorch to ONNX Converter -Converts the OmniParser YOLO model from PyTorch (.pt) to ONNX format -for use with .NET ONNX Runtime -""" - -import sys -from pathlib import Path - -try: - from ultralytics import YOLO -except ImportError: - print("[โœ—] Error: ultralytics not installed") - print(" Install with: pip install ultralytics") - sys.exit(1) - -def convert_to_onnx(pt_model_path, output_path): - """Convert PyTorch YOLO model to ONNX""" - print("=" * 60) - print(" OmniParser PyTorch โ†’ ONNX Converter") - print("=" * 60) - print() - - pt_path = Path(pt_model_path) - if not pt_path.exists(): - print(f"[โœ—] Error: Model not found at {pt_path}") - print(f" Download with:") - print(f" huggingface-cli download microsoft/OmniParser-v2.0 \\") - print(f" icon_detect/model.pt --local-dir weights") - sys.exit(1) - - print(f"[+] Loading PyTorch model from: {pt_path}") - try: - model = YOLO(str(pt_path)) - except Exception as e: - print(f"[โœ—] Failed to load model: {e}") - sys.exit(1) - - print("[+] Model loaded successfully") - print(f"[+] Converting to ONNX format...") - print(f" Output: {output_path}") - print() - - try: - # Export with simplification for better performance - model.export( - format='onnx', - simplify=True, - opset=12, # Compatible with most ONNX runtimes - dynamic=False, # Static shapes for better performance - imgsz=640 # Fixed input size - ) - - # The export creates a file next to the input with .onnx extension - generated_onnx = pt_path.with_suffix('.onnx') - - if generated_onnx.exists(): - # Move to desired location - import shutil - shutil.move(str(generated_onnx), output_path) - - import os - file_size = os.path.getsize(output_path) / (1024 * 1024) - - print() - print("[โœ“] Conversion successful!") - print(f" ONNX model: {output_path}") - print(f" Size: {file_size:.2f} MB") - print() - print("You can now use this model with FlowVision!") - - else: - print("[โœ—] ONNX file not found after export") - sys.exit(1) - - except Exception as e: - print(f"[โœ—] Conversion failed: {e}") - sys.exit(1) - -if __name__ == "__main__": - pt_model = "weights/icon_detect/model.pt" - onnx_output = "FlowVision/models/icon_detect.onnx" - - if len(sys.argv) > 1: - pt_model = sys.argv[1] - if len(sys.argv) > 2: - onnx_output = sys.argv[2] - - convert_to_onnx(pt_model, onnx_output) -"@ - - $scriptPath = "convert_omniparser_to_onnx.py" - $scriptContent | Out-File -FilePath $scriptPath -Encoding UTF8 - Write-Host "[โœ“] Created conversion script: $scriptPath" -ForegroundColor Green - Write-Host "" - Write-Host "Run it with: python $scriptPath" -ForegroundColor Cyan -} - -Write-Host "" -Write-Host "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -ForegroundColor Cyan -Write-Host "Alternative: Use Pre-converted ONNX" -ForegroundColor Cyan -Write-Host "" -Write-Host "If you have access to a pre-converted ONNX model," -ForegroundColor White -Write-Host "simply place it at:" -ForegroundColor White -Write-Host " $OutputPath\icon_detect.onnx" -ForegroundColor Cyan -Write-Host "" -Write-Host "The SimpleOmniParser will automatically detect and use it!" -ForegroundColor Green -Write-Host "" diff --git a/setup_omniparser_full.ps1 b/setup_omniparser_full.ps1 index 3577fed..e69de29 100644 --- a/setup_omniparser_full.ps1 +++ b/setup_omniparser_full.ps1 @@ -1,104 +0,0 @@ -# Complete OmniParser Setup - Both Models -# Downloads and converts BOTH detection and captioning models - -param( - [switch]$CaptionModel = $false -) - -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host " OmniParser Complete Setup" -ForegroundColor Cyan -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host "" - -# Check detection model -$detectionModel = ".\FlowVision\models\icon_detect.onnx" -if (Test-Path $detectionModel) { - Write-Host "[โœ“] Detection model found (YOLO)" -ForegroundColor Green - $size = (Get-Item $detectionModel).Length / 1MB - Write-Host " Size: $([math]::Round($size, 2)) MB" -ForegroundColor Gray -} else { - Write-Host "[โœ—] Detection model NOT found!" -ForegroundColor Red - Write-Host " Run: python convert_omniparser_to_onnx.py" -ForegroundColor Yellow - exit 1 -} - -Write-Host "" - -if ($CaptionModel) { - Write-Host "๐Ÿ“ Caption Model Setup" -ForegroundColor Cyan - Write-Host "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -ForegroundColor Cyan - Write-Host "" - Write-Host "โš ๏ธ WARNING: Caption models are LARGE and SLOW!" -ForegroundColor Yellow - Write-Host "" - Write-Host "Options:" -ForegroundColor White - Write-Host " 1. BLIP-2: ~7GB, slower, more detailed captions" -ForegroundColor Gray - Write-Host " 2. Florence: ~2GB, faster, good captions" -ForegroundColor Gray - Write-Host "" - Write-Host "For KISS approach, caption model is OPTIONAL." -ForegroundColor Green - Write-Host "The AI agent can work fine with just bounding boxes!" -ForegroundColor Green - Write-Host "" - - $choice = Read-Host "Download caption model? (1=BLIP-2, 2=Florence, N=Skip)" - - if ($choice -eq "1" -or $choice -eq "2") { - $modelName = if ($choice -eq "1") { "icon_caption_blip2" } else { "icon_caption_florence" } - - Write-Host "" - Write-Host "[+] Downloading $modelName from HuggingFace..." -ForegroundColor Green - Write-Host "" - - # Download using huggingface-cli - $cmd = "huggingface-cli download microsoft/OmniParser-v2.0 $modelName --local-dir weights" - Write-Host " Running: $cmd" -ForegroundColor Gray - Invoke-Expression $cmd - - Write-Host "" - Write-Host "[!] Note: Caption models are PyTorch format" -ForegroundColor Yellow - Write-Host " Converting to ONNX for .NET is complex and may not be worth it." -ForegroundColor Yellow - Write-Host " Consider using detection only for best KISS implementation!" -ForegroundColor Green - } -} else { - Write-Host "๐Ÿ“ Caption Model: SKIPPED (Recommended)" -ForegroundColor Green - Write-Host "" - Write-Host "You're using detection-only mode:" -ForegroundColor White - Write-Host " โœ“ Faster inference (~200ms)" -ForegroundColor Green - Write-Host " โœ“ Less memory (~150MB)" -ForegroundColor Green - Write-Host " โœ“ Simpler codebase" -ForegroundColor Green - Write-Host " โœ“ AI agent still works great!" -ForegroundColor Green - Write-Host "" - Write-Host "To enable captions later, run:" -ForegroundColor Cyan - Write-Host " .\setup_omniparser_full.ps1 -CaptionModel" -ForegroundColor Gray -} - -Write-Host "" -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host " Current Configuration" -ForegroundColor Cyan -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host "" -Write-Host "Detection Model: " -NoNewline -Write-Host "ENABLED โœ“" -ForegroundColor Green -Write-Host " - Detects UI element bounding boxes" -ForegroundColor Gray -Write-Host " - ~200ms per screenshot" -ForegroundColor Gray -Write-Host " - ~150MB memory" -ForegroundColor Gray -Write-Host "" - -Write-Host "Caption Model: " -NoNewline -if ($CaptionModel) { - Write-Host "ENABLED" -ForegroundColor Yellow - Write-Host " - Describes each element's purpose" -ForegroundColor Gray - Write-Host " - +500ms per screenshot" -ForegroundColor Gray - Write-Host " - +2GB memory" -ForegroundColor Gray -} else { - Write-Host "DISABLED (Recommended)" -ForegroundColor Green - Write-Host " - Keeps it simple and fast" -ForegroundColor Gray - Write-Host " - AI uses coordinates + OCR instead" -ForegroundColor Gray -} - -Write-Host "" -Write-Host "[โœ“] Setup complete!" -ForegroundColor Cyan -Write-Host "" -Write-Host "Next steps:" -ForegroundColor Yellow -Write-Host " 1. Build FlowVision project" -ForegroundColor White -Write-Host " 2. Set icon_detect.onnx as Embedded Resource" -ForegroundColor White -Write-Host " 3. Run and test screen capture" -ForegroundColor White -Write-Host "" diff --git a/sk-migration-guide.md b/sk-migration-guide.md deleted file mode 100644 index 971b464..0000000 --- a/sk-migration-guide.md +++ /dev/null @@ -1,393 +0,0 @@ -# Semantic Kernel to Agent Framework Migration Guide - -## What's Changed? -- **Namespace Updates**: From `Microsoft.SemanticKernel.Agents` to `Microsoft.Agents.AI` -- **Agent Creation**: Single fluent API calls vs multi-step builder patterns -- **Thread Management**: Built-in thread management vs manual thread creation -- **Tool Registration**: Direct function registration vs plugin wrapper systems -- **Dependency Injection**: Simplified service registration patterns -- **Invocation Patterns**: Streamlined options and result handling - -## Benefits of Migration -- **Simplified API**: Reduced complexity and boilerplate code -- **Better Performance**: Optimized object creation and memory usage -- **Unified Interface**: Consistent patterns across different AI providers -- **Enhanced Developer Experience**: More intuitive and discoverable APIs - -## Key Changes - -### 1. Namespace Updates - -#### Semantic Kernel - -```csharp -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Agents; -``` - -#### Agent Framework - -Agent Framework namespaces are under `Microsoft.Agents.AI`. -Agent Framework uses the core AI message and content types from `Microsoft.Extensions.AI` for communication between components. - -```csharp -using Microsoft.Extensions.AI; -using Microsoft.Agents.AI; -``` - -### 2. Agent Creation Simplification - -#### Semantic Kernel - -Every agent in Semantic Kernel depends on a `Kernel` instance and will have -an empty `Kernel` if not provided. - -```csharp - Kernel kernel = Kernel - .AddOpenAIChatClient(modelId, apiKey) - .Build(); - - ChatCompletionAgent agent = new() { Instructions = ParrotInstructions, Kernel = kernel }; -``` - -Azure AI Foundry requires an agent resource to be created in the cloud before creating a local agent class that uses it. - -```csharp -PersistentAgentsClient azureAgentClient = AzureAIAgent.CreateAgentsClient(azureEndpoint, new AzureCliCredential()); - -PersistentAgent definition = await azureAgentClient.Administration.CreateAgentAsync( - deploymentName, - instructions: ParrotInstructions); - -AzureAIAgent agent = new(definition, azureAgentClient); - ``` - -#### Agent Framework - -Agent creation in Agent Framework is made simpler with extensions provided by all main providers. - -```csharp -AIAgent openAIAgent = chatClient.CreateAIAgent(instructions: ParrotInstructions); -AIAgent azureFoundryAgent = await persistentAgentsClient.CreateAIAgentAsync(instructions: ParrotInstructions); -AIAgent openAIAssistantAgent = await assistantClient.CreateAIAgentAsync(instructions: ParrotInstructions); -``` - -Additionally for hosted agent providers you can also use the `GetAIAgent` to retrieve an agent from an existing hosted agent. - -```csharp -AIAgent azureFoundryAgent = await persistentAgentsClient.GetAIAgentAsync(agentId); -``` - -### 3. Agent Thread Creation - -#### Semantic Kernel - -The caller has to know the thread type and create it manually. - -```csharp -// Create a thread for the agent conversation. -AgentThread thread = new OpenAIAssistantAgentThread(this.AssistantClient); -AgentThread thread = new AzureAIAgentThread(this.Client); -AgentThread thread = new OpenAIResponseAgentThread(this.Client); -``` - -#### Agent Framework - -The agent is responsible for creating the thread. - -```csharp -// New -AgentThread thread = agent.GetNewThread(); -``` - -### 4. Hosted Agent Thread Cleanup - -This case applies exclusively to a few AI providers that still provide hosted threads. - -#### Semantic Kernel - -Threads have a `self` deletion method - -i.e: OpenAI Assistants Provider -```csharp -await thread.DeleteAsync(); -``` - -#### Agent Framework - -> [!NOTE] -> OpenAI Responses introduced a new conversation model that simplifies how conversations are handled. This simplifies hosted thread management compared to the now deprecated OpenAI Assistants model. For more information see the [OpenAI Assistants migration guide](https://platform.openai.com/docs/assistants/migration). - -Agent Framework doesn't have a thread deletion API in the `AgentThread` type as not all providers support hosted threads or thread deletion and this will become more common as more providers shift to responses based architectures. - -If you require thread deletion and the provider allows this, the caller **should** keep track of the created threads and delete them later when necessary via the provider's sdk. - -i.e: OpenAI Assistants Provider -```csharp -await assistantClient.DeleteThreadAsync(thread.ConversationId); -``` - -### 5. Tool Registration - -#### Semantic Kernel - -In semantic kernel to expose a function as a tool you must: - -1. Decorate the function with a `[KernelFunction]` attribute. -2. Have a `Plugin` class or use the `KernelPluginFactory` to wrap the function. -3. Have a `Kernel` to add your plugin to. -4. Pass the `Kernel` to the agent. - -```csharp -KernelFunction function = KernelFunctionFactory.CreateFromMethod(GetWeather); -KernelPlugin plugin = KernelPluginFactory.CreateFromFunctions("KernelPluginName", [function]); -Kernel kernel = ... // Create kernel -kernel.Plugins.Add(plugin); - -ChatCompletionAgent agent = new() { Kernel = kernel, ... }; -``` - -#### Agent Framework - -In agent framework in a single call you can register tools directly in the agent creation process. - -```csharp -AIAgent agent = chatClient.CreateAIAgent(tools: [AIFunctionFactory.Create(GetWeather)]); -``` - -### 6. Agent Non-Streaming Invocation - -Key differences can be seen in the method names from `Invoke` to `Run`, return types and parameters `AgentRunOptions`. - -#### Semantic Kernel - -The Non-Streaming uses a streaming pattern `IAsyncEnumerable>` for returning multiple agent messages. - -```csharp -await foreach (AgentResponseItem result in agent.InvokeAsync(userInput, thread, agentOptions)) -{ - Console.WriteLine(result.Message); -} -``` - -#### Agent Framework - -The Non-Streaming returns a single `AgentRunResponse` with the agent response that can contain multiple messages. -The text result of the run is available in `AgentRunResponse.Text` or `AgentRunResponse.ToString()`. -All messages created as part of the response is returned in the `AgentRunResponse.Messages` list. -This may include tool call messages, function results, reasoning updates and final results. - -```csharp -AgentRunResponse agentResponse = await agent.RunAsync(userInput, thread); -``` - -### 7. Agent Streaming Invocation - -Key differences in the method names from `Invoke` to `Run`, return types and parameters `AgentRunOptions`. - -#### Semantic Kernel - -```csharp -await foreach (StreamingChatMessageContent update in agent.InvokeStreamingAsync(userInput, thread)) -{ - Console.Write(update); -} -``` - -#### Agent Framework - -Similar streaming API pattern with the key difference being that it returns `AgentRunResponseUpdate` objects including more agent related information per update. - -All updates produced by any service underlying the AIAgent is returned. The textual result of the agent is available by concatenating the `AgentRunResponse.Text` values. - -```csharp -await foreach (AgentRunResponseUpdate update in agent.RunStreamingAsync(userInput, thread)) -{ - Console.Write(update); // Update is ToString() friendly -} -``` - -### 8. Tool Function Signatures - -**Problem**: SK plugin methods need `[KernelFunction]` attributes - -```csharp -public class MenuPlugin -{ - [KernelFunction] // Required for SK - public static MenuItem[] GetMenu() => ...; -} -``` - -**Solution**: AF can use methods directly without attributes - -```csharp -public class MenuTools -{ - [Description("Get menu items")] // Optional description - public static MenuItem[] GetMenu() => ...; -} -``` - -### 9. Options Configuration - -**Problem**: Complex options setup in SK - -```csharp -OpenAIPromptExecutionSettings settings = new() { MaxTokens = 1000 }; -AgentInvokeOptions options = new() { KernelArguments = new(settings) }; -``` - -**Solution**: Simplified options in AF - -```csharp -ChatClientAgentRunOptions options = new(new() { MaxOutputTokens = 1000 }); -``` - -> [!IMPORTANT] -> This example shows passing implementation specific options to a `ChatClientAgent`. Not all `AIAgents` support `ChatClientAgentRunOptions`. -> `ChatClientAgent` is provided to build agents based on underlying inference services, and therefore supports inference options like `MaxOutputTokens`. - -### 10. Dependency Injection - -#### Semantic Kernel - -A `Kernel` registration is required in the service container to be able to create an agent -as every agent abstractions needs to be initialized with a `Kernel` property. - -Semantic Kernel uses the `Agent` type as the base abstraction class for agents. - -```csharp -services.AddKernel().AddProvider(...); -serviceContainer.AddKeyedSingleton( - TutorName, - (sp, key) => - new ChatCompletionAgent() - { - // Passing the kernel is required - Kernel = sp.GetRequiredService(), - }); -``` - -### 11. **Agent Type Consolidation** - -#### Semantic Kernel - -Semantic kernel provides specific agent classes for various services, e.g. - -- `ChatCompletionAgent` for use with chat-completion-based inference services. -- `OpenAIAssistantAgent` for use with the OpenAI Assistants service. -- `AzureAIAgent` for use with the Azure AI Foundry Agents service. - -#### Agent Framework - -The agent framework supports all the abovementioned services via a single agent type, `ChatClientAgent`. - -`ChatClientAgent` can be used to build agents using any underlying service that provides an SDK implementing the `Microsoft.Extensions.AI.IChatClient` interface. - -#### Agent Framework - -The Agent framework provides the `AIAgent` type as the base abstraction class. - -```csharp -services.AddKeyedSingleton(() => client.CreateAIAgent(...)); -``` - -## Migration Samples - -This folder contains **separate console application projects** demonstrating how to transition from **Semantic Kernel (SK)** to the new **Agent Framework (AF)**. - -Each project shows side-by-side comparisons of equivalent functionality in both frameworks and can be run independently. - -Each sample code contains the following: -1. **SK Agent** (Semantic Kernel before) -2. **AF Agent** (Agent Framework after) - -### Running the samples from Visual Studio - -Open the solution in Visual Studio and set the desired sample project as the startup project. Then, run the project using the built-in debugger or by pressing `F5`. - -You will be prompted for any required environment variables if they are not already set. - -### Prerequisites - -Before you begin, ensure you have the following: - -- [.NET 8.0 SDK or later](https://dotnet.microsoft.com/download) -- For Azure AI Foundry samples: Azure OpenAI service endpoint and deployment configured -- For OpenAI samples: OpenAI API key -- For OpenAI Assistants samples: OpenAI API key with Assistant API access - -### Environment Variables - -Set the appropriate environment variables based on the sample type you want to run: - -**For Azure AI Foundry projects:** -```powershell -$env:AZURE_FOUNDRY_PROJECT_ENDPOINT = "https://-resource.services.ai.azure.com/api/projects/" -``` - -**For OpenAI and OpenAI Assistants projects:** -```powershell -$env:OPENAI_API_KEY = "sk-..." -``` - -**For Azure OpenAI and Azure OpenAI Assistants projects:** -```powershell -$env:AZURE_OPENAI_ENDPOINT = "https://.cognitiveservices.azure.com/" -$env:AZURE_OPENAI_DEPLOYMENT_NAME = "gpt-4o" # Optional, defaults to gpt-4o -``` - -**Optional debug mode:** -```powershell -$env:AF_SHOW_ALL_DEMO_SETTING_VALUES = "Y" -``` - -If environment variables are not set, the demos will prompt you to enter values interactively. - -### Samples - -The migration samples are organized into three categories, each demonstrating different AI service integrations: - -|Category|Description| -|---|---| -|[AzureAIFoundry](./AzureAIFoundry/)|Azure OpenAI service integration samples| -|[AzureOpenAI](./AzureOpenAI/)|Direct Azure OpenAI API integration samples| -|[AzureOpenAIAssistants](./AzureOpenAIAssistants/)|Azure OpenAI Assistants API integration samples| -|[AzureOpenAIResponses](./AzureOpenAIResponses/)|Azure OpenAI Responses API integration samples| -|[OpenAI](./OpenAI/)|Direct OpenAI API integration samples| -|[OpenAIAssistants](./OpenAIAssistants/)|OpenAI Assistants API integration samples| -|[OpenAIResponses](./OpenAIResponses/)|OpenAI Responses API integration samples| - -## Running the samples from the console - -To run any migration sample, navigate to the desired sample directory: - -```powershell -# Azure AI Foundry Examples -cd "AzureAIFoundry\Step01_Basics" -dotnet run - -# Azure OpenAI Examples -cd "AzureOpenAI\Step01_Basics" -dotnet run - -# OpenAI Examples -cd "OpenAI\Step01_Basics" -dotnet run - -# OpenAI Assistants Examples -cd "OpenAIAssistants\Step01_Basics" -dotnet run - -# OpenAI Responses Examples -cd "OpenAIResponses\Step01_Basics" - -# Azure OpenAI Examples -cd "AzureOpenAI\Step01_Basics" -dotnet run - -# Azure OpenAI Assistants Examples -cd "AzureOpenAIAssistants\Step01_Basics" -dotnet run -``` diff --git a/test_simple_omniparser.ps1 b/test_simple_omniparser.ps1 index d4528c4..e69de29 100644 --- a/test_simple_omniparser.ps1 +++ b/test_simple_omniparser.ps1 @@ -1,144 +0,0 @@ -# Quick Test Script for Simple OmniParser -# Tests model loading and basic inference - -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host " Testing Simple OmniParser Implementation" -ForegroundColor Cyan -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host "" - -# Check if model exists -$modelPaths = @( - ".\FlowVision\models\icon_detect.onnx", - ".\FlowVision\bin\Debug\models\icon_detect.onnx", - ".\FlowVision\bin\Release\models\icon_detect.onnx" -) - -$modelFound = $false -foreach ($path in $modelPaths) { - if (Test-Path $path) { - Write-Host "[โœ“] Model found at: $path" -ForegroundColor Green - $fileSize = (Get-Item $path).Length / 1MB - Write-Host " Size: $([math]::Round($fileSize, 2)) MB" -ForegroundColor Gray - $modelFound = $true - break - } -} - -if (-not $modelFound) { - Write-Host "[โœ—] Model not found!" -ForegroundColor Red - Write-Host " Run: .\download_omniparser_model.ps1" -ForegroundColor Yellow - Write-Host "" - exit 1 -} - -Write-Host "" - -# Check if project files exist -Write-Host "Checking implementation files..." -ForegroundColor Cyan -Write-Host "" - -$files = @{ - "SimpleOmniParser.cs" = ".\FlowVision\lib\Classes\SimpleOmniParser.cs" - "ScreenCaptureOmniParserPlugin.cs" = ".\FlowVision\lib\Plugins\ScreenCaptureOmniParserPlugin.cs" -} - -$allFilesExist = $true -foreach ($file in $files.GetEnumerator()) { - if (Test-Path $file.Value) { - Write-Host "[โœ“] $($file.Key)" -ForegroundColor Green - } - else { - Write-Host "[โœ—] $($file.Key) - NOT FOUND" -ForegroundColor Red - $allFilesExist = $false - } -} - -Write-Host "" - -if (-not $allFilesExist) { - Write-Host "[โœ—] Some files are missing!" -ForegroundColor Red - exit 1 -} - -# Check dependencies -Write-Host "Checking dependencies..." -ForegroundColor Cyan -Write-Host "" - -$csprojPath = ".\FlowVision\FlowVision.csproj" -if (Test-Path $csprojPath) { - $csproj = Get-Content $csprojPath -Raw - - $deps = @{ - "Microsoft.ML.OnnxRuntime" = $csproj -match "Microsoft\.ML\.OnnxRuntime" - "System.Numerics.Tensors" = $csproj -match "System\.Numerics\.Tensors" - } - - foreach ($dep in $deps.GetEnumerator()) { - if ($dep.Value) { - Write-Host "[โœ“] $($dep.Key)" -ForegroundColor Green - } - else { - Write-Host "[โœ—] $($dep.Key) - NOT INSTALLED" -ForegroundColor Red - } - } -} - -Write-Host "" - -# Check build output -Write-Host "Checking build output..." -ForegroundColor Cyan -Write-Host "" - -$exePaths = @( - ".\FlowVision\bin\Debug\FlowVision.exe", - ".\FlowVision\bin\Release\FlowVision.exe" -) - -$exeFound = $false -foreach ($path in $exePaths) { - if (Test-Path $path) { - Write-Host "[โœ“] Executable found: $path" -ForegroundColor Green - $fileSize = (Get-Item $path).Length / 1MB - Write-Host " Size: $([math]::Round($fileSize, 2)) MB" -ForegroundColor Gray - - $buildTime = (Get-Item $path).LastWriteTime - $age = (Get-Date) - $buildTime - Write-Host " Last build: $($buildTime.ToString('yyyy-MM-dd HH:mm:ss')) ($([math]::Round($age.TotalMinutes, 1)) minutes ago)" -ForegroundColor Gray - $exeFound = $true - break - } -} - -if (-not $exeFound) { - Write-Host "[!] No executable found - project needs to be built" -ForegroundColor Yellow -} - -Write-Host "" -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host " Test Results" -ForegroundColor Cyan -Write-Host "===============================================" -ForegroundColor Cyan -Write-Host "" - -if ($modelFound -and $allFilesExist) { - Write-Host "[โœ“] All checks passed!" -ForegroundColor Green - Write-Host "" - Write-Host "Next steps:" -ForegroundColor Yellow - Write-Host " 1. Build the project in Visual Studio" - Write-Host " 2. Run FlowVision.exe" - Write-Host " 3. Test screen capture with OmniParser" - Write-Host "" - Write-Host "Expected behavior:" -ForegroundColor Cyan - Write-Host " - First capture: ~500ms (model loading)" - Write-Host " - Subsequent captures: ~200ms" - Write-Host " - No server startup messages" - Write-Host " - Direct ONNX inference" - Write-Host "" - exit 0 -} -else { - Write-Host "[โœ—] Some checks failed!" -ForegroundColor Red - Write-Host "" - Write-Host "Please resolve the issues above before testing." -ForegroundColor Yellow - Write-Host "" - exit 1 -}