From f157b858a31fc5793f32d20a30f38245651339ee Mon Sep 17 00:00:00 2001 From: CakeRepository Date: Fri, 28 Nov 2025 11:26:26 -0600 Subject: [PATCH] Add Gemini and local plugin support; UI and config updates Introduces Google Gemini provider support in AIProviderConfigForm and AIClientFactory, adds Clipboard and FileSystem plugins, and updates ToolConfig to enable these plugins. Refactors Actioner and MultiAgentActioner to use the new AIClientFactory for provider-agnostic chat client creation. Improves LM Studio config validation and UI, adds auto-detect for local AI endpoints, and updates OmniParserForm to reflect embedded ONNX model usage. Also includes minor fixes and enhancements to plugin APIs, OCR helper, and project file resource handling. --- FlowVision/AIProviderConfigForm.cs | 342 +++++++++++++++--- FlowVision/FlowVision.csproj | 11 +- FlowVision/OmniParserForm.Designer.cs | 61 ++-- FlowVision/OmniParserForm.cs | 30 +- FlowVision/Program.cs | 14 + FlowVision/lib/Classes/LMStudioConfig.cs | 85 ++--- FlowVision/lib/Classes/OcrHelper.cs | 48 +++ FlowVision/lib/Classes/ToolConfig.cs | 4 +- FlowVision/lib/Classes/ai/AIClientFactory.cs | 46 +++ FlowVision/lib/Classes/ai/Actioner.cs | 44 ++- FlowVision/lib/Classes/ai/LMStudioActioner.cs | 24 +- .../lib/Classes/ai/MultiAgentActioner.cs | 109 ++++-- FlowVision/lib/Plugins/ClipboardPlugin.cs | 59 +++ FlowVision/lib/Plugins/FileSystemPlugin.cs | 81 +++++ FlowVision/lib/Plugins/KeyboardPlugin.cs | 35 +- FlowVision/lib/Plugins/MousePlugin.cs | 96 ++++- .../Plugins/ScreenCaptureOmniParserPlugin.cs | 39 ++ .../lib/Plugins/WindowSelectionPlugin.cs | 136 ++++++- 18 files changed, 1039 insertions(+), 225 deletions(-) create mode 100644 FlowVision/lib/Classes/ai/AIClientFactory.cs create mode 100644 FlowVision/lib/Plugins/ClipboardPlugin.cs create mode 100644 FlowVision/lib/Plugins/FileSystemPlugin.cs diff --git a/FlowVision/AIProviderConfigForm.cs b/FlowVision/AIProviderConfigForm.cs index 7adacc0..659ffce 100644 --- a/FlowVision/AIProviderConfigForm.cs +++ b/FlowVision/AIProviderConfigForm.cs @@ -21,6 +21,12 @@ public partial class AIProviderConfigForm : Form private TextBox azureDeploymentTextBox; private TextBox azureEndpointTextBox; private TextBox azureApiKeyTextBox; + private NumericUpDown azureTemperatureUpDown; // Added missing field + + // Gemini controls + private Panel geminiPanel; + private TextBox geminiApiKeyTextBox; + private TextBox geminiModelTextBox; // LM Studio controls private Panel lmStudioPanel; @@ -96,6 +102,7 @@ private void InitializeComponent() providerComboBox.Items.AddRange(new object[] { "Azure OpenAI (Cloud)", "LM Studio (Local)", + "Google Gemini", "GitHub Models (Free Tier)" }); providerComboBox.SelectedIndexChanged += ProviderComboBox_SelectedIndexChanged; @@ -182,6 +189,58 @@ private void InitializeComponent() // Create provider-specific panels CreateAzurePanel(); CreateLMStudioPanel(); + CreateGeminiPanel(); + } + + private void CreateGeminiPanel() + { + geminiPanel = new Panel + { + Location = new Point(0, 0), + Size = new Size(580, 300), + Visible = false + }; + + int y = 0; + int labelWidth = 150; + int controlWidth = 400; + + // API Key + var apiKeyLabel = new Label { Text = "Gemini API Key:", Location = new Point(0, y + 3), Width = labelWidth }; + geminiPanel.Controls.Add(apiKeyLabel); + geminiApiKeyTextBox = new TextBox + { + Location = new Point(labelWidth + 10, y), + Width = controlWidth, + UseSystemPasswordChar = true + }; + geminiPanel.Controls.Add(geminiApiKeyTextBox); + y += 35; + + // Model Name + var modelLabel = new Label { Text = "Model Name:", Location = new Point(0, y + 3), Width = labelWidth }; + geminiPanel.Controls.Add(modelLabel); + geminiModelTextBox = new TextBox + { + Location = new Point(labelWidth + 10, y), + Width = controlWidth, + Text = "gemini-1.5-flash" + }; + geminiPanel.Controls.Add(geminiModelTextBox); + y += 35; + + // Info + var helpLabel = new Label + { + Text = "Get your API Key from: https://aistudio.google.com/app/apikey\n" + + "Standard Endpoint: https://generativelanguage.googleapis.com/v1beta/openai/", + Location = new Point(0, y), + Size = new Size(550, 40), + ForeColor = Color.Gray + }; + geminiPanel.Controls.Add(helpLabel); + + configPanel.Controls.Add(geminiPanel); } private void CreateAzurePanel() @@ -223,6 +282,22 @@ private void CreateAzurePanel() azurePanel.Controls.Add(azureApiKeyTextBox); y += 35; + // Temperature (Newly added) + var tempLabel = new Label { Text = "Temperature:", Location = new Point(0, y + 3), Width = labelWidth }; + azurePanel.Controls.Add(tempLabel); + azureTemperatureUpDown = new NumericUpDown + { + Location = new Point(labelWidth + 10, y), + Width = 100, + Minimum = 0, + Maximum = 2, + DecimalPlaces = 2, + Increment = 0.1M, + Value = 0.7M + }; + azurePanel.Controls.Add(azureTemperatureUpDown); + y += 35; + // Help text var helpLabel = new Label { @@ -247,11 +322,12 @@ private void CreateLMStudioPanel() int y = 0; int labelWidth = 150; - int controlWidth = 400; + int controlWidth = 300; // Reduced width to make room for Auto-Detect button // Endpoint URL var endpointLabel = new Label { Text = "Server Endpoint:", Location = new Point(0, y + 3), Width = labelWidth }; lmStudioPanel.Controls.Add(endpointLabel); + lmStudioEndpointTextBox = new TextBox { Location = new Point(labelWidth + 10, y), @@ -259,6 +335,19 @@ private void CreateLMStudioPanel() Text = "http://localhost:1234/v1" }; lmStudioPanel.Controls.Add(lmStudioEndpointTextBox); + + // Auto-Detect Button + var autoDetectButton = new Button + { + Text = "Auto-Detect", + Location = new Point(labelWidth + controlWidth + 20, y - 1), + Width = 100, + Height = 23, + BackColor = Color.AliceBlue + }; + autoDetectButton.Click += AutoDetectButton_Click; + lmStudioPanel.Controls.Add(autoDetectButton); + y += 35; // Model Name @@ -267,7 +356,7 @@ private void CreateLMStudioPanel() lmStudioModelTextBox = new TextBox { Location = new Point(labelWidth + 10, y), - Width = controlWidth, + Width = 400, Text = "local-model" }; lmStudioPanel.Controls.Add(lmStudioModelTextBox); @@ -280,13 +369,24 @@ private void CreateLMStudioPanel() { Location = new Point(labelWidth + 10, y), Width = 100, - Minimum = 0, - Maximum = 2, - DecimalPlaces = 2, - Increment = 0.1M, - Value = 0.7M + Minimum = 1, // Fixed to 1 + Maximum = 1, // Fixed to 1 + DecimalPlaces = 1, // Fixed to 1 decimal place + Increment = 0.0M, // No increment as it's fixed + Value = 1.0M, // Fixed value + Enabled = false // Disable user input }; lmStudioPanel.Controls.Add(lmStudioTemperatureUpDown); + + // Add an info label for temperature + var tempInfoLabel = new Label + { + Text = "Fixed at 1.0 for LM Studio models.", + Location = new Point(labelWidth + 115, y + 3), + AutoSize = true, + ForeColor = Color.DarkGray + }; + lmStudioPanel.Controls.Add(tempInfoLabel); y += 35; // Max Tokens @@ -297,7 +397,7 @@ private void CreateLMStudioPanel() Location = new Point(labelWidth + 10, y), Width = 100, Minimum = 128, - Maximum = 32768, + Maximum = 1000000, // Increased to support large context models Increment = 128, Value = 2048 }; @@ -320,11 +420,122 @@ private void CreateLMStudioPanel() configPanel.Controls.Add(lmStudioPanel); } + private async void AutoDetectButton_Click(object sender, EventArgs e) + { + statusLabel.Text = "Searching for local AI server..."; + statusLabel.ForeColor = Color.Blue; + + string[] commonEndpoints = new[] + { + "http://localhost:1234/v1", // LM Studio default + "http://127.0.0.1:1234/v1", // LM Studio IP + "http://localhost:11434/v1", // Ollama default + "http://localhost:5000/v1", // LocalAI/Oobabooga default + "http://localhost:8080/v1" // Llama.cpp server + }; + + foreach (var endpoint in commonEndpoints) + { + try + { + var client = new OpenAI.OpenAIClient( + new System.ClientModel.ApiKeyCredential("any-key"), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri(endpoint) }); + + // Just try to list models or verify endpoint validity + // Note: OpenAI client doesn't have a simple 'ping', so we assume if Uri creation works + // and we can create a client, it's a candidate. A real ping would require an API call. + // Let's try a lightweight API call to verify. + + // Create a dummy chat client to test connectivity + var chatClient = client.GetChatClient("test-model"); + var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; + + var messages = new System.Collections.Generic.List + { + new Microsoft.Extensions.AI.ChatMessage(Microsoft.Extensions.AI.ChatRole.User, "hi") + }; + + // Set a short timeout for detection + // Note: .NET 4.8 async timeout cancellation is tricky, relying on fast failure + try { + // We don't actually wait for a full response, just seeing if connection is refused immediately + // If it hangs, it might be a valid server processing. + // For now, let's just assume the first valid URI that doesn't throw immediate connection refused is good. + await ichatClient.GetResponseAsync(messages); + } + catch (Exception ex) when (ex.Message.Contains("404") || !ex.Message.Contains("connection")) + { + // 404 means server is there but model not found - that's a success for finding the server! + // Not connection error means we reached something. + } + + lmStudioEndpointTextBox.Text = endpoint; + statusLabel.Text = $"✓ Found server at {endpoint}!"; + statusLabel.ForeColor = Color.Green; + return; + } + catch + { + // Continue to next endpoint + } + } + + statusLabel.Text = "✗ No local server found. Is LM Studio running?"; + statusLabel.ForeColor = Color.Red; + } + + private async Task TestLMStudioConnection() + { + try + { + if (!Uri.TryCreate(lmStudioEndpointTextBox.Text, UriKind.Absolute, out Uri result)) + { + throw new UriFormatException("Invalid Endpoint URL format. It should look like: http://localhost:1234/v1"); + } + + var client = new OpenAI.OpenAIClient( + new System.ClientModel.ApiKeyCredential("lm-studio"), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri(lmStudioEndpointTextBox.Text) }); + + var chatClient = client.GetChatClient(lmStudioModelTextBox.Text); + + var messages = new System.Collections.Generic.List + { + new Microsoft.Extensions.AI.ChatMessage( + Microsoft.Extensions.AI.ChatRole.User, + "Say 'test' in one word") + }; + + // Cast to IChatClient - can't use AsIChatClient on OpenAI.ChatClient directly in .NET 4.8 + var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; + var response = await ichatClient.GetResponseAsync(messages); + + statusLabel.Text = "✓ LM Studio connection successful!"; + statusLabel.ForeColor = Color.Green; + } + catch (Exception ex) + { + if (ex.Message.Contains("Connection refused") || ex.Message.Contains("No connection")) + { + statusLabel.Text = "✗ Cannot connect. Is LM Studio running with server started?"; + } + else + { + statusLabel.Text = $"✗ Connection failed: {ex.Message}"; + } + statusLabel.ForeColor = Color.Red; + } + } + private void LoadConfiguration() { // Try to load existing configuration to determine current provider var lmConfig = LMStudioConfig.LoadConfig(); var azureConfig = APIConfig.LoadConfig(currentModel); + + // Load global tool config for syncing temperature + var toolConfig = ToolConfig.LoadConfig("toolsconfig"); if (lmConfig.Enabled) { @@ -335,7 +546,23 @@ private void LoadConfiguration() lmStudioEndpointTextBox.Text = lmConfig.EndpointURL; lmStudioModelTextBox.Text = lmConfig.ModelName; lmStudioTemperatureUpDown.Value = (decimal)lmConfig.Temperature; - lmStudioMaxTokensUpDown.Value = lmConfig.MaxTokens; + + // Safely set max tokens + if (lmConfig.MaxTokens < lmStudioMaxTokensUpDown.Minimum) + lmStudioMaxTokensUpDown.Value = lmStudioMaxTokensUpDown.Minimum; + else if (lmConfig.MaxTokens > lmStudioMaxTokensUpDown.Maximum) + lmStudioMaxTokensUpDown.Value = lmStudioMaxTokensUpDown.Maximum; + else + lmStudioMaxTokensUpDown.Value = lmConfig.MaxTokens; + } + else if (azureConfig.ProviderType == "Gemini") + { + // Gemini is enabled + providerComboBox.SelectedIndex = 2; // Google Gemini + enableProviderCheckBox.Checked = true; + + geminiApiKeyTextBox.Text = azureConfig.APIKey; + geminiModelTextBox.Text = azureConfig.DeploymentName; } else { @@ -346,6 +573,9 @@ private void LoadConfiguration() azureDeploymentTextBox.Text = azureConfig.DeploymentName; azureEndpointTextBox.Text = azureConfig.EndpointURL; azureApiKeyTextBox.Text = azureConfig.APIKey; + + // Init Azure temperature control (use ToolConfig temperature if available, else default) + azureTemperatureUpDown.Value = (decimal)toolConfig.Temperature; } } @@ -354,6 +584,7 @@ private void ProviderComboBox_SelectedIndexChanged(object sender, EventArgs e) // Hide all panels azurePanel.Visible = false; lmStudioPanel.Visible = false; + geminiPanel.Visible = false; // Show selected panel switch (providerComboBox.SelectedIndex) @@ -364,7 +595,10 @@ private void ProviderComboBox_SelectedIndexChanged(object sender, EventArgs e) case 1: // LM Studio lmStudioPanel.Visible = true; break; - case 2: // GitHub Models + case 2: // Google Gemini + geminiPanel.Visible = true; + break; + case 3: // GitHub Models MessageBox.Show("GitHub Models support coming soon!\nFor now, configure as Azure OpenAI with GitHub endpoint.", "Coming Soon", MessageBoxButtons.OK, MessageBoxIcon.Information); providerComboBox.SelectedIndex = 0; @@ -384,6 +618,20 @@ private async void TestButton_Click(object sender, EventArgs e) { await TestLMStudioConnection(); } + else if (providerComboBox.SelectedIndex == 2) // Gemini + { + var client = new OpenAI.OpenAIClient( + new System.ClientModel.ApiKeyCredential(geminiApiKeyTextBox.Text), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri("https://generativelanguage.googleapis.com/v1beta/openai/") } + ); + var chatClient = client.GetChatClient(geminiModelTextBox.Text); + var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; + await ichatClient.GetResponseAsync(new System.Collections.Generic.List{ + new Microsoft.Extensions.AI.ChatMessage(Microsoft.Extensions.AI.ChatRole.User, "hi") + }); + statusLabel.Text = "✓ Gemini connection successful!"; + statusLabel.ForeColor = Color.Green; + } else // Azure OpenAI { await TestAzureConnection(); @@ -431,44 +679,6 @@ private async Task TestAzureConnection() } } - private async Task TestLMStudioConnection() - { - try - { - var client = new OpenAI.OpenAIClient( - new System.ClientModel.ApiKeyCredential("lm-studio"), - new OpenAI.OpenAIClientOptions { Endpoint = new Uri(lmStudioEndpointTextBox.Text) }); - - var chatClient = client.GetChatClient(lmStudioModelTextBox.Text); - - var messages = new System.Collections.Generic.List - { - new Microsoft.Extensions.AI.ChatMessage( - Microsoft.Extensions.AI.ChatRole.User, - "Say 'test' in one word") - }; - - // Cast to IChatClient - can't use AsIChatClient on OpenAI.ChatClient directly in .NET 4.8 - var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; - var response = await ichatClient.GetResponseAsync(messages); - - statusLabel.Text = "✓ LM Studio connection successful!"; - statusLabel.ForeColor = Color.Green; - } - catch (Exception ex) - { - if (ex.Message.Contains("Connection refused") || ex.Message.Contains("No connection")) - { - statusLabel.Text = "✗ Cannot connect. Is LM Studio running with server started?"; - } - else - { - statusLabel.Text = $"✗ Connection failed: {ex.Message}"; - } - statusLabel.ForeColor = Color.Red; - } - } - private void SaveButton_Click(object sender, EventArgs e) { try @@ -477,6 +687,10 @@ private void SaveButton_Click(object sender, EventArgs e) { SaveLMStudioConfig(); } + else if (providerComboBox.SelectedIndex == 2) // Gemini + { + SaveGeminiConfig(); + } else // Azure OpenAI { SaveAzureConfig(); @@ -500,6 +714,23 @@ private void SaveButton_Click(object sender, EventArgs e) } } + private void SaveGeminiConfig() + { + var config = new APIConfig + { + DeploymentName = geminiModelTextBox.Text, + EndpointURL = "https://generativelanguage.googleapis.com/v1beta/openai/", + APIKey = geminiApiKeyTextBox.Text, + ProviderType = "Gemini" + }; + config.SaveConfig(currentModel); + + // Disable LM Studio + var lmConfig = LMStudioConfig.LoadConfig(); + lmConfig.Enabled = false; + lmConfig.SaveConfig(); + } + private void SaveAzureConfig() { var config = new APIConfig @@ -515,10 +746,20 @@ private void SaveAzureConfig() var lmConfig = LMStudioConfig.LoadConfig(); lmConfig.Enabled = false; lmConfig.SaveConfig(); + + // Sync global tool config temperature + var toolConfig = ToolConfig.LoadConfig("toolsconfig"); + toolConfig.Temperature = (double)azureTemperatureUpDown.Value; + toolConfig.SaveConfig("toolsconfig"); } private void SaveLMStudioConfig() { + if (!Uri.TryCreate(lmStudioEndpointTextBox.Text, UriKind.Absolute, out Uri result)) + { + throw new UriFormatException("Invalid Endpoint URL format. It should look like: http://localhost:1234/v1"); + } + var config = new LMStudioConfig { EndpointURL = lmStudioEndpointTextBox.Text, @@ -538,6 +779,11 @@ private void SaveLMStudioConfig() ProviderType = "LMStudio" }; azureConfig.SaveConfig(currentModel); + + // Sync global tool config temperature + var toolConfig = ToolConfig.LoadConfig("toolsconfig"); + toolConfig.Temperature = (double)lmStudioTemperatureUpDown.Value; + toolConfig.SaveConfig("toolsconfig"); } } } diff --git a/FlowVision/FlowVision.csproj b/FlowVision/FlowVision.csproj index f9ee151..d4b1db2 100644 --- a/FlowVision/FlowVision.csproj +++ b/FlowVision/FlowVision.csproj @@ -211,6 +211,7 @@ + @@ -250,7 +251,9 @@ UserControl + + @@ -301,6 +304,7 @@ + @@ -327,13 +331,6 @@ - - - - - - - diff --git a/FlowVision/OmniParserForm.Designer.cs b/FlowVision/OmniParserForm.Designer.cs index 6250925..1110bea 100644 --- a/FlowVision/OmniParserForm.Designer.cs +++ b/FlowVision/OmniParserForm.Designer.cs @@ -28,52 +28,57 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { - this.omniParserServerURL = new System.Windows.Forms.TextBox(); this.label1 = new System.Windows.Forms.Label(); - this.saveButton = new System.Windows.Forms.Button(); + this.statusLabel = new System.Windows.Forms.Label(); + this.infoLabel = new System.Windows.Forms.Label(); this.SuspendLayout(); // - // omniParserServerURL - // - this.omniParserServerURL.Font = new System.Drawing.Font("Comic Sans MS", 12F); - this.omniParserServerURL.Location = new System.Drawing.Point(139, 7); - this.omniParserServerURL.Name = "omniParserServerURL"; - this.omniParserServerURL.Size = new System.Drawing.Size(355, 30); - this.omniParserServerURL.TabIndex = 0; - // // label1 // this.label1.AutoSize = true; - this.label1.Font = new System.Drawing.Font("Comic Sans MS", 11.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); - this.label1.Location = new System.Drawing.Point(12, 12); + this.label1.Font = new System.Drawing.Font("Segoe UI", 12F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, ((byte)(0))); + this.label1.Location = new System.Drawing.Point(20, 20); this.label1.Name = "label1"; - this.label1.Size = new System.Drawing.Size(121, 20); + this.label1.Size = new System.Drawing.Size(154, 21); this.label1.TabIndex = 1; - this.label1.Text = "OmniParser URL"; + this.label1.Text = "OmniParser Status:"; + // + // statusLabel + // + this.statusLabel.AutoSize = true; + this.statusLabel.Font = new System.Drawing.Font("Segoe UI", 12F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); + this.statusLabel.ForeColor = System.Drawing.Color.Green; + this.statusLabel.Location = new System.Drawing.Point(180, 20); + this.statusLabel.Name = "statusLabel"; + this.statusLabel.Size = new System.Drawing.Size(196, 21); + this.statusLabel.TabIndex = 3; + this.statusLabel.Text = "✓ Embedded Mode Active"; // - // saveButton + // infoLabel // - this.saveButton.Location = new System.Drawing.Point(162, 41); - this.saveButton.Name = "saveButton"; - this.saveButton.Size = new System.Drawing.Size(140, 23); - this.saveButton.TabIndex = 2; - this.saveButton.Text = "Save"; - this.saveButton.UseVisualStyleBackColor = true; - this.saveButton.Click += new System.EventHandler(this.saveButton_Click); + this.infoLabel.AutoSize = true; + this.infoLabel.Font = new System.Drawing.Font("Segoe UI", 9.75F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); + this.infoLabel.ForeColor = System.Drawing.Color.DimGray; + this.infoLabel.Location = new System.Drawing.Point(21, 55); + this.infoLabel.Name = "infoLabel"; + this.infoLabel.Size = new System.Drawing.Size(380, 34); + this.infoLabel.TabIndex = 4; + this.infoLabel.Text = "FlowVision is using the internal ONNX detection model.\r\nNo external server or configuration is required."; // // OmniParserForm // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.ClientSize = new System.Drawing.Size(506, 76); - this.Controls.Add(this.saveButton); + this.ClientSize = new System.Drawing.Size(440, 110); + this.Controls.Add(this.infoLabel); + this.Controls.Add(this.statusLabel); this.Controls.Add(this.label1); - this.Controls.Add(this.omniParserServerURL); this.MaximizeBox = false; this.MinimizeBox = false; this.Name = "OmniParserForm"; this.ShowIcon = false; - this.Text = "OmniParserServer Config"; + this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen; + this.Text = "OmniParser Configuration"; this.Load += new System.EventHandler(this.OmniParserForm_Load); this.ResumeLayout(false); this.PerformLayout(); @@ -82,8 +87,8 @@ private void InitializeComponent() #endregion - private System.Windows.Forms.TextBox omniParserServerURL; private System.Windows.Forms.Label label1; - private System.Windows.Forms.Button saveButton; + private System.Windows.Forms.Label statusLabel; + private System.Windows.Forms.Label infoLabel; } } \ No newline at end of file diff --git a/FlowVision/OmniParserForm.cs b/FlowVision/OmniParserForm.cs index c9aeec0..aaa2a08 100644 --- a/FlowVision/OmniParserForm.cs +++ b/FlowVision/OmniParserForm.cs @@ -1,11 +1,4 @@ using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Data; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows.Forms; using FlowVision.lib.Classes; @@ -13,35 +6,14 @@ namespace FlowVision { public partial class OmniParserForm : Form { - private OmniParserConfig _config; - public OmniParserForm() { InitializeComponent(); - _config = OmniParserConfig.LoadConfig(); - } - - private void saveButton_Click(object sender, EventArgs e) - { - string url = omniParserServerURL.Text.Trim(); - if (string.IsNullOrEmpty(url)) - { - MessageBox.Show("Please enter a valid URL."); - return; - } - - // Save the URL to the config file - _config.ServerURL = url; - _config.SaveConfig(); - - // Optionally, you can close the form after saving - this.Close(); } private void OmniParserForm_Load(object sender, EventArgs e) { - // Load the URL from the config file - omniParserServerURL.Text = _config.ServerURL; + // Form is now just an informational status display } } } diff --git a/FlowVision/Program.cs b/FlowVision/Program.cs index 84102c4..d7f93bb 100644 --- a/FlowVision/Program.cs +++ b/FlowVision/Program.cs @@ -14,6 +14,20 @@ static class Program [STAThread] static void Main() { + // Preload the ONNX model on a background thread so it's ready when needed + Task.Run(() => + { + try + { + // Accessing the Instance property triggers the model loading + var parser = FlowVision.lib.Classes.SimpleOmniParser.Instance; + } + catch + { + // Ignore startup errors - they will be caught/logged when the user actually tries to use it + } + }); + Application.EnableVisualStyles(); Application.SetCompatibleTextRenderingDefault(false); Application.Run(new Form1()); diff --git a/FlowVision/lib/Classes/LMStudioConfig.cs b/FlowVision/lib/Classes/LMStudioConfig.cs index fb196f0..c03b76e 100644 --- a/FlowVision/lib/Classes/LMStudioConfig.cs +++ b/FlowVision/lib/Classes/LMStudioConfig.cs @@ -4,77 +4,59 @@ namespace FlowVision.lib.Classes { - /// - /// Configuration for LM Studio local AI integration - /// public class LMStudioConfig { - /// - /// LM Studio server endpoint (default: http://localhost:1234/v1) - /// public string EndpointURL { get; set; } = "http://localhost:1234/v1"; - - /// - /// Model name (e.g., "gpt-3.5-turbo", "local-model", or whatever LM Studio shows) - /// Can be left as default - LM Studio typically uses the loaded model automatically - /// public string ModelName { get; set; } = "local-model"; - - /// - /// API key (LM Studio doesn't require one, but field kept for compatibility) - /// Use "lm-studio" or "not-needed" as placeholder - /// - public string APIKey { get; set; } = "lm-studio"; - - /// - /// Whether to use LM Studio or fall back to Azure - /// - public bool Enabled { get; set; } = false; - - /// - /// Temperature setting for local model - /// public double Temperature { get; set; } = 0.7; - - /// - /// Max tokens for completion - /// public int MaxTokens { get; set; } = 2048; - - /// - /// Timeout in seconds for LM Studio requests - /// - public int TimeoutSeconds { get; set; } = 300; + public bool Enabled { get; set; } = false; + public string APIKey { get; set; } = "lm-studio"; // OpenAI client requires a key even if local + public int TimeoutSeconds { get; set; } = 120; - private static string ConfigFilePath() - { - return Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), - "FlowVision", - "lmstudioconfig.json"); - } + // Track if the config was successfully loaded from disk + [System.Text.Json.Serialization.JsonIgnore] + public bool IsValid { get; set; } = true; + + private static string ConfigFilePath => Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "FlowVision", + "lmstudioconfig.json"); public static LMStudioConfig LoadConfig() { try { - Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath())); + Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath)); - if (File.Exists(ConfigFilePath())) + if (File.Exists(ConfigFilePath)) { - string jsonContent = File.ReadAllText(ConfigFilePath()); - if (!string.IsNullOrWhiteSpace(jsonContent)) + string jsonContent = File.ReadAllText(ConfigFilePath); + // Basic validation for empty file + if (string.IsNullOrWhiteSpace(jsonContent)) { - var config = JsonSerializer.Deserialize(jsonContent); - return config ?? new LMStudioConfig(); + return new LMStudioConfig { IsValid = false }; + } + + var config = JsonSerializer.Deserialize(jsonContent); + if (config != null) + { + config.IsValid = true; + return config; } } } catch (Exception ex) { - Console.WriteLine($"Error loading LM Studio config: {ex.Message}"); + // Return a config marked as invalid so the UI can warn the user + return new LMStudioConfig + { + Enabled = false, + IsValid = false + }; } + // Return default if file doesn't exist return new LMStudioConfig(); } @@ -82,15 +64,16 @@ public void SaveConfig() { try { - Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath())); + Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath)); var options = new JsonSerializerOptions { WriteIndented = true }; string jsonContent = JsonSerializer.Serialize(this, options); - File.WriteAllText(ConfigFilePath(), jsonContent); + File.WriteAllText(ConfigFilePath, jsonContent); } catch (Exception ex) { Console.WriteLine($"Error saving LM Studio config: {ex.Message}"); + throw; // Re-throw so the UI can show the error } } } diff --git a/FlowVision/lib/Classes/OcrHelper.cs b/FlowVision/lib/Classes/OcrHelper.cs index 6b9a308..5ae1a6e 100644 --- a/FlowVision/lib/Classes/OcrHelper.cs +++ b/FlowVision/lib/Classes/OcrHelper.cs @@ -82,6 +82,54 @@ private static void Initialize() } } + /// + /// Search for specific text in an image and return its bounding box. + /// Returns the first match found. + /// + public static async Task FindTextLocationAsync(Bitmap image, string searchText) + { + if (!_isAvailable || _engine == null || string.IsNullOrWhiteSpace(searchText)) + return null; + + return await Task.Run(() => + { + try + { + lock (_lock) + { + using (var pix = PixConverter.ToPix(image)) + using (var page = _engine.Process(pix)) + using (var iter = page.GetIterator()) + { + iter.Begin(); + do + { + // Get text at current iterator level (Word) + string currentText = iter.GetText(PageIteratorLevel.Word)?.Trim(); + + // Simple case-insensitive match + if (!string.IsNullOrWhiteSpace(currentText) && + currentText.Equals(searchText, StringComparison.OrdinalIgnoreCase)) + { + // Found exact match! Get bounding box. + if (iter.TryGetBoundingBox(PageIteratorLevel.Word, out var rect)) + { + return (Rectangle?)new Rectangle(rect.X1, rect.Y1, rect.Width, rect.Height); + } + } + } while (iter.Next(PageIteratorLevel.Word)); + } + } + return (Rectangle?)null; + } + catch (Exception ex) + { + PluginLogger.LogError("OcrHelper", "FindTextLocationAsync", $"OCR search failed: {ex.Message}"); + return (Rectangle?)null; + } + }); + } + /// /// Extract text from a bitmap image /// diff --git a/FlowVision/lib/Classes/ToolConfig.cs b/FlowVision/lib/Classes/ToolConfig.cs index 27ea735..42d0450 100644 --- a/FlowVision/lib/Classes/ToolConfig.cs +++ b/FlowVision/lib/Classes/ToolConfig.cs @@ -12,12 +12,14 @@ public class ToolConfig public bool EnableKeyboardPlugin { get; set; } = true; public bool EnableMousePlugin { get; set; } = false; // Changed default to false public bool EnableWindowSelectionPlugin { get; set; } = true; // Added WindowSelectionPlugin + public bool EnableClipboardPlugin { get; set; } = true; // Added ClipboardPlugin + public bool EnableFileSystemPlugin { get; set; } = true; // Added FileSystemPlugin public bool EnableSpeechRecognition { get; set; } = true; // Added Speech Recognition option public string SpeechRecognitionLanguage { get; set; } = "en-US"; // Default language public string VoiceCommandPhrase { get; set; } = "send message"; // Default voice command phrase public bool EnableVoiceCommands { get; set; } = true; // Enable voice commands feature public bool EnablePluginLogging { get; set; } = true; - public double Temperature { get; set; } = 0.2; + public double Temperature { get; set; } = 1.0; public bool AutoInvokeKernelFunctions { get; set; } = true; public bool RetainChatHistory { get; set; } = true; public bool EnableMultiAgentMode { get; set; } = false; // Changed default to false diff --git a/FlowVision/lib/Classes/ai/AIClientFactory.cs b/FlowVision/lib/Classes/ai/AIClientFactory.cs new file mode 100644 index 0000000..2e9e85b --- /dev/null +++ b/FlowVision/lib/Classes/ai/AIClientFactory.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.Extensions.AI; +using Azure.AI.OpenAI; +using Azure; +using OpenAI; + +namespace FlowVision.lib.Classes.ai +{ + public static class AIClientFactory + { + public static IChatClient CreateClient(APIConfig config) + { + if (config == null) + throw new ArgumentNullException(nameof(config)); + + switch (config.ProviderType?.ToLowerInvariant()) + { + case "gemini": + // Use standard OpenAI client pointing to Google's endpoint + // Endpoint format: https://generativelanguage.googleapis.com/v1beta/openai/ + var geminiClient = new OpenAIClient( + new System.ClientModel.ApiKeyCredential(config.APIKey), + new OpenAIClientOptions { Endpoint = new Uri(config.EndpointURL) } + ); + return geminiClient.GetChatClient(config.DeploymentName).AsIChatClient(); + + case "lmstudio": + case "openai": // Generic OpenAI compatible + var openAIClient = new OpenAIClient( + new System.ClientModel.ApiKeyCredential(config.APIKey), + new OpenAIClientOptions { Endpoint = new Uri(config.EndpointURL) } + ); + return openAIClient.GetChatClient(config.DeploymentName).AsIChatClient(); + + case "azureopenai": + default: + // Default to Azure OpenAI + var azureClient = new AzureOpenAIClient( + new Uri(config.EndpointURL), + new AzureKeyCredential(config.APIKey) + ); + return azureClient.GetChatClient(config.DeploymentName).AsIChatClient(); + } + } + } +} diff --git a/FlowVision/lib/Classes/ai/Actioner.cs b/FlowVision/lib/Classes/ai/Actioner.cs index cc4a075..7381454 100644 --- a/FlowVision/lib/Classes/ai/Actioner.cs +++ b/FlowVision/lib/Classes/ai/Actioner.cs @@ -1,4 +1,10 @@ -using System; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -7,10 +13,8 @@ using System.Windows.Forms; using FlowVision.lib.Plugins; using Microsoft.Extensions.AI; -using Azure.AI.OpenAI; -using Azure; -using OpenAI; -using ChatMessage = Microsoft.Extensions.AI.ChatMessage; +using FlowVision.lib.Classes.ai; +using FlowVision; // Required for Form1 namespace FlowVision.lib.Classes { @@ -128,9 +132,8 @@ public async Task ExecuteAction(string actionPrompt) return "Error: Actioner model not configured"; } - // Create Azure OpenAI chat client with IChatClient interface - var azureClient = new AzureOpenAIClient(new Uri(config.EndpointURL), new AzureKeyCredential(config.APIKey)); - IChatClient baseChatClient = azureClient.GetChatClient(config.DeploymentName).AsIChatClient(); + // Use the Factory to create the client based on ProviderType + IChatClient baseChatClient = AIClientFactory.CreateClient(config); // Collect tools based on configuration var tools = new List(); @@ -175,6 +178,16 @@ public async Task ExecuteAction(string actionPrompt) tools.AddRange(PluginToolExtractor.ExtractTools(new RemoteControlPlugin())); } + if (toolConfig.EnableClipboardPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new ClipboardPlugin())); + } + + if (toolConfig.EnableFileSystemPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new FileSystemPlugin())); + } + // Configure chat options with tools var chatOptions = new ChatOptions { @@ -193,13 +206,20 @@ public async Task ExecuteAction(string actionPrompt) // Process the response with streaming var responseBuilder = new StringBuilder(); - await foreach (var update in actionerChat.GetStreamingResponseAsync(actionerHistory, chatOptions)) + try { - if (update.Text != null) + await foreach (var update in actionerChat.GetStreamingResponseAsync(actionerHistory, chatOptions)) { - responseBuilder.Append(update.Text); + if (update.Text != null) + { + responseBuilder.Append(update.Text); + } } } + catch (Exception ex) when (ex.Message.Contains("Unknown ChatFinishReason") || ex.Message.Contains("function_call_filter")) + { + PluginLogger.LogInfo("Actioner", "ExecuteAction", $"Ignored known SDK finish reason error: {ex.Message}"); + } // Task completed successfully PluginLogger.NotifyTaskComplete("Action Execution", true); @@ -222,7 +242,7 @@ public async Task ExecuteAction(string actionPrompt) } } - internal void SetChatHistory(List chatHistory) + internal void SetChatHistory(System.Collections.Generic.List chatHistory) { actionerHistory.Clear(); foreach (var message in chatHistory) diff --git a/FlowVision/lib/Classes/ai/LMStudioActioner.cs b/FlowVision/lib/Classes/ai/LMStudioActioner.cs index 3a3c590..d5b1f89 100644 --- a/FlowVision/lib/Classes/ai/LMStudioActioner.cs +++ b/FlowVision/lib/Classes/ai/LMStudioActioner.cs @@ -3,12 +3,18 @@ using System.Linq; using System.Reflection; using System.Text; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; using System.Threading.Tasks; using System.Windows.Forms; using FlowVision.lib.Plugins; using Microsoft.Extensions.AI; using OpenAI; using ChatMessage = Microsoft.Extensions.AI.ChatMessage; +using FlowVision; // Required for Form1 namespace FlowVision.lib.Classes { @@ -107,6 +113,20 @@ 2. After CaptureWholeScreen(), you MUST continue to actually click/type/interact return "Error: LM Studio endpoint not configured. Default is http://localhost:1234/v1"; } + // Check for invalid config load + if (!lmStudioConfig.IsValid) + { + PluginLogger.NotifyTaskComplete("LM Studio Action Execution", false); + return "Error: LM Studio configuration file is corrupt. Please go to settings and re-save the configuration."; + } + + // Validate URI format before attempting to create client + if (!Uri.TryCreate(lmStudioConfig.EndpointURL, UriKind.Absolute, out _)) + { + PluginLogger.NotifyTaskComplete("LM Studio Action Execution", false); + return $"Error: Invalid Endpoint URL '{lmStudioConfig.EndpointURL}'. Please correct it in settings."; + } + // Create OpenAI client pointing to LM Studio // LM Studio provides an OpenAI-compatible API var openAIClient = new OpenAIClient(new System.ClientModel.ApiKeyCredential(lmStudioConfig.APIKey), new OpenAIClientOptions @@ -164,7 +184,7 @@ 2. After CaptureWholeScreen(), you MUST continue to actually click/type/interact // Configure chat options with tools var chatOptions = new ChatOptions { - Temperature = (float)lmStudioConfig.Temperature, + Temperature = 1.0f, // Fixed at 1.0 for LM Studio models MaxOutputTokens = lmStudioConfig.MaxTokens, Tools = tools }; @@ -242,7 +262,7 @@ 2. After CaptureWholeScreen(), you MUST continue to actually click/type/interact } } - public void SetChatHistory(List chatHistory) + public void SetChatHistory(System.Collections.Generic.List chatHistory) { actionerHistory.Clear(); foreach (var message in chatHistory) diff --git a/FlowVision/lib/Classes/ai/MultiAgentActioner.cs b/FlowVision/lib/Classes/ai/MultiAgentActioner.cs index 417e4bf..35cb0d3 100644 --- a/FlowVision/lib/Classes/ai/MultiAgentActioner.cs +++ b/FlowVision/lib/Classes/ai/MultiAgentActioner.cs @@ -6,9 +6,10 @@ using System.Windows.Forms; using FlowVision.lib.Plugins; using Microsoft.Extensions.AI; -using Azure.AI.OpenAI; -using Azure; -using ChatMessage = Microsoft.Extensions.AI.ChatMessage; +using FlowVision.lib.Classes.ai; +using Azure.AI.OpenAI; // Needed for some types if referenced, but Factory returns IChatClient +using Azure; // Needed for AzureKeyCredential if strictly typed, but Factory handles it. +using FlowVision; // Required for Form1 namespace FlowVision.lib.Classes { @@ -23,7 +24,7 @@ public class MultiAgentActioner private List coordinatorHistory; private List plannerHistory; private List actionerHistory; - private AgentCoordinator agentCoordinator; + private AgentCoordinator agentCoordinator; // Configuration constants private const string TOOL_CONFIG = "toolsconfig"; @@ -144,17 +145,12 @@ public async Task ExecuteAction(string actionPrompt) return "Error: Actioner model not configured"; } - // Setup coordinator chat client (no tools, only coordination capabilities) - var coordinatorAzureClient = new AzureOpenAIClient(new Uri(coordinatorConfig.EndpointURL), new AzureKeyCredential(coordinatorConfig.APIKey)); - coordinatorChat = coordinatorAzureClient.GetChatClient(coordinatorConfig.DeploymentName).AsIChatClient(); - - // Setup planner chat client (no tools, only planning capabilities) - var plannerAzureClient = new AzureOpenAIClient(new Uri(plannerConfig.EndpointURL), new AzureKeyCredential(plannerConfig.APIKey)); - plannerChat = plannerAzureClient.GetChatClient(plannerConfig.DeploymentName).AsIChatClient(); - - // Setup actioner chat client with all tools - var actionerAzureClient = new AzureOpenAIClient(new Uri(actionerConfig.EndpointURL), new AzureKeyCredential(actionerConfig.APIKey)); - IChatClient actionerChatBase = actionerAzureClient.GetChatClient(actionerConfig.DeploymentName).AsIChatClient(); + // Setup clients using the Factory (supports Azure, Gemini, etc.) + coordinatorChat = AIClientFactory.CreateClient(coordinatorConfig); + plannerChat = AIClientFactory.CreateClient(plannerConfig); + + // Setup actioner client base + IChatClient actionerChatBase = AIClientFactory.CreateClient(actionerConfig); // Collect tools based on configuration var tools = new List(); @@ -199,6 +195,16 @@ public async Task ExecuteAction(string actionPrompt) tools.AddRange(PluginToolExtractor.ExtractTools(new RemoteControlPlugin())); } + if (toolConfig.EnableClipboardPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new ClipboardPlugin())); + } + + if (toolConfig.EnableFileSystemPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new FileSystemPlugin())); + } + // Setup actioner with function invocation using builder pattern actionerChat = new ChatClientBuilder(actionerChatBase).UseFunctionInvocation().Build(); @@ -209,12 +215,12 @@ public async Task ExecuteAction(string actionPrompt) var coordinatorOptions = new ChatOptions { - Temperature = 0.2f + Temperature = (float)toolConfig.Temperature }; var plannerOptions = new ChatOptions { - Temperature = 0.2f + Temperature = (float)toolConfig.Temperature }; var actionerOptions = new ChatOptions @@ -252,13 +258,19 @@ public async Task ExecuteAction(string actionPrompt) int currentIteration = 0; string finalResult = ""; List executionResults = new List(); + + // Novel Feature: Focus Tracking + // Keep track of window focus to detect popups (like "Save As") automatically + var windowTracker = new WindowSelectionPlugin(); while (!isComplete && currentIteration < maxIterations) { currentIteration++; PluginLogger.LogPluginUsage($"⚙️ Step {currentIteration}/{maxIterations}"); - + // Capture pre-action state + string preActionWindow = windowTracker.GetForegroundWindowInfo(); + // Ask actioner to perform the current step with clearer instructions actionerHistory.Add(new ChatMessage(ChatRole.User, $"Execute this step:\n\n{plan}\n\n" + @@ -281,6 +293,28 @@ public async Task ExecuteAction(string actionPrompt) // Get actioner response with tools string executionResult = await GetAgentResponseAsync(actionerChat, actionerHistory, actionerOptions); + // Capture post-action state + string postActionWindow = windowTracker.GetForegroundWindowInfo(); + + // FIX 1: Handle empty execution results (common with successful shell commands) + if (string.IsNullOrWhiteSpace(executionResult)) + { + executionResult = "The command executed successfully with no output."; + } + + // Novel Feature: Inject Focus Change Alert + // If the active window changed (e.g. "Save As" dialog popped up), explicitly tell the Planner. + if (preActionWindow != postActionWindow) + { + string alert = $"\n\n[SYSTEM ALERT]: Active window focus changed!\n" + + $"Previous: {preActionWindow}\n" + + $"Current: {postActionWindow}\n" + + $"Use the new Handle ({postActionWindow.Split(',')[0]}) for subsequent interactions."; + + executionResult += alert; + PluginLogger.LogInfo("MultiAgentActioner", "ExecuteAction", "Detected window focus change, alerting Planner."); + } + // Store the execution result for the final response executionResults.Add(executionResult); @@ -290,6 +324,10 @@ public async Task ExecuteAction(string actionPrompt) "EXECUTION_RESPONSE", executionResult); + // FIX 2: Manage context window to prevent token overflow + ManageContextWindow(actionerHistory, 10); + ManageContextWindow(plannerHistory, 10); + // Add the execution result to the planner's history with clearer prompting plannerHistory.Add(new ChatMessage(ChatRole.User, @@ -398,13 +436,22 @@ private async Task GetAgentResponseAsync( { var responseBuilder = new StringBuilder(); - await foreach (var update in chatService.GetStreamingResponseAsync(history, options)) + try { - if (update.Text != null) + await foreach (var update in chatService.GetStreamingResponseAsync(history, options)) { - responseBuilder.Append(update.Text); + if (update.Text != null) + { + responseBuilder.Append(update.Text); + } } } + catch (Exception ex) when (ex.Message.Contains("Unknown ChatFinishReason") || ex.Message.Contains("function_call_filter")) + { + // Swallow known SDK mapping errors for specific provider finish reasons + // This allows us to keep the text generated so far + PluginLogger.LogInfo("MultiAgentActioner", "GetAgentResponseAsync", $"Ignored known SDK finish reason error: {ex.Message}"); + } string response = responseBuilder.ToString(); history.Add(new ChatMessage(ChatRole.Assistant, response)); @@ -412,6 +459,22 @@ private async Task GetAgentResponseAsync( return response; } + private void ManageContextWindow(List history, int maxMessages) + { + // Always keep the system prompt (assumed to be at index 0) + if (history.Count <= maxMessages + 1) return; + + // Calculate how many messages to remove + // We want to keep: SystemPrompt (1) + Last N messages + int messagesToRemove = history.Count - (maxMessages + 1); + + if (messagesToRemove > 0) + { + // Remove messages starting from index 1 (preserve System Prompt) + history.RemoveRange(1, messagesToRemove); + } + } + /// /// Extracts the first actionable step from the planner's plan. /// Looks for lines that mention a tool/plugin or a direct action. @@ -443,7 +506,7 @@ private string ExtractActionableStep(string plan) return null; } - public void SetChatHistory(List chatHistory) + public void SetChatHistory(System.Collections.Generic.List chatHistory) { // Set up coordinator history with system prompt coordinatorHistory.Clear(); @@ -469,4 +532,4 @@ public void SetChatHistory(List chatHistory) } } } -} \ No newline at end of file +} diff --git a/FlowVision/lib/Plugins/ClipboardPlugin.cs b/FlowVision/lib/Plugins/ClipboardPlugin.cs new file mode 100644 index 0000000..e1ce087 --- /dev/null +++ b/FlowVision/lib/Plugins/ClipboardPlugin.cs @@ -0,0 +1,59 @@ +using System; +using System.ComponentModel; +using System.Windows.Forms; +using FlowVision.lib.Classes; + +namespace FlowVision.lib.Plugins +{ + internal class ClipboardPlugin + { + [Description("Sets the system clipboard text content")] + public void SetClipboardText(string text) + { + PluginLogger.LogPluginUsage("ClipboardPlugin", "SetClipboardText", text); + + if (Application.OpenForms.Count > 0) + { + Application.OpenForms[0].Invoke(new Action(() => { + try + { + Clipboard.SetText(text); + } + catch (Exception ex) + { + PluginLogger.LogError("ClipboardPlugin", "SetClipboardText", ex.Message); + } + })); + } + } + + [Description("Gets the current text content from the system clipboard")] + public string GetClipboardText() + { + PluginLogger.LogPluginUsage("ClipboardPlugin", "GetClipboardText"); + string clipboardText = ""; + + if (Application.OpenForms.Count > 0) + { + clipboardText = (string)Application.OpenForms[0].Invoke(new Func(() => { + try + { + if (Clipboard.ContainsText()) + { + return Clipboard.GetText(); + } + else + { + return "[Clipboard is empty or contains non-text data]"; + } + } + catch (Exception ex) + { + return $"Error reading clipboard: {ex.Message}"; + } + })); + } + return clipboardText; + } + } +} diff --git a/FlowVision/lib/Plugins/FileSystemPlugin.cs b/FlowVision/lib/Plugins/FileSystemPlugin.cs new file mode 100644 index 0000000..4400e84 --- /dev/null +++ b/FlowVision/lib/Plugins/FileSystemPlugin.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.IO; +using System.Linq; +using FlowVision.lib.Classes; + +namespace FlowVision.lib.Plugins +{ + internal class FileSystemPlugin + { + [Description("Gets the current working directory of the application")] + public string GetCurrentDirectory() + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "GetCurrentDirectory"); + return Directory.GetCurrentDirectory(); + } + + [Description("Lists files and directories in the specified path. Returns first 50 entries.")] + public string ListDirectory(string path) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "ListDirectory", path); + try + { + if (!Directory.Exists(path)) return $"Directory not found: {path}"; + + var dirs = Directory.GetDirectories(path).Select(d => $"[DIR] {Path.GetFileName(d)}"); + var files = Directory.GetFiles(path).Select(f => Path.GetFileName(f)); + + var all = dirs.Concat(files).Take(50); + return string.Join("\n", all); + } + catch (Exception ex) + { + return $"Error: {ex.Message}"; + } + } + + [Description("Checks if a file exists at the specified path")] + public bool FileExists(string path) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "FileExists", path); + return File.Exists(path); + } + + [Description("Reads the content of a text file (max 2000 chars)")] + public string ReadFile(string path) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "ReadFile", path); + try + { + if (!File.Exists(path)) return "File not found"; + + string content = File.ReadAllText(path); + if (content.Length > 2000) + { + return content.Substring(0, 2000) + "\n...[Truncated]..."; + } + return content; + } + catch (Exception ex) + { + return $"Error: {ex.Message}"; + } + } + + [Description("Writes text content to a file. Overwrites if exists.")] + public string WriteFile(string path, string content) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "WriteFile", path); + try + { + File.WriteAllText(path, content); + return $"Successfully wrote to {path}"; + } + catch (Exception ex) + { + return $"Error writing file: {ex.Message}"; + } + } + } +} diff --git a/FlowVision/lib/Plugins/KeyboardPlugin.cs b/FlowVision/lib/Plugins/KeyboardPlugin.cs index 4d2f341..05747e1 100644 --- a/FlowVision/lib/Plugins/KeyboardPlugin.cs +++ b/FlowVision/lib/Plugins/KeyboardPlugin.cs @@ -53,7 +53,7 @@ public async Task SendKey(string keyCombo) } } - [Description("Send keyboard input to a specific window by handle")] + [Description("Send keyboard input to a specific window by handle. keys format: standard SendKeys (e.g. 'Hello', '{ENTER}', '^c').")] public async Task SendKeyToWindow(string windowHandleString, string keyCombo) { PluginLogger.LogPluginUsage("KeyboardPlugin", "SendKeyToWindow", @@ -72,7 +72,8 @@ public async Task SendKeyToWindow(string windowHandleString, string keyCom } // Wait a bit for the window to become active - await Task.Delay(200); + // Increased to 500ms to ensure slower apps (like Notepad startup) are ready + await Task.Delay(500); // Send the keys SendKeys.SendWait(keyCombo); @@ -132,6 +133,16 @@ public async Task CtrlKeyToWindow(string windowHandleString, string letter return await SendKeyToWindow(windowHandleString, $"^({letter})"); } + [DllImport("user32.dll")] + private static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); + + [DllImport("user32.dll")] + private static extern IntPtr SetFocus(IntPtr hWnd); + + private const int SW_RESTORE = 9; + + // ... (existing imports) + /// /// Brings a window to the foreground and ensures it has focus using multiple techniques /// @@ -142,13 +153,12 @@ private bool BringWindowToForegroundWithFocus(IntPtr hWnd) try { + // 1. Force window restore if minimized + ShowWindow(hWnd, SW_RESTORE); + // Get the current foreground window IntPtr currentForeground = GetForegroundWindow(); - // If it's already in foreground, we're done - if (currentForeground == hWnd) - return true; - // Get thread IDs uint currentThreadId = GetCurrentThreadId(); uint targetThreadId = GetWindowThreadProcessId(hWnd, out _); @@ -161,15 +171,28 @@ private bool BringWindowToForegroundWithFocus(IntPtr hWnd) AttachThreadInput(currentThreadId, foregroundThreadId, true); needsDetach = true; } + + // Also attach to the target thread if it's different + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, targetThreadId, true); + } // Try to set foreground window bool success = SetForegroundWindow(hWnd); + + // Force focus to the specific handle + SetFocus(hWnd); // Detach if we attached if (needsDetach) { AttachThreadInput(currentThreadId, foregroundThreadId, false); } + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, targetThreadId, false); + } // Give it a moment to process Thread.Sleep(50); diff --git a/FlowVision/lib/Plugins/MousePlugin.cs b/FlowVision/lib/Plugins/MousePlugin.cs index 87aa59d..048c3c7 100644 --- a/FlowVision/lib/Plugins/MousePlugin.cs +++ b/FlowVision/lib/Plugins/MousePlugin.cs @@ -18,20 +18,49 @@ internal class MousePlugin [DllImport("user32.dll", SetLastError = true)] private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + [DllImport("user32.dll", SetLastError = true)] + private static extern bool SetForegroundWindow(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr GetForegroundWindow(); + + [DllImport("user32.dll")] + private static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); + + [DllImport("kernel32.dll")] + private static extern uint GetCurrentThreadId(); + + [DllImport("user32.dll", SetLastError = true)] + private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId); + + [DllImport("user32.dll")] + private static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); + + [DllImport("user32.dll")] + private static extern IntPtr SetFocus(IntPtr hWnd); + + private const int SW_RESTORE = 9; private const uint MOUSEEVENTF_LEFTDOWN = 0x02; private const uint MOUSEEVENTF_LEFTUP = 0x04; private const uint MOUSEEVENTF_RIGHTDOWN = 0x08; private const uint MOUSEEVENTF_RIGHTUP = 0x10; - [Description("Clicks at the specified normalized bounding box coordinates on a specific window handle.")] - public async Task ClickOnWindow(string windowHandleString, double[] bBox, bool leftClick, int clickTimes) + [Description("Clicks at the specified normalized bounding box coordinates on a specific window handle. Box is [x1, y1, x2, y2].")] + public async Task ClickOnWindow(string windowHandleString, double x1, double y1, double x2, double y2, bool leftClick, int clickTimes) { // Log the plugin usage PluginLogger.LogPluginUsage("MousePlugin", "ClickOnWindow", - $"window={windowHandleString}, pos={string.Join(",", bBox)}, leftClick={leftClick}, times={clickTimes}"); + $"window={windowHandleString}, box=[{x1},{y1},{x2},{y2}], leftClick={leftClick}"); IntPtr windowHandle = new IntPtr(Convert.ToInt32(windowHandleString)); + // Ensure window is visible and focused + if (!BringWindowToForegroundWithFocus(windowHandle)) + { + PluginLogger.LogError("MousePlugin", "ClickOnWindow", "Failed to focus window"); + return false; + } + if (!GetWindowRect(windowHandle, out RECT rc)) { throw new InvalidOperationException("Failed to get window rectangle."); @@ -41,15 +70,16 @@ public async Task ClickOnWindow(string windowHandleString, double[] bBox, int windowHeight = rc.Bottom - rc.Top; // Calculate absolute position based on bounding box (normalized) - int x = rc.Left + (int)((bBox[0] + bBox[2]) / 2 * windowWidth); - int y = rc.Top + (int)((bBox[1] + bBox[3]) / 2 * windowHeight); + int x = rc.Left + (int)((x1 + x2) / 2 * windowWidth); + int y = rc.Top + (int)((y1 + y2) / 2 * windowHeight); if (!SetCursorPos(x, y)) { throw new InvalidOperationException("Failed to set cursor position."); } - await Task.Delay(100); + // Increased delay to allow UI to register hover state + await Task.Delay(200); for (int i = 0; i < clickTimes; i++) { @@ -66,7 +96,14 @@ public async Task ScrollOnWindow(string windowHandleString, int scrollAmou // Log the plugin usage PluginLogger.LogPluginUsage("MousePlugin", "ScrollOnWindow", $"window={windowHandleString}, amount={scrollAmount}"); + IntPtr windowHandle = new IntPtr(Convert.ToInt32(windowHandleString)); + + if (!BringWindowToForegroundWithFocus(windowHandle)) + { + return false; + } + if (!GetWindowRect(windowHandle, out RECT rc)) { throw new InvalidOperationException("Failed to get window rectangle."); @@ -77,7 +114,7 @@ public async Task ScrollOnWindow(string windowHandleString, int scrollAmou { throw new InvalidOperationException("Failed to set cursor position."); } - await Task.Delay(100); + await Task.Delay(200); mouse_event(0x0800, 0, 0, (uint)scrollAmount, UIntPtr.Zero); return true; } @@ -91,6 +128,51 @@ private void SimulateClick(int x, int y, bool leftClick) mouse_event(up, (uint)x, (uint)y, 0, UIntPtr.Zero); } + /// + /// Brings a window to the foreground and ensures it has focus using multiple techniques + /// + private bool BringWindowToForegroundWithFocus(IntPtr hWnd) + { + if (hWnd == IntPtr.Zero) return false; + + try + { + ShowWindow(hWnd, SW_RESTORE); + IntPtr currentForeground = GetForegroundWindow(); + if (currentForeground == hWnd) return true; + + uint currentThreadId = GetCurrentThreadId(); + uint targetThreadId = GetWindowThreadProcessId(hWnd, out _); + uint foregroundThreadId = GetWindowThreadProcessId(currentForeground, out _); + + bool needsDetach = false; + if (currentThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, foregroundThreadId, true); + needsDetach = true; + } + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, targetThreadId, true); + } + + bool success = SetForegroundWindow(hWnd); + SetFocus(hWnd); + + if (needsDetach) AttachThreadInput(currentThreadId, foregroundThreadId, false); + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + AttachThreadInput(currentThreadId, targetThreadId, false); + + System.Threading.Thread.Sleep(100); + return GetForegroundWindow() == hWnd; + } + catch (Exception ex) + { + PluginLogger.LogError("MousePlugin", "BringWindowToForegroundWithFocus", $"Error: {ex.Message}"); + return false; + } + } + [StructLayout(LayoutKind.Sequential)] private struct RECT { diff --git a/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs b/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs index b865766..24998c3 100644 --- a/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs +++ b/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs @@ -64,6 +64,45 @@ public async Task> CaptureWholeScreen() } } + [Description("Find specific text on screen using OCR. Useful if visual detection fails.")] + public async Task FindTextOnScreen(string searchText) + { + PluginLogger.LogPluginUsage("ScreenCaptureOmniParserPlugin", "FindTextOnScreen", searchText); + + // Capture whole screen + using (Bitmap screenshot = new Bitmap(SystemInformation.VirtualScreen.Width, SystemInformation.VirtualScreen.Height)) + { + using (Graphics gfx = Graphics.FromImage(screenshot)) + { + gfx.CopyFromScreen( + SystemInformation.VirtualScreen.X, + SystemInformation.VirtualScreen.Y, + 0, 0, + SystemInformation.VirtualScreen.Size, + CopyPixelOperation.SourceCopy); + } + + // Use OCR to find the text + var rect = await OcrHelper.FindTextLocationAsync(screenshot, searchText); + + if (rect.HasValue) + { + // Return in format compatible with ClickOnWindow (normalized bbox: x1, y1, x2, y2) + float width = screenshot.Width; + float height = screenshot.Height; + + float x1 = rect.Value.X / width; + float y1 = rect.Value.Y / height; + float x2 = (rect.Value.X + rect.Value.Width) / width; + float y2 = (rect.Value.Y + rect.Value.Height) / height; + + return $"Found '{searchText}' at normalized box: [{x1:F4}, {y1:F4}, {x2:F4}, {y2:F4}]"; + } + + return $"Text '{searchText}' not found on screen."; + } + } + /// /// Process screenshot with Simple OmniParser (KISS implementation) /// diff --git a/FlowVision/lib/Plugins/WindowSelectionPlugin.cs b/FlowVision/lib/Plugins/WindowSelectionPlugin.cs index 0a4560e..988bdfa 100644 --- a/FlowVision/lib/Plugins/WindowSelectionPlugin.cs +++ b/FlowVision/lib/Plugins/WindowSelectionPlugin.cs @@ -3,6 +3,7 @@ using System.ComponentModel; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Text; using System.Threading.Tasks; using FlowVision.lib.Classes; @@ -16,6 +17,52 @@ internal class WindowSelectionPlugin [DllImport("user32.dll", SetLastError = true)] private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + // --- New P/Invoke Definitions for Safe Enumeration --- + private delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam); + + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam); + + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool IsWindowVisible(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Auto)] + private static extern int GetWindowTextLength(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Auto)] + private static extern IntPtr SendMessageTimeout( + IntPtr hWnd, + uint Msg, + IntPtr wParam, + IntPtr lParam, + uint fuFlags, + uint uTimeout, + out IntPtr lpdwResult); + + [DllImport("user32.dll", SetLastError = true)] + private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); + + private const uint WM_GETTEXT = 0x000D; + private const uint WM_GETTEXTLENGTH = 0x000E; + private const uint SMTO_ABORTIFHUNG = 0x0002; + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr GetForegroundWindow(); + + // ------------------------------------------------------ + + [Description("Gets the handle and title of the currently active foreground window.")] + public string GetForegroundWindowInfo() + { + IntPtr hWnd = GetForegroundWindow(); + if (hWnd == IntPtr.Zero) return "No active window"; + + string title = GetWindowTitleSafe(hWnd); + return $"Handle: {hWnd}, Title: {title}"; + } + [Description("Used to set current handle as foreground")] public async Task ForegroundSelect(string handleString) { @@ -37,24 +84,91 @@ public string ListWindowHandles() PluginLogger.LogPluginUsage("WindowSelectionPlugin", "ListWindowHandles"); var windowList = new List(); - Process[] processes = Process.GetProcesses(); - foreach (Process p in processes) + + EnumWindows((hWnd, lParam) => { + // Filter invisible windows to reduce noise and hangs + if (!IsWindowVisible(hWnd)) + return true; // Continue enumeration + + // Safely get window title with timeout + string title = GetWindowTitleSafe(hWnd); + + // Skip untitled windows (often hidden helper windows) + if (string.IsNullOrWhiteSpace(title)) + return true; + + // Get process name + string processName = "Unknown"; try { - if (p.MainWindowHandle == IntPtr.Zero) - continue; - string item = $"Handle: {p.MainWindowHandle}, Title: {p.MainWindowTitle}, Process: {p.ProcessName}"; - windowList.Add(item); + GetWindowThreadProcessId(hWnd, out uint processId); + using (var p = Process.GetProcessById((int)processId)) + { + processName = p.ProcessName; + } } - catch - { - continue; - } - } + catch { /* Process might have exited or access denied */ } + + string item = $"Handle: {hWnd}, Title: {title}, Process: {processName}"; + windowList.Add(item); + + return true; // Continue enumeration + }, IntPtr.Zero); + return string.Join("\n", windowList); } + /// + /// Safely retrieves window title with a timeout to prevent hanging on unresponsive windows. + /// + private string GetWindowTitleSafe(IntPtr hWnd) + { + const int timeoutMs = 100; // Short timeout to ensure responsiveness + + // 1. Get text length with timeout + IntPtr result; + IntPtr ret = SendMessageTimeout( + hWnd, + WM_GETTEXTLENGTH, + IntPtr.Zero, + IntPtr.Zero, + SMTO_ABORTIFHUNG, + timeoutMs, + out result); + + if (ret == IntPtr.Zero) return string.Empty; // Failed or timed out + + int length = (int)result; + if (length == 0) return string.Empty; + + // 2. Get actual text with timeout + // Allocate unmanaged memory for the string buffer + // Length + 1 for null terminator, * 2 for Unicode characters + int bufferSize = (length + 1) * 2; + IntPtr buffer = Marshal.AllocHGlobal(bufferSize); + + try + { + ret = SendMessageTimeout( + hWnd, + WM_GETTEXT, + new IntPtr(length + 1), + buffer, + SMTO_ABORTIFHUNG, + timeoutMs, + out result); + + if (ret == IntPtr.Zero) return string.Empty; + + return Marshal.PtrToStringAuto(buffer); + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + /// /// Checks if the provided window handle is valid. ///