diff --git a/FlowVision/AIProviderConfigForm.cs b/FlowVision/AIProviderConfigForm.cs index 7adacc0..659ffce 100644 --- a/FlowVision/AIProviderConfigForm.cs +++ b/FlowVision/AIProviderConfigForm.cs @@ -21,6 +21,12 @@ public partial class AIProviderConfigForm : Form private TextBox azureDeploymentTextBox; private TextBox azureEndpointTextBox; private TextBox azureApiKeyTextBox; + private NumericUpDown azureTemperatureUpDown; // Added missing field + + // Gemini controls + private Panel geminiPanel; + private TextBox geminiApiKeyTextBox; + private TextBox geminiModelTextBox; // LM Studio controls private Panel lmStudioPanel; @@ -96,6 +102,7 @@ private void InitializeComponent() providerComboBox.Items.AddRange(new object[] { "Azure OpenAI (Cloud)", "LM Studio (Local)", + "Google Gemini", "GitHub Models (Free Tier)" }); providerComboBox.SelectedIndexChanged += ProviderComboBox_SelectedIndexChanged; @@ -182,6 +189,58 @@ private void InitializeComponent() // Create provider-specific panels CreateAzurePanel(); CreateLMStudioPanel(); + CreateGeminiPanel(); + } + + private void CreateGeminiPanel() + { + geminiPanel = new Panel + { + Location = new Point(0, 0), + Size = new Size(580, 300), + Visible = false + }; + + int y = 0; + int labelWidth = 150; + int controlWidth = 400; + + // API Key + var apiKeyLabel = new Label { Text = "Gemini API Key:", Location = new Point(0, y + 3), Width = labelWidth }; + geminiPanel.Controls.Add(apiKeyLabel); + geminiApiKeyTextBox = new TextBox + { + Location = new Point(labelWidth + 10, y), + Width = controlWidth, + UseSystemPasswordChar = true + }; + geminiPanel.Controls.Add(geminiApiKeyTextBox); + y += 35; + + // Model Name + var modelLabel = new Label { Text = "Model Name:", Location = new Point(0, y + 3), Width = labelWidth }; + geminiPanel.Controls.Add(modelLabel); + geminiModelTextBox = new TextBox + { + Location = new Point(labelWidth + 10, y), + Width = controlWidth, + Text = "gemini-1.5-flash" + }; + geminiPanel.Controls.Add(geminiModelTextBox); + y += 35; + + // Info + var helpLabel = new Label + { + Text = "Get your API Key from: https://aistudio.google.com/app/apikey\n" + + "Standard Endpoint: https://generativelanguage.googleapis.com/v1beta/openai/", + Location = new Point(0, y), + Size = new Size(550, 40), + ForeColor = Color.Gray + }; + geminiPanel.Controls.Add(helpLabel); + + configPanel.Controls.Add(geminiPanel); } private void CreateAzurePanel() @@ -223,6 +282,22 @@ private void CreateAzurePanel() azurePanel.Controls.Add(azureApiKeyTextBox); y += 35; + // Temperature (Newly added) + var tempLabel = new Label { Text = "Temperature:", Location = new Point(0, y + 3), Width = labelWidth }; + azurePanel.Controls.Add(tempLabel); + azureTemperatureUpDown = new NumericUpDown + { + Location = new Point(labelWidth + 10, y), + Width = 100, + Minimum = 0, + Maximum = 2, + DecimalPlaces = 2, + Increment = 0.1M, + Value = 0.7M + }; + azurePanel.Controls.Add(azureTemperatureUpDown); + y += 35; + // Help text var helpLabel = new Label { @@ -247,11 +322,12 @@ private void CreateLMStudioPanel() int y = 0; int labelWidth = 150; - int controlWidth = 400; + int controlWidth = 300; // Reduced width to make room for Auto-Detect button // Endpoint URL var endpointLabel = new Label { Text = "Server Endpoint:", Location = new Point(0, y + 3), Width = labelWidth }; lmStudioPanel.Controls.Add(endpointLabel); + lmStudioEndpointTextBox = new TextBox { Location = new Point(labelWidth + 10, y), @@ -259,6 +335,19 @@ private void CreateLMStudioPanel() Text = "http://localhost:1234/v1" }; lmStudioPanel.Controls.Add(lmStudioEndpointTextBox); + + // Auto-Detect Button + var autoDetectButton = new Button + { + Text = "Auto-Detect", + Location = new Point(labelWidth + controlWidth + 20, y - 1), + Width = 100, + Height = 23, + BackColor = Color.AliceBlue + }; + autoDetectButton.Click += AutoDetectButton_Click; + lmStudioPanel.Controls.Add(autoDetectButton); + y += 35; // Model Name @@ -267,7 +356,7 @@ private void CreateLMStudioPanel() lmStudioModelTextBox = new TextBox { Location = new Point(labelWidth + 10, y), - Width = controlWidth, + Width = 400, Text = "local-model" }; lmStudioPanel.Controls.Add(lmStudioModelTextBox); @@ -280,13 +369,24 @@ private void CreateLMStudioPanel() { Location = new Point(labelWidth + 10, y), Width = 100, - Minimum = 0, - Maximum = 2, - DecimalPlaces = 2, - Increment = 0.1M, - Value = 0.7M + Minimum = 1, // Fixed to 1 + Maximum = 1, // Fixed to 1 + DecimalPlaces = 1, // Fixed to 1 decimal place + Increment = 0.0M, // No increment as it's fixed + Value = 1.0M, // Fixed value + Enabled = false // Disable user input }; lmStudioPanel.Controls.Add(lmStudioTemperatureUpDown); + + // Add an info label for temperature + var tempInfoLabel = new Label + { + Text = "Fixed at 1.0 for LM Studio models.", + Location = new Point(labelWidth + 115, y + 3), + AutoSize = true, + ForeColor = Color.DarkGray + }; + lmStudioPanel.Controls.Add(tempInfoLabel); y += 35; // Max Tokens @@ -297,7 +397,7 @@ private void CreateLMStudioPanel() Location = new Point(labelWidth + 10, y), Width = 100, Minimum = 128, - Maximum = 32768, + Maximum = 1000000, // Increased to support large context models Increment = 128, Value = 2048 }; @@ -320,11 +420,122 @@ private void CreateLMStudioPanel() configPanel.Controls.Add(lmStudioPanel); } + private async void AutoDetectButton_Click(object sender, EventArgs e) + { + statusLabel.Text = "Searching for local AI server..."; + statusLabel.ForeColor = Color.Blue; + + string[] commonEndpoints = new[] + { + "http://localhost:1234/v1", // LM Studio default + "http://127.0.0.1:1234/v1", // LM Studio IP + "http://localhost:11434/v1", // Ollama default + "http://localhost:5000/v1", // LocalAI/Oobabooga default + "http://localhost:8080/v1" // Llama.cpp server + }; + + foreach (var endpoint in commonEndpoints) + { + try + { + var client = new OpenAI.OpenAIClient( + new System.ClientModel.ApiKeyCredential("any-key"), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri(endpoint) }); + + // Just try to list models or verify endpoint validity + // Note: OpenAI client doesn't have a simple 'ping', so we assume if Uri creation works + // and we can create a client, it's a candidate. A real ping would require an API call. + // Let's try a lightweight API call to verify. + + // Create a dummy chat client to test connectivity + var chatClient = client.GetChatClient("test-model"); + var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; + + var messages = new System.Collections.Generic.List + { + new Microsoft.Extensions.AI.ChatMessage(Microsoft.Extensions.AI.ChatRole.User, "hi") + }; + + // Set a short timeout for detection + // Note: .NET 4.8 async timeout cancellation is tricky, relying on fast failure + try { + // We don't actually wait for a full response, just seeing if connection is refused immediately + // If it hangs, it might be a valid server processing. + // For now, let's just assume the first valid URI that doesn't throw immediate connection refused is good. + await ichatClient.GetResponseAsync(messages); + } + catch (Exception ex) when (ex.Message.Contains("404") || !ex.Message.Contains("connection")) + { + // 404 means server is there but model not found - that's a success for finding the server! + // Not connection error means we reached something. + } + + lmStudioEndpointTextBox.Text = endpoint; + statusLabel.Text = $"✓ Found server at {endpoint}!"; + statusLabel.ForeColor = Color.Green; + return; + } + catch + { + // Continue to next endpoint + } + } + + statusLabel.Text = "✗ No local server found. Is LM Studio running?"; + statusLabel.ForeColor = Color.Red; + } + + private async Task TestLMStudioConnection() + { + try + { + if (!Uri.TryCreate(lmStudioEndpointTextBox.Text, UriKind.Absolute, out Uri result)) + { + throw new UriFormatException("Invalid Endpoint URL format. It should look like: http://localhost:1234/v1"); + } + + var client = new OpenAI.OpenAIClient( + new System.ClientModel.ApiKeyCredential("lm-studio"), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri(lmStudioEndpointTextBox.Text) }); + + var chatClient = client.GetChatClient(lmStudioModelTextBox.Text); + + var messages = new System.Collections.Generic.List + { + new Microsoft.Extensions.AI.ChatMessage( + Microsoft.Extensions.AI.ChatRole.User, + "Say 'test' in one word") + }; + + // Cast to IChatClient - can't use AsIChatClient on OpenAI.ChatClient directly in .NET 4.8 + var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; + var response = await ichatClient.GetResponseAsync(messages); + + statusLabel.Text = "✓ LM Studio connection successful!"; + statusLabel.ForeColor = Color.Green; + } + catch (Exception ex) + { + if (ex.Message.Contains("Connection refused") || ex.Message.Contains("No connection")) + { + statusLabel.Text = "✗ Cannot connect. Is LM Studio running with server started?"; + } + else + { + statusLabel.Text = $"✗ Connection failed: {ex.Message}"; + } + statusLabel.ForeColor = Color.Red; + } + } + private void LoadConfiguration() { // Try to load existing configuration to determine current provider var lmConfig = LMStudioConfig.LoadConfig(); var azureConfig = APIConfig.LoadConfig(currentModel); + + // Load global tool config for syncing temperature + var toolConfig = ToolConfig.LoadConfig("toolsconfig"); if (lmConfig.Enabled) { @@ -335,7 +546,23 @@ private void LoadConfiguration() lmStudioEndpointTextBox.Text = lmConfig.EndpointURL; lmStudioModelTextBox.Text = lmConfig.ModelName; lmStudioTemperatureUpDown.Value = (decimal)lmConfig.Temperature; - lmStudioMaxTokensUpDown.Value = lmConfig.MaxTokens; + + // Safely set max tokens + if (lmConfig.MaxTokens < lmStudioMaxTokensUpDown.Minimum) + lmStudioMaxTokensUpDown.Value = lmStudioMaxTokensUpDown.Minimum; + else if (lmConfig.MaxTokens > lmStudioMaxTokensUpDown.Maximum) + lmStudioMaxTokensUpDown.Value = lmStudioMaxTokensUpDown.Maximum; + else + lmStudioMaxTokensUpDown.Value = lmConfig.MaxTokens; + } + else if (azureConfig.ProviderType == "Gemini") + { + // Gemini is enabled + providerComboBox.SelectedIndex = 2; // Google Gemini + enableProviderCheckBox.Checked = true; + + geminiApiKeyTextBox.Text = azureConfig.APIKey; + geminiModelTextBox.Text = azureConfig.DeploymentName; } else { @@ -346,6 +573,9 @@ private void LoadConfiguration() azureDeploymentTextBox.Text = azureConfig.DeploymentName; azureEndpointTextBox.Text = azureConfig.EndpointURL; azureApiKeyTextBox.Text = azureConfig.APIKey; + + // Init Azure temperature control (use ToolConfig temperature if available, else default) + azureTemperatureUpDown.Value = (decimal)toolConfig.Temperature; } } @@ -354,6 +584,7 @@ private void ProviderComboBox_SelectedIndexChanged(object sender, EventArgs e) // Hide all panels azurePanel.Visible = false; lmStudioPanel.Visible = false; + geminiPanel.Visible = false; // Show selected panel switch (providerComboBox.SelectedIndex) @@ -364,7 +595,10 @@ private void ProviderComboBox_SelectedIndexChanged(object sender, EventArgs e) case 1: // LM Studio lmStudioPanel.Visible = true; break; - case 2: // GitHub Models + case 2: // Google Gemini + geminiPanel.Visible = true; + break; + case 3: // GitHub Models MessageBox.Show("GitHub Models support coming soon!\nFor now, configure as Azure OpenAI with GitHub endpoint.", "Coming Soon", MessageBoxButtons.OK, MessageBoxIcon.Information); providerComboBox.SelectedIndex = 0; @@ -384,6 +618,20 @@ private async void TestButton_Click(object sender, EventArgs e) { await TestLMStudioConnection(); } + else if (providerComboBox.SelectedIndex == 2) // Gemini + { + var client = new OpenAI.OpenAIClient( + new System.ClientModel.ApiKeyCredential(geminiApiKeyTextBox.Text), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri("https://generativelanguage.googleapis.com/v1beta/openai/") } + ); + var chatClient = client.GetChatClient(geminiModelTextBox.Text); + var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; + await ichatClient.GetResponseAsync(new System.Collections.Generic.List{ + new Microsoft.Extensions.AI.ChatMessage(Microsoft.Extensions.AI.ChatRole.User, "hi") + }); + statusLabel.Text = "✓ Gemini connection successful!"; + statusLabel.ForeColor = Color.Green; + } else // Azure OpenAI { await TestAzureConnection(); @@ -431,44 +679,6 @@ private async Task TestAzureConnection() } } - private async Task TestLMStudioConnection() - { - try - { - var client = new OpenAI.OpenAIClient( - new System.ClientModel.ApiKeyCredential("lm-studio"), - new OpenAI.OpenAIClientOptions { Endpoint = new Uri(lmStudioEndpointTextBox.Text) }); - - var chatClient = client.GetChatClient(lmStudioModelTextBox.Text); - - var messages = new System.Collections.Generic.List - { - new Microsoft.Extensions.AI.ChatMessage( - Microsoft.Extensions.AI.ChatRole.User, - "Say 'test' in one word") - }; - - // Cast to IChatClient - can't use AsIChatClient on OpenAI.ChatClient directly in .NET 4.8 - var ichatClient = (Microsoft.Extensions.AI.IChatClient)(object)chatClient; - var response = await ichatClient.GetResponseAsync(messages); - - statusLabel.Text = "✓ LM Studio connection successful!"; - statusLabel.ForeColor = Color.Green; - } - catch (Exception ex) - { - if (ex.Message.Contains("Connection refused") || ex.Message.Contains("No connection")) - { - statusLabel.Text = "✗ Cannot connect. Is LM Studio running with server started?"; - } - else - { - statusLabel.Text = $"✗ Connection failed: {ex.Message}"; - } - statusLabel.ForeColor = Color.Red; - } - } - private void SaveButton_Click(object sender, EventArgs e) { try @@ -477,6 +687,10 @@ private void SaveButton_Click(object sender, EventArgs e) { SaveLMStudioConfig(); } + else if (providerComboBox.SelectedIndex == 2) // Gemini + { + SaveGeminiConfig(); + } else // Azure OpenAI { SaveAzureConfig(); @@ -500,6 +714,23 @@ private void SaveButton_Click(object sender, EventArgs e) } } + private void SaveGeminiConfig() + { + var config = new APIConfig + { + DeploymentName = geminiModelTextBox.Text, + EndpointURL = "https://generativelanguage.googleapis.com/v1beta/openai/", + APIKey = geminiApiKeyTextBox.Text, + ProviderType = "Gemini" + }; + config.SaveConfig(currentModel); + + // Disable LM Studio + var lmConfig = LMStudioConfig.LoadConfig(); + lmConfig.Enabled = false; + lmConfig.SaveConfig(); + } + private void SaveAzureConfig() { var config = new APIConfig @@ -515,10 +746,20 @@ private void SaveAzureConfig() var lmConfig = LMStudioConfig.LoadConfig(); lmConfig.Enabled = false; lmConfig.SaveConfig(); + + // Sync global tool config temperature + var toolConfig = ToolConfig.LoadConfig("toolsconfig"); + toolConfig.Temperature = (double)azureTemperatureUpDown.Value; + toolConfig.SaveConfig("toolsconfig"); } private void SaveLMStudioConfig() { + if (!Uri.TryCreate(lmStudioEndpointTextBox.Text, UriKind.Absolute, out Uri result)) + { + throw new UriFormatException("Invalid Endpoint URL format. It should look like: http://localhost:1234/v1"); + } + var config = new LMStudioConfig { EndpointURL = lmStudioEndpointTextBox.Text, @@ -538,6 +779,11 @@ private void SaveLMStudioConfig() ProviderType = "LMStudio" }; azureConfig.SaveConfig(currentModel); + + // Sync global tool config temperature + var toolConfig = ToolConfig.LoadConfig("toolsconfig"); + toolConfig.Temperature = (double)lmStudioTemperatureUpDown.Value; + toolConfig.SaveConfig("toolsconfig"); } } } diff --git a/FlowVision/FlowVision.csproj b/FlowVision/FlowVision.csproj index f9ee151..d4b1db2 100644 --- a/FlowVision/FlowVision.csproj +++ b/FlowVision/FlowVision.csproj @@ -211,6 +211,7 @@ + @@ -250,7 +251,9 @@ UserControl + + @@ -301,6 +304,7 @@ + @@ -327,13 +331,6 @@ - - - - - - - diff --git a/FlowVision/OmniParserForm.Designer.cs b/FlowVision/OmniParserForm.Designer.cs index 6250925..1110bea 100644 --- a/FlowVision/OmniParserForm.Designer.cs +++ b/FlowVision/OmniParserForm.Designer.cs @@ -28,52 +28,57 @@ protected override void Dispose(bool disposing) /// private void InitializeComponent() { - this.omniParserServerURL = new System.Windows.Forms.TextBox(); this.label1 = new System.Windows.Forms.Label(); - this.saveButton = new System.Windows.Forms.Button(); + this.statusLabel = new System.Windows.Forms.Label(); + this.infoLabel = new System.Windows.Forms.Label(); this.SuspendLayout(); // - // omniParserServerURL - // - this.omniParserServerURL.Font = new System.Drawing.Font("Comic Sans MS", 12F); - this.omniParserServerURL.Location = new System.Drawing.Point(139, 7); - this.omniParserServerURL.Name = "omniParserServerURL"; - this.omniParserServerURL.Size = new System.Drawing.Size(355, 30); - this.omniParserServerURL.TabIndex = 0; - // // label1 // this.label1.AutoSize = true; - this.label1.Font = new System.Drawing.Font("Comic Sans MS", 11.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); - this.label1.Location = new System.Drawing.Point(12, 12); + this.label1.Font = new System.Drawing.Font("Segoe UI", 12F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, ((byte)(0))); + this.label1.Location = new System.Drawing.Point(20, 20); this.label1.Name = "label1"; - this.label1.Size = new System.Drawing.Size(121, 20); + this.label1.Size = new System.Drawing.Size(154, 21); this.label1.TabIndex = 1; - this.label1.Text = "OmniParser URL"; + this.label1.Text = "OmniParser Status:"; + // + // statusLabel + // + this.statusLabel.AutoSize = true; + this.statusLabel.Font = new System.Drawing.Font("Segoe UI", 12F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); + this.statusLabel.ForeColor = System.Drawing.Color.Green; + this.statusLabel.Location = new System.Drawing.Point(180, 20); + this.statusLabel.Name = "statusLabel"; + this.statusLabel.Size = new System.Drawing.Size(196, 21); + this.statusLabel.TabIndex = 3; + this.statusLabel.Text = "✓ Embedded Mode Active"; // - // saveButton + // infoLabel // - this.saveButton.Location = new System.Drawing.Point(162, 41); - this.saveButton.Name = "saveButton"; - this.saveButton.Size = new System.Drawing.Size(140, 23); - this.saveButton.TabIndex = 2; - this.saveButton.Text = "Save"; - this.saveButton.UseVisualStyleBackColor = true; - this.saveButton.Click += new System.EventHandler(this.saveButton_Click); + this.infoLabel.AutoSize = true; + this.infoLabel.Font = new System.Drawing.Font("Segoe UI", 9.75F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); + this.infoLabel.ForeColor = System.Drawing.Color.DimGray; + this.infoLabel.Location = new System.Drawing.Point(21, 55); + this.infoLabel.Name = "infoLabel"; + this.infoLabel.Size = new System.Drawing.Size(380, 34); + this.infoLabel.TabIndex = 4; + this.infoLabel.Text = "FlowVision is using the internal ONNX detection model.\r\nNo external server or configuration is required."; // // OmniParserForm // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.ClientSize = new System.Drawing.Size(506, 76); - this.Controls.Add(this.saveButton); + this.ClientSize = new System.Drawing.Size(440, 110); + this.Controls.Add(this.infoLabel); + this.Controls.Add(this.statusLabel); this.Controls.Add(this.label1); - this.Controls.Add(this.omniParserServerURL); this.MaximizeBox = false; this.MinimizeBox = false; this.Name = "OmniParserForm"; this.ShowIcon = false; - this.Text = "OmniParserServer Config"; + this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen; + this.Text = "OmniParser Configuration"; this.Load += new System.EventHandler(this.OmniParserForm_Load); this.ResumeLayout(false); this.PerformLayout(); @@ -82,8 +87,8 @@ private void InitializeComponent() #endregion - private System.Windows.Forms.TextBox omniParserServerURL; private System.Windows.Forms.Label label1; - private System.Windows.Forms.Button saveButton; + private System.Windows.Forms.Label statusLabel; + private System.Windows.Forms.Label infoLabel; } } \ No newline at end of file diff --git a/FlowVision/OmniParserForm.cs b/FlowVision/OmniParserForm.cs index c9aeec0..aaa2a08 100644 --- a/FlowVision/OmniParserForm.cs +++ b/FlowVision/OmniParserForm.cs @@ -1,11 +1,4 @@ using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Data; -using System.Drawing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows.Forms; using FlowVision.lib.Classes; @@ -13,35 +6,14 @@ namespace FlowVision { public partial class OmniParserForm : Form { - private OmniParserConfig _config; - public OmniParserForm() { InitializeComponent(); - _config = OmniParserConfig.LoadConfig(); - } - - private void saveButton_Click(object sender, EventArgs e) - { - string url = omniParserServerURL.Text.Trim(); - if (string.IsNullOrEmpty(url)) - { - MessageBox.Show("Please enter a valid URL."); - return; - } - - // Save the URL to the config file - _config.ServerURL = url; - _config.SaveConfig(); - - // Optionally, you can close the form after saving - this.Close(); } private void OmniParserForm_Load(object sender, EventArgs e) { - // Load the URL from the config file - omniParserServerURL.Text = _config.ServerURL; + // Form is now just an informational status display } } } diff --git a/FlowVision/Program.cs b/FlowVision/Program.cs index 84102c4..d7f93bb 100644 --- a/FlowVision/Program.cs +++ b/FlowVision/Program.cs @@ -14,6 +14,20 @@ static class Program [STAThread] static void Main() { + // Preload the ONNX model on a background thread so it's ready when needed + Task.Run(() => + { + try + { + // Accessing the Instance property triggers the model loading + var parser = FlowVision.lib.Classes.SimpleOmniParser.Instance; + } + catch + { + // Ignore startup errors - they will be caught/logged when the user actually tries to use it + } + }); + Application.EnableVisualStyles(); Application.SetCompatibleTextRenderingDefault(false); Application.Run(new Form1()); diff --git a/FlowVision/lib/Classes/LMStudioConfig.cs b/FlowVision/lib/Classes/LMStudioConfig.cs index fb196f0..c03b76e 100644 --- a/FlowVision/lib/Classes/LMStudioConfig.cs +++ b/FlowVision/lib/Classes/LMStudioConfig.cs @@ -4,77 +4,59 @@ namespace FlowVision.lib.Classes { - /// - /// Configuration for LM Studio local AI integration - /// public class LMStudioConfig { - /// - /// LM Studio server endpoint (default: http://localhost:1234/v1) - /// public string EndpointURL { get; set; } = "http://localhost:1234/v1"; - - /// - /// Model name (e.g., "gpt-3.5-turbo", "local-model", or whatever LM Studio shows) - /// Can be left as default - LM Studio typically uses the loaded model automatically - /// public string ModelName { get; set; } = "local-model"; - - /// - /// API key (LM Studio doesn't require one, but field kept for compatibility) - /// Use "lm-studio" or "not-needed" as placeholder - /// - public string APIKey { get; set; } = "lm-studio"; - - /// - /// Whether to use LM Studio or fall back to Azure - /// - public bool Enabled { get; set; } = false; - - /// - /// Temperature setting for local model - /// public double Temperature { get; set; } = 0.7; - - /// - /// Max tokens for completion - /// public int MaxTokens { get; set; } = 2048; - - /// - /// Timeout in seconds for LM Studio requests - /// - public int TimeoutSeconds { get; set; } = 300; + public bool Enabled { get; set; } = false; + public string APIKey { get; set; } = "lm-studio"; // OpenAI client requires a key even if local + public int TimeoutSeconds { get; set; } = 120; - private static string ConfigFilePath() - { - return Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), - "FlowVision", - "lmstudioconfig.json"); - } + // Track if the config was successfully loaded from disk + [System.Text.Json.Serialization.JsonIgnore] + public bool IsValid { get; set; } = true; + + private static string ConfigFilePath => Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "FlowVision", + "lmstudioconfig.json"); public static LMStudioConfig LoadConfig() { try { - Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath())); + Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath)); - if (File.Exists(ConfigFilePath())) + if (File.Exists(ConfigFilePath)) { - string jsonContent = File.ReadAllText(ConfigFilePath()); - if (!string.IsNullOrWhiteSpace(jsonContent)) + string jsonContent = File.ReadAllText(ConfigFilePath); + // Basic validation for empty file + if (string.IsNullOrWhiteSpace(jsonContent)) { - var config = JsonSerializer.Deserialize(jsonContent); - return config ?? new LMStudioConfig(); + return new LMStudioConfig { IsValid = false }; + } + + var config = JsonSerializer.Deserialize(jsonContent); + if (config != null) + { + config.IsValid = true; + return config; } } } catch (Exception ex) { - Console.WriteLine($"Error loading LM Studio config: {ex.Message}"); + // Return a config marked as invalid so the UI can warn the user + return new LMStudioConfig + { + Enabled = false, + IsValid = false + }; } + // Return default if file doesn't exist return new LMStudioConfig(); } @@ -82,15 +64,16 @@ public void SaveConfig() { try { - Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath())); + Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath)); var options = new JsonSerializerOptions { WriteIndented = true }; string jsonContent = JsonSerializer.Serialize(this, options); - File.WriteAllText(ConfigFilePath(), jsonContent); + File.WriteAllText(ConfigFilePath, jsonContent); } catch (Exception ex) { Console.WriteLine($"Error saving LM Studio config: {ex.Message}"); + throw; // Re-throw so the UI can show the error } } } diff --git a/FlowVision/lib/Classes/OcrHelper.cs b/FlowVision/lib/Classes/OcrHelper.cs index 6b9a308..5ae1a6e 100644 --- a/FlowVision/lib/Classes/OcrHelper.cs +++ b/FlowVision/lib/Classes/OcrHelper.cs @@ -82,6 +82,54 @@ private static void Initialize() } } + /// + /// Search for specific text in an image and return its bounding box. + /// Returns the first match found. + /// + public static async Task FindTextLocationAsync(Bitmap image, string searchText) + { + if (!_isAvailable || _engine == null || string.IsNullOrWhiteSpace(searchText)) + return null; + + return await Task.Run(() => + { + try + { + lock (_lock) + { + using (var pix = PixConverter.ToPix(image)) + using (var page = _engine.Process(pix)) + using (var iter = page.GetIterator()) + { + iter.Begin(); + do + { + // Get text at current iterator level (Word) + string currentText = iter.GetText(PageIteratorLevel.Word)?.Trim(); + + // Simple case-insensitive match + if (!string.IsNullOrWhiteSpace(currentText) && + currentText.Equals(searchText, StringComparison.OrdinalIgnoreCase)) + { + // Found exact match! Get bounding box. + if (iter.TryGetBoundingBox(PageIteratorLevel.Word, out var rect)) + { + return (Rectangle?)new Rectangle(rect.X1, rect.Y1, rect.Width, rect.Height); + } + } + } while (iter.Next(PageIteratorLevel.Word)); + } + } + return (Rectangle?)null; + } + catch (Exception ex) + { + PluginLogger.LogError("OcrHelper", "FindTextLocationAsync", $"OCR search failed: {ex.Message}"); + return (Rectangle?)null; + } + }); + } + /// /// Extract text from a bitmap image /// diff --git a/FlowVision/lib/Classes/ToolConfig.cs b/FlowVision/lib/Classes/ToolConfig.cs index 27ea735..42d0450 100644 --- a/FlowVision/lib/Classes/ToolConfig.cs +++ b/FlowVision/lib/Classes/ToolConfig.cs @@ -12,12 +12,14 @@ public class ToolConfig public bool EnableKeyboardPlugin { get; set; } = true; public bool EnableMousePlugin { get; set; } = false; // Changed default to false public bool EnableWindowSelectionPlugin { get; set; } = true; // Added WindowSelectionPlugin + public bool EnableClipboardPlugin { get; set; } = true; // Added ClipboardPlugin + public bool EnableFileSystemPlugin { get; set; } = true; // Added FileSystemPlugin public bool EnableSpeechRecognition { get; set; } = true; // Added Speech Recognition option public string SpeechRecognitionLanguage { get; set; } = "en-US"; // Default language public string VoiceCommandPhrase { get; set; } = "send message"; // Default voice command phrase public bool EnableVoiceCommands { get; set; } = true; // Enable voice commands feature public bool EnablePluginLogging { get; set; } = true; - public double Temperature { get; set; } = 0.2; + public double Temperature { get; set; } = 1.0; public bool AutoInvokeKernelFunctions { get; set; } = true; public bool RetainChatHistory { get; set; } = true; public bool EnableMultiAgentMode { get; set; } = false; // Changed default to false diff --git a/FlowVision/lib/Classes/ai/AIClientFactory.cs b/FlowVision/lib/Classes/ai/AIClientFactory.cs new file mode 100644 index 0000000..2e9e85b --- /dev/null +++ b/FlowVision/lib/Classes/ai/AIClientFactory.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.Extensions.AI; +using Azure.AI.OpenAI; +using Azure; +using OpenAI; + +namespace FlowVision.lib.Classes.ai +{ + public static class AIClientFactory + { + public static IChatClient CreateClient(APIConfig config) + { + if (config == null) + throw new ArgumentNullException(nameof(config)); + + switch (config.ProviderType?.ToLowerInvariant()) + { + case "gemini": + // Use standard OpenAI client pointing to Google's endpoint + // Endpoint format: https://generativelanguage.googleapis.com/v1beta/openai/ + var geminiClient = new OpenAIClient( + new System.ClientModel.ApiKeyCredential(config.APIKey), + new OpenAIClientOptions { Endpoint = new Uri(config.EndpointURL) } + ); + return geminiClient.GetChatClient(config.DeploymentName).AsIChatClient(); + + case "lmstudio": + case "openai": // Generic OpenAI compatible + var openAIClient = new OpenAIClient( + new System.ClientModel.ApiKeyCredential(config.APIKey), + new OpenAIClientOptions { Endpoint = new Uri(config.EndpointURL) } + ); + return openAIClient.GetChatClient(config.DeploymentName).AsIChatClient(); + + case "azureopenai": + default: + // Default to Azure OpenAI + var azureClient = new AzureOpenAIClient( + new Uri(config.EndpointURL), + new AzureKeyCredential(config.APIKey) + ); + return azureClient.GetChatClient(config.DeploymentName).AsIChatClient(); + } + } + } +} diff --git a/FlowVision/lib/Classes/ai/Actioner.cs b/FlowVision/lib/Classes/ai/Actioner.cs index cc4a075..7381454 100644 --- a/FlowVision/lib/Classes/ai/Actioner.cs +++ b/FlowVision/lib/Classes/ai/Actioner.cs @@ -1,4 +1,10 @@ -using System; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -7,10 +13,8 @@ using System.Windows.Forms; using FlowVision.lib.Plugins; using Microsoft.Extensions.AI; -using Azure.AI.OpenAI; -using Azure; -using OpenAI; -using ChatMessage = Microsoft.Extensions.AI.ChatMessage; +using FlowVision.lib.Classes.ai; +using FlowVision; // Required for Form1 namespace FlowVision.lib.Classes { @@ -128,9 +132,8 @@ public async Task ExecuteAction(string actionPrompt) return "Error: Actioner model not configured"; } - // Create Azure OpenAI chat client with IChatClient interface - var azureClient = new AzureOpenAIClient(new Uri(config.EndpointURL), new AzureKeyCredential(config.APIKey)); - IChatClient baseChatClient = azureClient.GetChatClient(config.DeploymentName).AsIChatClient(); + // Use the Factory to create the client based on ProviderType + IChatClient baseChatClient = AIClientFactory.CreateClient(config); // Collect tools based on configuration var tools = new List(); @@ -175,6 +178,16 @@ public async Task ExecuteAction(string actionPrompt) tools.AddRange(PluginToolExtractor.ExtractTools(new RemoteControlPlugin())); } + if (toolConfig.EnableClipboardPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new ClipboardPlugin())); + } + + if (toolConfig.EnableFileSystemPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new FileSystemPlugin())); + } + // Configure chat options with tools var chatOptions = new ChatOptions { @@ -193,13 +206,20 @@ public async Task ExecuteAction(string actionPrompt) // Process the response with streaming var responseBuilder = new StringBuilder(); - await foreach (var update in actionerChat.GetStreamingResponseAsync(actionerHistory, chatOptions)) + try { - if (update.Text != null) + await foreach (var update in actionerChat.GetStreamingResponseAsync(actionerHistory, chatOptions)) { - responseBuilder.Append(update.Text); + if (update.Text != null) + { + responseBuilder.Append(update.Text); + } } } + catch (Exception ex) when (ex.Message.Contains("Unknown ChatFinishReason") || ex.Message.Contains("function_call_filter")) + { + PluginLogger.LogInfo("Actioner", "ExecuteAction", $"Ignored known SDK finish reason error: {ex.Message}"); + } // Task completed successfully PluginLogger.NotifyTaskComplete("Action Execution", true); @@ -222,7 +242,7 @@ public async Task ExecuteAction(string actionPrompt) } } - internal void SetChatHistory(List chatHistory) + internal void SetChatHistory(System.Collections.Generic.List chatHistory) { actionerHistory.Clear(); foreach (var message in chatHistory) diff --git a/FlowVision/lib/Classes/ai/LMStudioActioner.cs b/FlowVision/lib/Classes/ai/LMStudioActioner.cs index 3a3c590..d5b1f89 100644 --- a/FlowVision/lib/Classes/ai/LMStudioActioner.cs +++ b/FlowVision/lib/Classes/ai/LMStudioActioner.cs @@ -3,12 +3,18 @@ using System.Linq; using System.Reflection; using System.Text; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; using System.Threading.Tasks; using System.Windows.Forms; using FlowVision.lib.Plugins; using Microsoft.Extensions.AI; using OpenAI; using ChatMessage = Microsoft.Extensions.AI.ChatMessage; +using FlowVision; // Required for Form1 namespace FlowVision.lib.Classes { @@ -107,6 +113,20 @@ 2. After CaptureWholeScreen(), you MUST continue to actually click/type/interact return "Error: LM Studio endpoint not configured. Default is http://localhost:1234/v1"; } + // Check for invalid config load + if (!lmStudioConfig.IsValid) + { + PluginLogger.NotifyTaskComplete("LM Studio Action Execution", false); + return "Error: LM Studio configuration file is corrupt. Please go to settings and re-save the configuration."; + } + + // Validate URI format before attempting to create client + if (!Uri.TryCreate(lmStudioConfig.EndpointURL, UriKind.Absolute, out _)) + { + PluginLogger.NotifyTaskComplete("LM Studio Action Execution", false); + return $"Error: Invalid Endpoint URL '{lmStudioConfig.EndpointURL}'. Please correct it in settings."; + } + // Create OpenAI client pointing to LM Studio // LM Studio provides an OpenAI-compatible API var openAIClient = new OpenAIClient(new System.ClientModel.ApiKeyCredential(lmStudioConfig.APIKey), new OpenAIClientOptions @@ -164,7 +184,7 @@ 2. After CaptureWholeScreen(), you MUST continue to actually click/type/interact // Configure chat options with tools var chatOptions = new ChatOptions { - Temperature = (float)lmStudioConfig.Temperature, + Temperature = 1.0f, // Fixed at 1.0 for LM Studio models MaxOutputTokens = lmStudioConfig.MaxTokens, Tools = tools }; @@ -242,7 +262,7 @@ 2. After CaptureWholeScreen(), you MUST continue to actually click/type/interact } } - public void SetChatHistory(List chatHistory) + public void SetChatHistory(System.Collections.Generic.List chatHistory) { actionerHistory.Clear(); foreach (var message in chatHistory) diff --git a/FlowVision/lib/Classes/ai/MultiAgentActioner.cs b/FlowVision/lib/Classes/ai/MultiAgentActioner.cs index 417e4bf..35cb0d3 100644 --- a/FlowVision/lib/Classes/ai/MultiAgentActioner.cs +++ b/FlowVision/lib/Classes/ai/MultiAgentActioner.cs @@ -6,9 +6,10 @@ using System.Windows.Forms; using FlowVision.lib.Plugins; using Microsoft.Extensions.AI; -using Azure.AI.OpenAI; -using Azure; -using ChatMessage = Microsoft.Extensions.AI.ChatMessage; +using FlowVision.lib.Classes.ai; +using Azure.AI.OpenAI; // Needed for some types if referenced, but Factory returns IChatClient +using Azure; // Needed for AzureKeyCredential if strictly typed, but Factory handles it. +using FlowVision; // Required for Form1 namespace FlowVision.lib.Classes { @@ -23,7 +24,7 @@ public class MultiAgentActioner private List coordinatorHistory; private List plannerHistory; private List actionerHistory; - private AgentCoordinator agentCoordinator; + private AgentCoordinator agentCoordinator; // Configuration constants private const string TOOL_CONFIG = "toolsconfig"; @@ -144,17 +145,12 @@ public async Task ExecuteAction(string actionPrompt) return "Error: Actioner model not configured"; } - // Setup coordinator chat client (no tools, only coordination capabilities) - var coordinatorAzureClient = new AzureOpenAIClient(new Uri(coordinatorConfig.EndpointURL), new AzureKeyCredential(coordinatorConfig.APIKey)); - coordinatorChat = coordinatorAzureClient.GetChatClient(coordinatorConfig.DeploymentName).AsIChatClient(); - - // Setup planner chat client (no tools, only planning capabilities) - var plannerAzureClient = new AzureOpenAIClient(new Uri(plannerConfig.EndpointURL), new AzureKeyCredential(plannerConfig.APIKey)); - plannerChat = plannerAzureClient.GetChatClient(plannerConfig.DeploymentName).AsIChatClient(); - - // Setup actioner chat client with all tools - var actionerAzureClient = new AzureOpenAIClient(new Uri(actionerConfig.EndpointURL), new AzureKeyCredential(actionerConfig.APIKey)); - IChatClient actionerChatBase = actionerAzureClient.GetChatClient(actionerConfig.DeploymentName).AsIChatClient(); + // Setup clients using the Factory (supports Azure, Gemini, etc.) + coordinatorChat = AIClientFactory.CreateClient(coordinatorConfig); + plannerChat = AIClientFactory.CreateClient(plannerConfig); + + // Setup actioner client base + IChatClient actionerChatBase = AIClientFactory.CreateClient(actionerConfig); // Collect tools based on configuration var tools = new List(); @@ -199,6 +195,16 @@ public async Task ExecuteAction(string actionPrompt) tools.AddRange(PluginToolExtractor.ExtractTools(new RemoteControlPlugin())); } + if (toolConfig.EnableClipboardPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new ClipboardPlugin())); + } + + if (toolConfig.EnableFileSystemPlugin) + { + tools.AddRange(PluginToolExtractor.ExtractTools(new FileSystemPlugin())); + } + // Setup actioner with function invocation using builder pattern actionerChat = new ChatClientBuilder(actionerChatBase).UseFunctionInvocation().Build(); @@ -209,12 +215,12 @@ public async Task ExecuteAction(string actionPrompt) var coordinatorOptions = new ChatOptions { - Temperature = 0.2f + Temperature = (float)toolConfig.Temperature }; var plannerOptions = new ChatOptions { - Temperature = 0.2f + Temperature = (float)toolConfig.Temperature }; var actionerOptions = new ChatOptions @@ -252,13 +258,19 @@ public async Task ExecuteAction(string actionPrompt) int currentIteration = 0; string finalResult = ""; List executionResults = new List(); + + // Novel Feature: Focus Tracking + // Keep track of window focus to detect popups (like "Save As") automatically + var windowTracker = new WindowSelectionPlugin(); while (!isComplete && currentIteration < maxIterations) { currentIteration++; PluginLogger.LogPluginUsage($"⚙️ Step {currentIteration}/{maxIterations}"); - + // Capture pre-action state + string preActionWindow = windowTracker.GetForegroundWindowInfo(); + // Ask actioner to perform the current step with clearer instructions actionerHistory.Add(new ChatMessage(ChatRole.User, $"Execute this step:\n\n{plan}\n\n" + @@ -281,6 +293,28 @@ public async Task ExecuteAction(string actionPrompt) // Get actioner response with tools string executionResult = await GetAgentResponseAsync(actionerChat, actionerHistory, actionerOptions); + // Capture post-action state + string postActionWindow = windowTracker.GetForegroundWindowInfo(); + + // FIX 1: Handle empty execution results (common with successful shell commands) + if (string.IsNullOrWhiteSpace(executionResult)) + { + executionResult = "The command executed successfully with no output."; + } + + // Novel Feature: Inject Focus Change Alert + // If the active window changed (e.g. "Save As" dialog popped up), explicitly tell the Planner. + if (preActionWindow != postActionWindow) + { + string alert = $"\n\n[SYSTEM ALERT]: Active window focus changed!\n" + + $"Previous: {preActionWindow}\n" + + $"Current: {postActionWindow}\n" + + $"Use the new Handle ({postActionWindow.Split(',')[0]}) for subsequent interactions."; + + executionResult += alert; + PluginLogger.LogInfo("MultiAgentActioner", "ExecuteAction", "Detected window focus change, alerting Planner."); + } + // Store the execution result for the final response executionResults.Add(executionResult); @@ -290,6 +324,10 @@ public async Task ExecuteAction(string actionPrompt) "EXECUTION_RESPONSE", executionResult); + // FIX 2: Manage context window to prevent token overflow + ManageContextWindow(actionerHistory, 10); + ManageContextWindow(plannerHistory, 10); + // Add the execution result to the planner's history with clearer prompting plannerHistory.Add(new ChatMessage(ChatRole.User, @@ -398,13 +436,22 @@ private async Task GetAgentResponseAsync( { var responseBuilder = new StringBuilder(); - await foreach (var update in chatService.GetStreamingResponseAsync(history, options)) + try { - if (update.Text != null) + await foreach (var update in chatService.GetStreamingResponseAsync(history, options)) { - responseBuilder.Append(update.Text); + if (update.Text != null) + { + responseBuilder.Append(update.Text); + } } } + catch (Exception ex) when (ex.Message.Contains("Unknown ChatFinishReason") || ex.Message.Contains("function_call_filter")) + { + // Swallow known SDK mapping errors for specific provider finish reasons + // This allows us to keep the text generated so far + PluginLogger.LogInfo("MultiAgentActioner", "GetAgentResponseAsync", $"Ignored known SDK finish reason error: {ex.Message}"); + } string response = responseBuilder.ToString(); history.Add(new ChatMessage(ChatRole.Assistant, response)); @@ -412,6 +459,22 @@ private async Task GetAgentResponseAsync( return response; } + private void ManageContextWindow(List history, int maxMessages) + { + // Always keep the system prompt (assumed to be at index 0) + if (history.Count <= maxMessages + 1) return; + + // Calculate how many messages to remove + // We want to keep: SystemPrompt (1) + Last N messages + int messagesToRemove = history.Count - (maxMessages + 1); + + if (messagesToRemove > 0) + { + // Remove messages starting from index 1 (preserve System Prompt) + history.RemoveRange(1, messagesToRemove); + } + } + /// /// Extracts the first actionable step from the planner's plan. /// Looks for lines that mention a tool/plugin or a direct action. @@ -443,7 +506,7 @@ private string ExtractActionableStep(string plan) return null; } - public void SetChatHistory(List chatHistory) + public void SetChatHistory(System.Collections.Generic.List chatHistory) { // Set up coordinator history with system prompt coordinatorHistory.Clear(); @@ -469,4 +532,4 @@ public void SetChatHistory(List chatHistory) } } } -} \ No newline at end of file +} diff --git a/FlowVision/lib/Plugins/ClipboardPlugin.cs b/FlowVision/lib/Plugins/ClipboardPlugin.cs new file mode 100644 index 0000000..e1ce087 --- /dev/null +++ b/FlowVision/lib/Plugins/ClipboardPlugin.cs @@ -0,0 +1,59 @@ +using System; +using System.ComponentModel; +using System.Windows.Forms; +using FlowVision.lib.Classes; + +namespace FlowVision.lib.Plugins +{ + internal class ClipboardPlugin + { + [Description("Sets the system clipboard text content")] + public void SetClipboardText(string text) + { + PluginLogger.LogPluginUsage("ClipboardPlugin", "SetClipboardText", text); + + if (Application.OpenForms.Count > 0) + { + Application.OpenForms[0].Invoke(new Action(() => { + try + { + Clipboard.SetText(text); + } + catch (Exception ex) + { + PluginLogger.LogError("ClipboardPlugin", "SetClipboardText", ex.Message); + } + })); + } + } + + [Description("Gets the current text content from the system clipboard")] + public string GetClipboardText() + { + PluginLogger.LogPluginUsage("ClipboardPlugin", "GetClipboardText"); + string clipboardText = ""; + + if (Application.OpenForms.Count > 0) + { + clipboardText = (string)Application.OpenForms[0].Invoke(new Func(() => { + try + { + if (Clipboard.ContainsText()) + { + return Clipboard.GetText(); + } + else + { + return "[Clipboard is empty or contains non-text data]"; + } + } + catch (Exception ex) + { + return $"Error reading clipboard: {ex.Message}"; + } + })); + } + return clipboardText; + } + } +} diff --git a/FlowVision/lib/Plugins/FileSystemPlugin.cs b/FlowVision/lib/Plugins/FileSystemPlugin.cs new file mode 100644 index 0000000..4400e84 --- /dev/null +++ b/FlowVision/lib/Plugins/FileSystemPlugin.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.IO; +using System.Linq; +using FlowVision.lib.Classes; + +namespace FlowVision.lib.Plugins +{ + internal class FileSystemPlugin + { + [Description("Gets the current working directory of the application")] + public string GetCurrentDirectory() + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "GetCurrentDirectory"); + return Directory.GetCurrentDirectory(); + } + + [Description("Lists files and directories in the specified path. Returns first 50 entries.")] + public string ListDirectory(string path) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "ListDirectory", path); + try + { + if (!Directory.Exists(path)) return $"Directory not found: {path}"; + + var dirs = Directory.GetDirectories(path).Select(d => $"[DIR] {Path.GetFileName(d)}"); + var files = Directory.GetFiles(path).Select(f => Path.GetFileName(f)); + + var all = dirs.Concat(files).Take(50); + return string.Join("\n", all); + } + catch (Exception ex) + { + return $"Error: {ex.Message}"; + } + } + + [Description("Checks if a file exists at the specified path")] + public bool FileExists(string path) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "FileExists", path); + return File.Exists(path); + } + + [Description("Reads the content of a text file (max 2000 chars)")] + public string ReadFile(string path) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "ReadFile", path); + try + { + if (!File.Exists(path)) return "File not found"; + + string content = File.ReadAllText(path); + if (content.Length > 2000) + { + return content.Substring(0, 2000) + "\n...[Truncated]..."; + } + return content; + } + catch (Exception ex) + { + return $"Error: {ex.Message}"; + } + } + + [Description("Writes text content to a file. Overwrites if exists.")] + public string WriteFile(string path, string content) + { + PluginLogger.LogPluginUsage("FileSystemPlugin", "WriteFile", path); + try + { + File.WriteAllText(path, content); + return $"Successfully wrote to {path}"; + } + catch (Exception ex) + { + return $"Error writing file: {ex.Message}"; + } + } + } +} diff --git a/FlowVision/lib/Plugins/KeyboardPlugin.cs b/FlowVision/lib/Plugins/KeyboardPlugin.cs index 4d2f341..05747e1 100644 --- a/FlowVision/lib/Plugins/KeyboardPlugin.cs +++ b/FlowVision/lib/Plugins/KeyboardPlugin.cs @@ -53,7 +53,7 @@ public async Task SendKey(string keyCombo) } } - [Description("Send keyboard input to a specific window by handle")] + [Description("Send keyboard input to a specific window by handle. keys format: standard SendKeys (e.g. 'Hello', '{ENTER}', '^c').")] public async Task SendKeyToWindow(string windowHandleString, string keyCombo) { PluginLogger.LogPluginUsage("KeyboardPlugin", "SendKeyToWindow", @@ -72,7 +72,8 @@ public async Task SendKeyToWindow(string windowHandleString, string keyCom } // Wait a bit for the window to become active - await Task.Delay(200); + // Increased to 500ms to ensure slower apps (like Notepad startup) are ready + await Task.Delay(500); // Send the keys SendKeys.SendWait(keyCombo); @@ -132,6 +133,16 @@ public async Task CtrlKeyToWindow(string windowHandleString, string letter return await SendKeyToWindow(windowHandleString, $"^({letter})"); } + [DllImport("user32.dll")] + private static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); + + [DllImport("user32.dll")] + private static extern IntPtr SetFocus(IntPtr hWnd); + + private const int SW_RESTORE = 9; + + // ... (existing imports) + /// /// Brings a window to the foreground and ensures it has focus using multiple techniques /// @@ -142,13 +153,12 @@ private bool BringWindowToForegroundWithFocus(IntPtr hWnd) try { + // 1. Force window restore if minimized + ShowWindow(hWnd, SW_RESTORE); + // Get the current foreground window IntPtr currentForeground = GetForegroundWindow(); - // If it's already in foreground, we're done - if (currentForeground == hWnd) - return true; - // Get thread IDs uint currentThreadId = GetCurrentThreadId(); uint targetThreadId = GetWindowThreadProcessId(hWnd, out _); @@ -161,15 +171,28 @@ private bool BringWindowToForegroundWithFocus(IntPtr hWnd) AttachThreadInput(currentThreadId, foregroundThreadId, true); needsDetach = true; } + + // Also attach to the target thread if it's different + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, targetThreadId, true); + } // Try to set foreground window bool success = SetForegroundWindow(hWnd); + + // Force focus to the specific handle + SetFocus(hWnd); // Detach if we attached if (needsDetach) { AttachThreadInput(currentThreadId, foregroundThreadId, false); } + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, targetThreadId, false); + } // Give it a moment to process Thread.Sleep(50); diff --git a/FlowVision/lib/Plugins/MousePlugin.cs b/FlowVision/lib/Plugins/MousePlugin.cs index 87aa59d..048c3c7 100644 --- a/FlowVision/lib/Plugins/MousePlugin.cs +++ b/FlowVision/lib/Plugins/MousePlugin.cs @@ -18,20 +18,49 @@ internal class MousePlugin [DllImport("user32.dll", SetLastError = true)] private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + [DllImport("user32.dll", SetLastError = true)] + private static extern bool SetForegroundWindow(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr GetForegroundWindow(); + + [DllImport("user32.dll")] + private static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); + + [DllImport("kernel32.dll")] + private static extern uint GetCurrentThreadId(); + + [DllImport("user32.dll", SetLastError = true)] + private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId); + + [DllImport("user32.dll")] + private static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); + + [DllImport("user32.dll")] + private static extern IntPtr SetFocus(IntPtr hWnd); + + private const int SW_RESTORE = 9; private const uint MOUSEEVENTF_LEFTDOWN = 0x02; private const uint MOUSEEVENTF_LEFTUP = 0x04; private const uint MOUSEEVENTF_RIGHTDOWN = 0x08; private const uint MOUSEEVENTF_RIGHTUP = 0x10; - [Description("Clicks at the specified normalized bounding box coordinates on a specific window handle.")] - public async Task ClickOnWindow(string windowHandleString, double[] bBox, bool leftClick, int clickTimes) + [Description("Clicks at the specified normalized bounding box coordinates on a specific window handle. Box is [x1, y1, x2, y2].")] + public async Task ClickOnWindow(string windowHandleString, double x1, double y1, double x2, double y2, bool leftClick, int clickTimes) { // Log the plugin usage PluginLogger.LogPluginUsage("MousePlugin", "ClickOnWindow", - $"window={windowHandleString}, pos={string.Join(",", bBox)}, leftClick={leftClick}, times={clickTimes}"); + $"window={windowHandleString}, box=[{x1},{y1},{x2},{y2}], leftClick={leftClick}"); IntPtr windowHandle = new IntPtr(Convert.ToInt32(windowHandleString)); + // Ensure window is visible and focused + if (!BringWindowToForegroundWithFocus(windowHandle)) + { + PluginLogger.LogError("MousePlugin", "ClickOnWindow", "Failed to focus window"); + return false; + } + if (!GetWindowRect(windowHandle, out RECT rc)) { throw new InvalidOperationException("Failed to get window rectangle."); @@ -41,15 +70,16 @@ public async Task ClickOnWindow(string windowHandleString, double[] bBox, int windowHeight = rc.Bottom - rc.Top; // Calculate absolute position based on bounding box (normalized) - int x = rc.Left + (int)((bBox[0] + bBox[2]) / 2 * windowWidth); - int y = rc.Top + (int)((bBox[1] + bBox[3]) / 2 * windowHeight); + int x = rc.Left + (int)((x1 + x2) / 2 * windowWidth); + int y = rc.Top + (int)((y1 + y2) / 2 * windowHeight); if (!SetCursorPos(x, y)) { throw new InvalidOperationException("Failed to set cursor position."); } - await Task.Delay(100); + // Increased delay to allow UI to register hover state + await Task.Delay(200); for (int i = 0; i < clickTimes; i++) { @@ -66,7 +96,14 @@ public async Task ScrollOnWindow(string windowHandleString, int scrollAmou // Log the plugin usage PluginLogger.LogPluginUsage("MousePlugin", "ScrollOnWindow", $"window={windowHandleString}, amount={scrollAmount}"); + IntPtr windowHandle = new IntPtr(Convert.ToInt32(windowHandleString)); + + if (!BringWindowToForegroundWithFocus(windowHandle)) + { + return false; + } + if (!GetWindowRect(windowHandle, out RECT rc)) { throw new InvalidOperationException("Failed to get window rectangle."); @@ -77,7 +114,7 @@ public async Task ScrollOnWindow(string windowHandleString, int scrollAmou { throw new InvalidOperationException("Failed to set cursor position."); } - await Task.Delay(100); + await Task.Delay(200); mouse_event(0x0800, 0, 0, (uint)scrollAmount, UIntPtr.Zero); return true; } @@ -91,6 +128,51 @@ private void SimulateClick(int x, int y, bool leftClick) mouse_event(up, (uint)x, (uint)y, 0, UIntPtr.Zero); } + /// + /// Brings a window to the foreground and ensures it has focus using multiple techniques + /// + private bool BringWindowToForegroundWithFocus(IntPtr hWnd) + { + if (hWnd == IntPtr.Zero) return false; + + try + { + ShowWindow(hWnd, SW_RESTORE); + IntPtr currentForeground = GetForegroundWindow(); + if (currentForeground == hWnd) return true; + + uint currentThreadId = GetCurrentThreadId(); + uint targetThreadId = GetWindowThreadProcessId(hWnd, out _); + uint foregroundThreadId = GetWindowThreadProcessId(currentForeground, out _); + + bool needsDetach = false; + if (currentThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, foregroundThreadId, true); + needsDetach = true; + } + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + { + AttachThreadInput(currentThreadId, targetThreadId, true); + } + + bool success = SetForegroundWindow(hWnd); + SetFocus(hWnd); + + if (needsDetach) AttachThreadInput(currentThreadId, foregroundThreadId, false); + if (targetThreadId != currentThreadId && targetThreadId != foregroundThreadId) + AttachThreadInput(currentThreadId, targetThreadId, false); + + System.Threading.Thread.Sleep(100); + return GetForegroundWindow() == hWnd; + } + catch (Exception ex) + { + PluginLogger.LogError("MousePlugin", "BringWindowToForegroundWithFocus", $"Error: {ex.Message}"); + return false; + } + } + [StructLayout(LayoutKind.Sequential)] private struct RECT { diff --git a/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs b/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs index b865766..24998c3 100644 --- a/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs +++ b/FlowVision/lib/Plugins/ScreenCaptureOmniParserPlugin.cs @@ -64,6 +64,45 @@ public async Task> CaptureWholeScreen() } } + [Description("Find specific text on screen using OCR. Useful if visual detection fails.")] + public async Task FindTextOnScreen(string searchText) + { + PluginLogger.LogPluginUsage("ScreenCaptureOmniParserPlugin", "FindTextOnScreen", searchText); + + // Capture whole screen + using (Bitmap screenshot = new Bitmap(SystemInformation.VirtualScreen.Width, SystemInformation.VirtualScreen.Height)) + { + using (Graphics gfx = Graphics.FromImage(screenshot)) + { + gfx.CopyFromScreen( + SystemInformation.VirtualScreen.X, + SystemInformation.VirtualScreen.Y, + 0, 0, + SystemInformation.VirtualScreen.Size, + CopyPixelOperation.SourceCopy); + } + + // Use OCR to find the text + var rect = await OcrHelper.FindTextLocationAsync(screenshot, searchText); + + if (rect.HasValue) + { + // Return in format compatible with ClickOnWindow (normalized bbox: x1, y1, x2, y2) + float width = screenshot.Width; + float height = screenshot.Height; + + float x1 = rect.Value.X / width; + float y1 = rect.Value.Y / height; + float x2 = (rect.Value.X + rect.Value.Width) / width; + float y2 = (rect.Value.Y + rect.Value.Height) / height; + + return $"Found '{searchText}' at normalized box: [{x1:F4}, {y1:F4}, {x2:F4}, {y2:F4}]"; + } + + return $"Text '{searchText}' not found on screen."; + } + } + /// /// Process screenshot with Simple OmniParser (KISS implementation) /// diff --git a/FlowVision/lib/Plugins/WindowSelectionPlugin.cs b/FlowVision/lib/Plugins/WindowSelectionPlugin.cs index 0a4560e..988bdfa 100644 --- a/FlowVision/lib/Plugins/WindowSelectionPlugin.cs +++ b/FlowVision/lib/Plugins/WindowSelectionPlugin.cs @@ -3,6 +3,7 @@ using System.ComponentModel; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Text; using System.Threading.Tasks; using FlowVision.lib.Classes; @@ -16,6 +17,52 @@ internal class WindowSelectionPlugin [DllImport("user32.dll", SetLastError = true)] private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + // --- New P/Invoke Definitions for Safe Enumeration --- + private delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam); + + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam); + + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool IsWindowVisible(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Auto)] + private static extern int GetWindowTextLength(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Auto)] + private static extern IntPtr SendMessageTimeout( + IntPtr hWnd, + uint Msg, + IntPtr wParam, + IntPtr lParam, + uint fuFlags, + uint uTimeout, + out IntPtr lpdwResult); + + [DllImport("user32.dll", SetLastError = true)] + private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint processId); + + private const uint WM_GETTEXT = 0x000D; + private const uint WM_GETTEXTLENGTH = 0x000E; + private const uint SMTO_ABORTIFHUNG = 0x0002; + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr GetForegroundWindow(); + + // ------------------------------------------------------ + + [Description("Gets the handle and title of the currently active foreground window.")] + public string GetForegroundWindowInfo() + { + IntPtr hWnd = GetForegroundWindow(); + if (hWnd == IntPtr.Zero) return "No active window"; + + string title = GetWindowTitleSafe(hWnd); + return $"Handle: {hWnd}, Title: {title}"; + } + [Description("Used to set current handle as foreground")] public async Task ForegroundSelect(string handleString) { @@ -37,24 +84,91 @@ public string ListWindowHandles() PluginLogger.LogPluginUsage("WindowSelectionPlugin", "ListWindowHandles"); var windowList = new List(); - Process[] processes = Process.GetProcesses(); - foreach (Process p in processes) + + EnumWindows((hWnd, lParam) => { + // Filter invisible windows to reduce noise and hangs + if (!IsWindowVisible(hWnd)) + return true; // Continue enumeration + + // Safely get window title with timeout + string title = GetWindowTitleSafe(hWnd); + + // Skip untitled windows (often hidden helper windows) + if (string.IsNullOrWhiteSpace(title)) + return true; + + // Get process name + string processName = "Unknown"; try { - if (p.MainWindowHandle == IntPtr.Zero) - continue; - string item = $"Handle: {p.MainWindowHandle}, Title: {p.MainWindowTitle}, Process: {p.ProcessName}"; - windowList.Add(item); + GetWindowThreadProcessId(hWnd, out uint processId); + using (var p = Process.GetProcessById((int)processId)) + { + processName = p.ProcessName; + } } - catch - { - continue; - } - } + catch { /* Process might have exited or access denied */ } + + string item = $"Handle: {hWnd}, Title: {title}, Process: {processName}"; + windowList.Add(item); + + return true; // Continue enumeration + }, IntPtr.Zero); + return string.Join("\n", windowList); } + /// + /// Safely retrieves window title with a timeout to prevent hanging on unresponsive windows. + /// + private string GetWindowTitleSafe(IntPtr hWnd) + { + const int timeoutMs = 100; // Short timeout to ensure responsiveness + + // 1. Get text length with timeout + IntPtr result; + IntPtr ret = SendMessageTimeout( + hWnd, + WM_GETTEXTLENGTH, + IntPtr.Zero, + IntPtr.Zero, + SMTO_ABORTIFHUNG, + timeoutMs, + out result); + + if (ret == IntPtr.Zero) return string.Empty; // Failed or timed out + + int length = (int)result; + if (length == 0) return string.Empty; + + // 2. Get actual text with timeout + // Allocate unmanaged memory for the string buffer + // Length + 1 for null terminator, * 2 for Unicode characters + int bufferSize = (length + 1) * 2; + IntPtr buffer = Marshal.AllocHGlobal(bufferSize); + + try + { + ret = SendMessageTimeout( + hWnd, + WM_GETTEXT, + new IntPtr(length + 1), + buffer, + SMTO_ABORTIFHUNG, + timeoutMs, + out result); + + if (ret == IntPtr.Zero) return string.Empty; + + return Marshal.PtrToStringAuto(buffer); + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + /// /// Checks if the provided window handle is valid. ///