From 8345dc67929b2797ace34771d6165b84a4bf8675 Mon Sep 17 00:00:00 2001 From: CakeRepository Date: Fri, 3 Oct 2025 00:30:15 -0500 Subject: [PATCH] Enhance action execution prompts and improve chat client configuration - Updated action prompt in LMStudioActioner to include explicit execution instructions for users. - Added a workflow diagram to the Multi-Agent Architecture documentation to illustrate agent interactions and processes. - Improved logging for function invocation in the chat client setup. --- FlowVision/lib/Classes/ai/LMStudioActioner.cs | 30 ++++- docs/Prompt-Engineering-Guide.md | 64 +++++++++++ docs/Workflow-Builder-Tutorial.md | 104 ++++++++++++++++++ 3 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 docs/Prompt-Engineering-Guide.md create mode 100644 docs/Workflow-Builder-Tutorial.md diff --git a/FlowVision/lib/Classes/ai/LMStudioActioner.cs b/FlowVision/lib/Classes/ai/LMStudioActioner.cs index 4770dfd..3a3c590 100644 --- a/FlowVision/lib/Classes/ai/LMStudioActioner.cs +++ b/FlowVision/lib/Classes/ai/LMStudioActioner.cs @@ -83,8 +83,16 @@ public async Task ExecuteAction(string actionPrompt) // Add system message to actioner history actionerHistory.Add(new ChatMessage(ChatRole.System, toolConfig.ActionerSystemPrompt + toolDescriptions)); - // Add action prompt to actioner history - actionerHistory.Add(new ChatMessage(ChatRole.User, actionPrompt)); + // Add action prompt with explicit instruction to EXECUTE + string enhancedPrompt = $@"{actionPrompt} + +IMPORTANT REMINDER: +1. DO NOT just observe and describe - you must EXECUTE the action! +2. After CaptureWholeScreen(), you MUST continue to actually click/type/interact +3. Follow ALL steps: Observe → Plan → EXECUTE → Verify +4. Do not stop until you've performed the actual action requested"; + + actionerHistory.Add(new ChatMessage(ChatRole.User, enhancedPrompt)); // Verify LM Studio is enabled and configured if (!lmStudioConfig.Enabled) @@ -162,9 +170,21 @@ public async Task ExecuteAction(string actionPrompt) }; // Build chat client with function invocation if enabled - actionerChat = toolConfig.AutoInvokeKernelFunctions - ? new ChatClientBuilder(baseChatClient).UseFunctionInvocation().Build() - : baseChatClient; + if (toolConfig.AutoInvokeKernelFunctions) + { + // Use function invocation with default behavior + // The middleware will automatically loop and call tools + actionerChat = new ChatClientBuilder(baseChatClient) + .UseFunctionInvocation() + .Build(); + + PluginLogger.LogInfo("LMStudioActioner", "ExecuteAction", + "Function invocation enabled - will auto-invoke tools"); + } + else + { + actionerChat = baseChatClient; + } // Update loading message to show we're now processing the response PluginLogger.StopLoadingIndicator(); diff --git a/docs/Prompt-Engineering-Guide.md b/docs/Prompt-Engineering-Guide.md new file mode 100644 index 0000000..b1bbc6d --- /dev/null +++ b/docs/Prompt-Engineering-Guide.md @@ -0,0 +1,64 @@ +--- +layout: default +title: Prompt Engineering Guide +--- + +# Prompt Engineering Guide for Recursive Control + +Master the art of communicating with your AI agents to achieve precise, efficient automation. + +## Why Prompt Engineering Matters +Recursive Control's multi-agent system (Hermes, Daedalus, and Talos) interprets natural language, but structured prompts yield better results: +- **Clarity reduces errors**: Ambiguous requests lead to wrong assumptions. +- **Structure aids planning**: Well-organized prompts help the Planner agent create better steps. +- **Context improves accuracy**: Provide relevant details for better execution. + +## Core Principles +1. **Be Specific**: Include exact details like app names, file paths, or expected outcomes. +2. **Break It Down**: For complex tasks, suggest steps or use multi-stage prompts. +3. **Provide Context**: Mention current state, like open windows or recent actions. +4. **Use Verification**: Ask the agent to confirm steps or describe what it sees. +5. **Handle Errors Gracefully**: Include fallback instructions. + +## Prompt Patterns +### Basic Command +**Template**: "Perform [action] in [location/app] with [details]." + +**Example**: "Open Chrome and navigate to github.com/flowdevs-io/Recursive-Control." + +### Multi-Step Workflow +**Template**: "Do the following steps: 1. [Step 1] 2. [Step 2] ... Verify each step." + +**Example**: "Create a report: 1. Open Excel. 2. Add headers: Date, Task, Status. 3. Fill with today's data. 4. Save as 'daily-report.xlsx' in Documents." + +### Vision-Assisted +**Template**: "Capture the screen, describe what you see, then [action based on description]." + +**Example**: "Take a screenshot of the current window, identify the search bar, and type 'AI tools' into it." + +### Conditional Logic +**Template**: "If [condition], do [action A]; else do [action B]." + +**Example**: "If Chrome is open, navigate to YouTube; else open Chrome first then go to YouTube." + +## Advanced Techniques +### Chain of Thought +Encourage reasoning: "Think step-by-step: First, check if the app is open. If not, open it. Then..." + +### Role Playing +Assign roles: "As a efficient automation expert, optimize this workflow: [task]." + +### Few-Shot Examples +Provide samples: "Like how you opened Notepad last time, now open Paint and draw a square." + +## Common Pitfalls +- **Too Vague**: "Do something with files" → Agents might guess wrong. +- **Overly Complex**: Break long prompts into multiple interactions. +- **Assuming State**: Always verify: "Focus on the foreground window and describe it first." + +## Tips for Power Users +- Use the UI's chat history to build context across prompts. +- Combine plugins explicitly: "Use Playwright to automate browser, then CMD to process downloads." +- Test incrementally: Start with simple tasks and build up. + +Master these patterns to turn Recursive Control into your ultimate productivity copilot! diff --git a/docs/Workflow-Builder-Tutorial.md b/docs/Workflow-Builder-Tutorial.md new file mode 100644 index 0000000..cae4ce9 --- /dev/null +++ b/docs/Workflow-Builder-Tutorial.md @@ -0,0 +1,104 @@ +--- +layout: default +title: Workflow Builder Tutorial +--- + +# Workflow Builder Tutorial: Gamified Learning Adventures + +Level up your Recursive Control skills by building interactive workflows! This tutorial turns learning into a game: complete "quests" by crafting prompt chains that automate real tasks. Each quest includes objectives, step-by-step guidance, and verification challenges. + +Think of this as a simulator – test your prompts here before running them in the app. Earn "badges" by successfully completing each workflow (self-assessed via expected outcomes). + +## Quest 1: GitHub Issue Automator (Beginner Level) +**Objective**: Automate creating a GitHub issue in your repo using browser automation and keyboard inputs. + +**Badge**: Issue Master + +**Step-by-Step Prompt Chain**: +1. **Launch the Browser**: "Open Chrome and navigate to github.com/login." + - *Expected*: Browser opens to GitHub login page. (Verify: Describe the screen to confirm.) + +2. **Login**: "Focus on the username field and type 'yourusername', then tab to password and type 'yourpassword', then press enter." + - *Tip*: Use KeyboardPlugin for targeted input. (Challenge: Add verification – "If login fails, notify me.") + +3. **Navigate to Repo**: "Go to github.com/yourusername/your-repo/issues." + - *Expected*: Issues page loads. + +4. **Create Issue**: "Click the 'New issue' button, type 'Bug: App crashes on load' in title, add description 'Steps to reproduce: 1. Open app. 2. Click button.', then submit." + - *Challenge*: Use ScreenCapture to identify the button's bounding box before clicking. + +**Full Chain Prompt** (Copy-Paste Ready): +``` +Perform these steps to create a GitHub issue: +1. Open Chrome and go to github.com/login. +2. Log in with username 'yourusername' and password 'yourpassword'. +3. Navigate to github.com/yourusername/your-repo/issues. +4. Click 'New issue', fill title 'Bug: App crashes', description 'Steps: Open app, click button', and submit. +Verify each step with a screenshot description. +``` + +**Verification Quest**: Run this in Recursive Control. Did it create the issue? If not, refine the prompt (e.g., handle 2FA). + +## Quest 2: Daily Report Generator (Intermediate Level) +**Objective**: Automate opening Excel, filling data, and saving a report. + +**Badge**: Report Wizard + +**Step-by-Step Prompt Chain**: +1. **Open App**: "Launch Excel and create a new spreadsheet." +2. **Add Headers**: "Type 'Date' in A1, 'Task' in B1, 'Status' in C1." +3. **Fill Data**: "In A2 type today's date, B2 'Implement feature X', C2 'Completed'." +4. **Save**: "Save as 'daily-report.xlsx' in Documents." + +**Full Chain Prompt**: +``` +Build a daily report in Excel: +1. Open Excel, new file. +2. Headers: A1=Date, B1=Task, C1=Status. +3. Data: A2=today's date, B2=Review code, C2=Done. +4. Save to Documents as daily-report.xlsx. +Confirm each cell after typing. +``` + +**Challenge**: Add conditional logic – "If file exists, append instead of overwrite." + +## Quest 3: File Organizer Bot (Advanced Level) +**Objective**: Scan Downloads folder, organize files by type using CMD/PowerShell. + +**Badge**: Organization Overlord + +**Step-by-Step Prompt Chain**: +1. **Scan Folder**: "Use CMD to list files in Downloads." +2. **Create Folders**: "Make directories: Images, Documents, Others." +3. **Move Files**: "Move .jpg to Images, .pdf to Documents, others to Others." +4. **Verify**: "List contents of each new folder." + +**Full Chain Prompt**: +``` +Organize Downloads: +1. CMD: dir %USERPROFILE%\Downloads +2. Create folders: mkdir Images Documents Others +3. Move: move *.jpg Images, move *.pdf Documents, move *.* Others +4. Verify: dir Images, dir Documents, dir Others +Handle errors if folders exist. +``` + +**Epic Challenge**: Integrate vision – "Screenshot Downloads folder and describe file icons before moving." + +## Level Up Tips +- **Gamification Hack**: Track your success rate. Aim for 100% on 5 quests to "unlock" custom workflow creation. +- **Combine Quests**: Chain them, e.g., "Generate report, then create GitHub issue about it." +- **Debug Mode**: If a step fails, add "Describe the screen and suggest fixes" to your prompts. + +## Go Interactive: Jupyter Notebook Simulator +For hands-on practice, download our [Interactive Workflow Builder Notebook](tutorials/Workflow-Builder-Notebook.ipynb). It includes widgets to build and simulate prompt chains without running the full app! + +**How to Use**: +1. Install Jupyter: `pip install notebook` +2. Download the .ipynb file. +3. Run `jupyter notebook` and open the file. +4. Interact with widgets to test prompts in real-time. + +Completed all quests? Share your custom workflows on Discord! + +Back to [Getting Started](Getting-Started.html)