Fix and simplify ComputerUse sample (#5075)

* fix the computer use sample

* rollback changes to the search state enum

* address review comments

* address review comments

Author: SergeyMenshykh

dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Agent_Step15_ComputerUse.csproj

@@ -6,7 +6,7 @@
 
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
-    <NoWarn>$(NoWarn);OPENAICUA001</NoWarn>
+    <NoWarn>$(NoWarn);OPENAICUA001;MEAI001</NoWarn>
   </PropertyGroup>
 
   <ItemGroup>
@@ -19,13 +19,13 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Update="Assets\cua_browser_search.png">
+    <None Update="Assets\cua_browser_search.jpg">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
-    <None Update="Assets\cua_search_results.png">
+    <None Update="Assets\cua_search_results.jpg">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
-    <None Update="Assets\cua_search_typed.png">
+    <None Update="Assets\cua_search_typed.jpg">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>

dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_browser_search.jpg

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using Microsoft.Extensions.AI;
 using OpenAI.Responses;
 
 namespace Demo.ComputerUse;
@@ -16,83 +17,77 @@ internal enum SearchState
 
 internal static class ComputerUseUtil
 {
-    /// <summary>
-    /// Load and convert screenshot images to base64 data URLs.
-    /// </summary>
-    internal static Dictionary<string, byte[]> LoadScreenshotAssets()
+    internal static async Task<Dictionary<string, string>> UploadScreenshotAssetsAsync(IHostedFileClient fileClient)
     {
-        string baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets");
+        string assetsDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets");
+
+        (string key, string fileName)[] files =
+        [
+            ("browser_search", "cua_browser_search.jpg"),
+            ("search_typed", "cua_search_typed.jpg"),
+            ("search_results", "cua_search_results.jpg")
+        ];
 
-        ReadOnlySpan<(string key, string fileName)> screenshotFiles =
-            [
-                ("browser_search", "cua_browser_search.png"),
-                ("search_typed", "cua_search_typed.png"),
-                ("search_results", "cua_search_results.png")
-            ];
+        Dictionary<string, string> screenshots = [];
 
-        Dictionary<string, byte[]> screenshots = [];
-        foreach (var (key, fileName) in screenshotFiles)
+        foreach (var (key, fileName) in files)
         {
-            string fullPath = Path.GetFullPath(Path.Combine(baseDir, fileName));
-            screenshots[key] = File.ReadAllBytes(fullPath);
+            HostedFileContent result = await fileClient.UploadAsync(
+                Path.Combine(assetsDir, fileName), new HostedFileClientOptions() { Purpose = "assistants" });
+            screenshots[key] = result.FileId;
         }
 
         return screenshots;
     }
 
+    internal static async Task EnsureDeleteScreenshotAssetsAsync(IHostedFileClient fileClient, Dictionary<string, string> screenshots)
+    {
+        foreach (var (_, fileId) in screenshots)
+        {
+            try
+            {
+                await fileClient.DeleteAsync(fileId);
+            }
+            catch
+            {
+            }
+        }
+    }
+
     /// <summary>
-    /// Process a computer action and simulate its execution.
+    /// Simulates executing a computer action by advancing the state
+    /// and returning the screenshot file ID for the new state.
     /// </summary>
-    internal static (SearchState CurrentState, byte[] ImageBytes) HandleComputerActionAndTakeScreenshot(
+    internal static async Task<(SearchState State, string FileId)> GetScreenshotAsync(
         ComputerCallAction action,
         SearchState currentState,
-        Dictionary<string, byte[]> screenshots)
-    {
-        Console.WriteLine($"Simulating the execution of computer action: {action.Kind}");
-
-        SearchState newState = DetermineNextState(action, currentState);
-        string imageKey = GetImageKey(newState);
-
-        return (newState, screenshots[imageKey]);
-    }
-
-    private static SearchState DetermineNextState(ComputerCallAction action, SearchState currentState)
+        Dictionary<string, string> screenshots)
     {
-        string actionType = action.Kind.ToString();
-
-        if (actionType.Equals("type", StringComparison.OrdinalIgnoreCase) && action.TypeText is not null)
+        if (action.Kind == ComputerCallActionKind.Wait)
         {
-            return SearchState.Typed;
+            await Task.Delay(TimeSpan.FromSeconds(5));
         }
 
-        if (IsEnterKeyAction(action, actionType))
+        SearchState nextState = action.Kind switch
         {
-            Console.WriteLine("  -> Detected ENTER key press");
-            return SearchState.PressedEnter;
-        }
+            ComputerCallActionKind.Click when currentState == SearchState.Typed => SearchState.PressedEnter,
+            ComputerCallActionKind.Type when action.TypeText is not null => SearchState.Typed,
+            ComputerCallActionKind.KeyPress when IsEnterKey(action) => SearchState.PressedEnter,
+            _ => currentState
+        };
 
-        if (actionType.Equals("click", StringComparison.OrdinalIgnoreCase) && currentState == SearchState.Typed)
+        string imageKey = nextState switch
         {
-            Console.WriteLine("  -> Detected click after typing");
-            return SearchState.PressedEnter;
-        }
+            SearchState.PressedEnter => "search_results",
+            SearchState.Typed => "search_typed",
+            _ => "browser_search"
+        };
 
-        return currentState;
+        return (nextState, screenshots[imageKey]);
     }
 
-    private static bool IsEnterKeyAction(ComputerCallAction action, string actionType)
-    {
-        return (actionType.Equals("key", StringComparison.OrdinalIgnoreCase) ||
-                actionType.Equals("keypress", StringComparison.OrdinalIgnoreCase)) &&
-               action.KeyPressKeyCodes is not null &&
-               (action.KeyPressKeyCodes.Contains("Return", StringComparer.OrdinalIgnoreCase) ||
-                action.KeyPressKeyCodes.Contains("Enter", StringComparer.OrdinalIgnoreCase));
-    }
-
-    private static string GetImageKey(SearchState state) => state switch
-    {
-        SearchState.PressedEnter => "search_results",
-        SearchState.Typed => "search_typed",
-        _ => "browser_search"
-    };
+    private static bool IsEnterKey(ComputerCallAction action) =>
+        action.KeyPressKeyCodes is not null &&
+        (action.KeyPressKeyCodes.Contains("Return", StringComparer.OrdinalIgnoreCase) ||
+         action.KeyPressKeyCodes.Contains("Enter", StringComparer.OrdinalIgnoreCase));
 }

dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_browser_search.png

@@ -1,146 +1,109 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-// This sample shows how to use Computer Use Tool with a ChatClientAgent.
+// This sample shows how to use the Computer Use tool with AIProjectClient.AsAIAgent(...).
 
 using Azure.AI.Projects;
 using Azure.Identity;
+using Demo.ComputerUse;
 using Microsoft.Agents.AI;
 using Microsoft.Agents.AI.Foundry;
 using Microsoft.Extensions.AI;
 using OpenAI.Responses;
 
-namespace Demo.ComputerUse;
+string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
+string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_COMPUTER_USE_DEPLOYMENT_NAME") ?? "computer-use-preview";
 
-internal sealed class Program
-{
-    private static async Task Main(string[] args)
-    {
-        const string AgentInstructions = @"
-                    You are a computer automation assistant. 
-                    
-                    Be direct and efficient. When you reach the search results page, read and describe the actual search result titles and descriptions you can see.
-                ";
-
-        const string AgentName = "ComputerAgent-RAPI";
-
-        string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
-        string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "computer-use-preview";
-
-        // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
-        // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
-        // latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
-        AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential());
-
-        // Create a AIAgent with ComputerUseTool.
-        AIAgent agent = aiProjectClient.AsAIAgent(deploymentName,
-            instructions: AgentInstructions,
-            name: AgentName,
-            description: "Computer automation agent with screen interaction capabilities.",
-            tools: [
-                    FoundryAITool.CreateComputerTool(ComputerToolEnvironment.Browser, 1026, 769),
-                ]);
-
-        await InvokeComputerUseAgentAsync(agent);
-    }
+AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
+using IHostedFileClient fileClient = projectClient.GetProjectOpenAIClient().AsIHostedFileClient();
 
-    private static async Task InvokeComputerUseAgentAsync(AIAgent agent)
-    {
-        // Load screenshot assets
-        Dictionary<string, byte[]> screenshots = ComputerUseUtil.LoadScreenshotAssets();
+AIAgent agent = projectClient.AsAIAgent(
+    model: deploymentName,
+    name: "ComputerAgent",
+    instructions: "You are a computer automation assistant.",
+    tools: [FoundryAITool.CreateComputerTool(ComputerToolEnvironment.Browser, 1026, 769)]);
 
-        ChatOptions chatOptions = new();
-        CreateResponseOptions responseCreationOptions = new()
-        {
-            TruncationMode = ResponseTruncationMode.Auto
-        };
-        chatOptions.RawRepresentationFactory = (_) => responseCreationOptions;
-        ChatClientAgentRunOptions runOptions = new(chatOptions)
-        {
-            AllowBackgroundResponses = true,
-        };
+Dictionary<string, string> screenshots = [];
 
-        ChatMessage message = new(ChatRole.User, [
-            new TextContent("I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete."),
-            new DataContent(new BinaryData(screenshots["browser_search"]), "image/png")
-        ]);
-
-        // Initial request with screenshot - start with Bing search page
-        Console.WriteLine("Starting computer automation session (initial screenshot: cua_browser_search.png)...");
-
-        // We use PreviousResponseId to chain calls, sending only the new computer_call_output items
-        // instead of re-sending the full context.
-        AgentSession session = await agent.CreateSessionAsync();
-        AgentResponse response = await agent.RunAsync(message, session: session, options: runOptions);
-
-        // Main interaction loop
-        const int MaxIterations = 10;
-        int iteration = 0;
-        // Initialize state machine
-        SearchState currentState = SearchState.Initial;
+try
+{
+    // Upload pre-captured screenshots that simulate browser state transitions.
+    screenshots = await ComputerUseUtil.UploadScreenshotAssetsAsync(fileClient);
 
-        while (true)
+    // Enable auto-truncation for the Responses API.
+    ChatClientAgentRunOptions runOptions = new()
+    {
+        ChatOptions = new ChatOptions
         {
-            // Poll until the response is complete.
-            while (response.ContinuationToken is { } token)
-            {
-                // Wait before polling again.
-                await Task.Delay(TimeSpan.FromSeconds(2));
+            RawRepresentationFactory = (_) => new CreateResponseOptions() { TruncationMode = ResponseTruncationMode.Auto },
+        }
+    };
 
-                // Continue with the token.
-                runOptions.ContinuationToken = token;
+    // Send the initial request with a screenshot of the browser.
+    ChatMessage message = new(ChatRole.User, [
+        new TextContent("Search for 'OpenAI news'. Type it and submit. Once you see results, the task is complete."),
+        new AIContent() { RawRepresentation = ResponseContentPart.CreateInputImagePart(imageFileId: screenshots["browser_search"], imageDetailLevel: ResponseImageDetailLevel.High) }
+    ]);
 
-                response = await agent.RunAsync(session, runOptions);
-            }
+    Console.WriteLine("Starting computer use session...");
+
+    AgentSession session = await agent.CreateSessionAsync();
+    AgentResponse response = await agent.RunAsync(message, session: session, options: runOptions);
 
-            // Clear the continuation token so the next RunAsync call is a fresh request.
-            runOptions.ContinuationToken = null;
+    SearchState currentState = SearchState.Initial;
 
-            Console.WriteLine($"Agent response received (ID: {response.ResponseId})");
+    for (int i = 0; i < 10; i++)
+    {
+        // Find the next computer call action.
+        ComputerCallResponseItem? computerCall = response.Messages
+            .SelectMany(m => m.Contents)
+            .Select(c => c.RawRepresentation as ComputerCallResponseItem)
+            .FirstOrDefault(item => item is not null);
 
-            if (iteration >= MaxIterations)
+        if (computerCall is null)
+        {
+            if (currentState == SearchState.PressedEnter)
             {
-                Console.WriteLine($"\nReached maximum iterations ({MaxIterations}). Stopping.");
+                Console.WriteLine("No more computer actions. Done.");
+                Console.WriteLine(response);
                 break;
             }
 
-            iteration++;
-            Console.WriteLine($"\n--- Iteration {iteration} ---");
-
-            // Check for computer calls in the response
-            IEnumerable<ComputerCallResponseItem> computerCallResponseItems = response.Messages
-                .SelectMany(x => x.Contents)
-                .Where(c => c.RawRepresentation is ComputerCallResponseItem and not null)
-                .Select(c => (ComputerCallResponseItem)c.RawRepresentation!);
-
-            ComputerCallResponseItem? firstComputerCall = computerCallResponseItems.FirstOrDefault();
-            if (firstComputerCall is null)
+            // Check if the agent is asking for confirmation to proceed, and if so, respond affirmatively.
+            TextContent? textContent = response.Messages
+                .Where(m => m.Role == ChatRole.Assistant)
+                .SelectMany(m => m.Contents.OfType<TextContent>())
+                .FirstOrDefault();
+
+            if (textContent?.Text is { } text && (
+                text.Contains("Would you like me") ||
+                text.Contains("Should I") ||
+                text.Contains("proceed") ||
+                text.Contains('?')))
             {
-                Console.WriteLine("No computer call actions found. Ending interaction.");
-                Console.WriteLine($"Final Response: {response}");
-                break;
+                response = await agent.RunAsync("Please proceed.", session, runOptions);
+                continue;
             }
 
-            // Process the first computer call response
-            ComputerCallAction action = firstComputerCall.Action;
-            string currentCallId = firstComputerCall.CallId;
-
-            Console.WriteLine($"Processing computer call (ID: {currentCallId})");
+            break;
+        }
 
-            // Simulate executing the action and taking a screenshot
-            (SearchState CurrentState, byte[] ImageBytes) screenInfo = ComputerUseUtil.HandleComputerActionAndTakeScreenshot(action, currentState, screenshots);
-            currentState = screenInfo.CurrentState;
+        Console.WriteLine($"[{i + 1}] Action: {computerCall!.Action.Kind}");
 
-            Console.WriteLine("Sending action result back to agent...");
+        // Simulate the action and get the resulting screenshot.
+        (currentState, string fileId) = await ComputerUseUtil.GetScreenshotAsync(computerCall.Action, currentState, screenshots);
 
-            // Send only the computer_call_output — the session carries PreviousResponseId for context continuity.
-            AIContent callOutput = new()
-            {
-                RawRepresentation = new ComputerCallOutputResponseItem(
-                    currentCallId,
-                    output: ComputerCallOutput.CreateScreenshotOutput(new BinaryData(screenInfo.ImageBytes), "image/png"))
-            };
+        // Send the screenshot back as the computer call output.
+        AIContent callOutput = new()
+        {
+            RawRepresentation = new ComputerCallOutputResponseItem(
+                computerCall.CallId,
+                output: ComputerCallOutput.CreateScreenshotOutput(screenshotImageFileId: fileId))
+        };
 
-            response = await agent.RunAsync([new ChatMessage(ChatRole.User, [callOutput])], session: session, options: runOptions);
-        }
+        response = await agent.RunAsync([new ChatMessage(ChatRole.User, [callOutput])], session: session, options: runOptions);
     }
 }
+finally
+{
+    await ComputerUseUtil.EnsureDeleteScreenshotAssetsAsync(fileClient, screenshots);
+}