.NET Workflows - Support fidelity when converting to and from ChatMessage in declarative workflows

dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ChatMessageExtensions.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
 using Microsoft.Agents.ObjectModel;
@@ -83,7 +84,8 @@ public static IEnumerable<ChatMessage> ToChatMessages(this TableDataValue messag
     public static ChatMessage ToChatMessage(this RecordDataValue message) =>
         new(message.GetRole(), [.. message.GetContent()])
         {
-            AdditionalProperties = message.GetProperty<RecordDataValue>("metadata").ToMetadata()
+            MessageId = message.GetProperty<StringDataValue>(TypeSchema.Message.Fields.Id)?.Value,
+            AdditionalProperties = message.GetProperty<RecordDataValue>(TypeSchema.Message.Fields.Metadata).ToMetadata()
         };
 
     public static ChatMessage ToChatMessage(this StringDataValue message) => new(ChatRole.User, message.Value);
@@ -118,7 +120,7 @@ public static ChatRole ToChatRole(this AgentMessageRole role) =>
 
     public static ChatRole ToChatRole(this AgentMessageRole? role) => role?.ToChatRole() ?? ChatRole.User;
 
-    public static AIContent? ToContent(this AgentMessageContentType contentType, string? contentValue)
+    public static AIContent? ToContent(this AgentMessageContentType contentType, string? contentValue, string? mediaType = null)
     {
         if (string.IsNullOrEmpty(contentValue))
         {
@@ -128,7 +130,7 @@ public static ChatRole ToChatRole(this AgentMessageRole role) =>
         return
             contentType switch
             {
-                AgentMessageContentType.ImageUrl => GetImageContent(contentValue),
+                AgentMessageContentType.ImageUrl => GetImageContent(contentValue, mediaType ?? InferMediaType(contentValue)),
                 AgentMessageContentType.ImageFile => new HostedFileContent(contentValue),
                 _ => new TextContent(contentValue)
             };
@@ -159,25 +161,52 @@ private static IEnumerable<AIContent> GetContent(this RecordDataValue message)
             foreach (RecordDataValue contentItem in content.Values)
             {
                 StringDataValue? contentValue = contentItem.GetProperty<StringDataValue>(TypeSchema.MessageContent.Fields.Value);
+                StringDataValue? mediaTypeValue = contentItem.GetProperty<StringDataValue>(TypeSchema.MessageContent.Fields.MediaType);
                 if (contentValue is null || string.IsNullOrWhiteSpace(contentValue.Value))
                 {
                     continue;
                 }
+
                 yield return
                     contentItem.GetProperty<StringDataValue>(TypeSchema.MessageContent.Fields.Type)?.Value switch
                     {
-                        TypeSchema.MessageContent.ContentTypes.ImageUrl => GetImageContent(contentValue.Value),
+                        TypeSchema.MessageContent.ContentTypes.ImageUrl => GetImageContent(contentValue.Value, mediaTypeValue?.Value ?? InferMediaType(contentValue.Value)),
                         TypeSchema.MessageContent.ContentTypes.ImageFile => new HostedFileContent(contentValue.Value),
                         _ => new TextContent(contentValue.Value)
                     };
             }
         }
     }
 
-    private static AIContent GetImageContent(string uriText) =>
+    private static string InferMediaType(string value)
+    {
+        // Base64 encoded content includes media type
+        if (value.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
+        {
+            int semicolonIndex = value.IndexOf(';');
+            if (semicolonIndex > 5)
+            {
+                return value.Substring(5, semicolonIndex - 5);
+            }
+        }
+
+        // URL based input only supports image
+        string fileExtension = Path.GetExtension(value);
+        return
+            fileExtension.ToUpperInvariant() switch
+            {
+                ".JPG" or ".JPEG" => "image/jpeg",
+                ".PNG" => "image/png",
+                ".GIF" => "image/gif",
+                ".WEBP" => "image/webp",
+                _ => "image/*"
+            };
+    }
+
+    private static AIContent GetImageContent(string uriText, string mediaType) =>
         uriText.StartsWith("data:", StringComparison.OrdinalIgnoreCase) ?
-            new DataContent(uriText, "image/*") :
-            new UriContent(uriText, "image/*");
+            new DataContent(uriText, mediaType) :
+            new UriContent(uriText, mediaType);
 
     private static TValue? GetProperty<TValue>(this RecordDataValue record, string name)
         where TValue : DataValue

dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/AddConversationMessageExecutor.cs

@@ -40,7 +40,7 @@ private IEnumerable<AIContent> GetContent()
     {
         foreach (AddConversationMessageContent content in this.Model.Content)
         {
-            AIContent? messageContent = content.Type.Value.ToContent(this.Engine.Format(content.Value));
+            AIContent? messageContent = content.Type.Value.ToContent(this.Engine.Format(content.Value), content.MediaType);
             if (messageContent is not null)
             {
                 yield return messageContent;

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/MediaInputTest.cs

@@ -19,43 +19,48 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests;
 /// </summary>
 public sealed class MediaInputTest(ITestOutputHelper output) : IntegrationTest(output)
 {
-    private const string WorkflowFileName = "MediaInput.yaml";
+    private const string WorkflowWithConversationFileName = "MediaInputConversation.yaml";
+    private const string WorkflowWithAutoSendFileName = "MediaInputAutoSend.yaml";
     private const string PdfReference = "https://sample-files.com/downloads/documents/pdf/basic-text.pdf";
     private const string ImageReference = "https://sample-files.com/downloads/images/jpg/web_optimized_1200x800_97kb.jpg";
 
     [Theory]
-    [InlineData(ImageReference, "image/jpeg", Skip = "Failing consistently in the agent service api")]
-    [InlineData(PdfReference, "application/pdf", Skip = "Not currently supported by agent service api")]
-    public async Task ValidateFileUrlAsync(string fileSource, string mediaType)
+    [InlineData(ImageReference, "image/jpeg", true, Skip = "Failing due to agent service bug.")]
+    [InlineData(ImageReference, "image/jpeg", false, Skip = "Failing due to agent service bug.")]
+    public async Task ValidateFileUrlAsync(string fileSource, string mediaType, bool useConversation)
     {
         this.Output.WriteLine($"File: {ImageReference}");
-        await this.ValidateFileAsync(new UriContent(fileSource, mediaType));
+        await this.ValidateFileAsync(new UriContent(fileSource, mediaType), useConversation);
     }
 
     [Theory]
-    [InlineData(ImageReference, "image/jpeg")]
-    [InlineData(PdfReference, "application/pdf")]
-    public async Task ValidateFileDataAsync(string fileSource, string mediaType)
+    [InlineData(ImageReference, "image/jpeg", true)]
+    [InlineData(ImageReference, "image/jpeg", false, Skip = "Failing due to agent service bug.")]
+    [InlineData(PdfReference, "application/pdf", true)]
+    [InlineData(PdfReference, "application/pdf", false)]
+    public async Task ValidateFileDataAsync(string fileSource, string mediaType, bool useConversation)
     {
         byte[] fileData = await DownloadFileAsync(fileSource);
         string encodedData = Convert.ToBase64String(fileData);
         string fileUrl = $"data:{mediaType};base64,{encodedData}";
         this.Output.WriteLine($"Content: {fileUrl.Substring(0, 112)}...");
-        await this.ValidateFileAsync(new DataContent(fileUrl));
+        await this.ValidateFileAsync(new DataContent(fileUrl), useConversation);
     }
 
-    [Fact(Skip = "Not currently supported by agent service api")]
-    public async Task ValidateFileUploadAsync()
+    [Theory]
+    [InlineData(PdfReference, "doc.pdf", true, Skip = "Failing due to agent service bug.")]
+    [InlineData(PdfReference, "doc.pdf", false, Skip = "Failing due to agent service bug.")]
+    public async Task ValidateFileUploadAsync(string fileSource, string documentName, bool useConversation)
     {
-        byte[] fileData = await DownloadFileAsync(PdfReference);
+        byte[] fileData = await DownloadFileAsync(fileSource);
         AIProjectClient client = new(this.TestEndpoint, new AzureCliCredential());
         using MemoryStream contentStream = new(fileData);
         OpenAIFileClient fileClient = client.GetProjectOpenAIClient().GetOpenAIFileClient();
-        OpenAIFile fileInfo = await fileClient.UploadFileAsync(contentStream, "basic-text.pdf", FileUploadPurpose.Assistants);
+        OpenAIFile fileInfo = await fileClient.UploadFileAsync(contentStream, documentName, FileUploadPurpose.Assistants);
         try
         {
             this.Output.WriteLine($"File: {fileInfo.Id}");
-            await this.ValidateFileAsync(new HostedFileContent(fileInfo.Id));
+            await this.ValidateFileAsync(new HostedFileContent(fileInfo.Id), useConversation);
         }
         finally
         {
@@ -70,20 +75,26 @@ private static async Task<byte[]> DownloadFileAsync(string uri)
         return await client.GetByteArrayAsync(new Uri(uri));
     }
 
-    private async Task ValidateFileAsync(AIContent fileContent)
+    private async Task ValidateFileAsync(AIContent fileContent, bool useConversation)
     {
         AgentProvider agentProvider = AgentProvider.Create(this.Configuration, AgentProvider.Names.Vision);
         await agentProvider.CreateAgentsAsync().ConfigureAwait(false);
 
-        ChatMessage inputMessage = new(ChatRole.User, [new TextContent("I've provided a file:"), fileContent]);
+        ChatMessage inputMessage =
+            new(ChatRole.User,
+                [
+                    new TextContent("I've provided a file:"),
+                    fileContent
+                ]);
 
+        string workflowFileName = useConversation ? WorkflowWithConversationFileName : WorkflowWithAutoSendFileName;
         DeclarativeWorkflowOptions options = await this.CreateOptionsAsync();
-        Workflow workflow = DeclarativeWorkflowBuilder.Build<ChatMessage>(Path.Combine(Environment.CurrentDirectory, "Workflows", WorkflowFileName), options);
+        Workflow workflow = DeclarativeWorkflowBuilder.Build<ChatMessage>(Path.Combine(Environment.CurrentDirectory, "Workflows", workflowFileName), options);
 
-        WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(WorkflowFileName));
+        WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(workflowFileName));
         WorkflowEvents workflowEvents = await harness.RunWorkflowAsync(inputMessage).ConfigureAwait(false);
-        ConversationUpdateEvent conversationEvent = Assert.Single(workflowEvents.ConversationEvents);
-        this.Output.WriteLine("CONVERSATION: " + conversationEvent.ConversationId);
+        Assert.Equal(useConversation ? 1 : 2, workflowEvents.ConversationEvents.Count);
+        this.Output.WriteLine("CONVERSATION: " + workflowEvents.ConversationEvents[0].ConversationId);
         AgentRunResponseEvent agentResponseEvent = Assert.Single(workflowEvents.AgentResponseEvents);
         this.Output.WriteLine("RESPONSE: " + agentResponseEvent.Response.Text);
         Assert.NotEmpty(agentResponseEvent.Response.Text);

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/MediaInputAutoSend.yaml

@@ -0,0 +1,15 @@
+kind: Workflow
+trigger:
+
+  kind: OnConversationStart
+  id: workflow_test
+  actions:
+
+    - kind: InvokeAzureAgent
+      id: invoke_vision
+      agent:
+        name: VisionAgent
+      input:
+        messages: =System.LastMessage
+      output:
+        autoSend: true

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Extensions/ChatMessageExtensionsTests.cs

@@ -666,4 +666,73 @@ public void ToRecordWithMessageContainingMetadata()
         RecordValue metadataRecord = Assert.IsType<RecordValue>(metadataField, exactMatch: false);
         Assert.Equal(2, metadataRecord.Fields.Count());
     }
+
+    [Fact]
+    public void RoundTripChatMessageAsRecord()
+    {
+        // Arrange
+        ChatMessage message =
+            new(ChatRole.User,
+                [
+                    new TextContent("Test message"),
+                    new UriContent("https://example.com/image.jpg", "image/jpeg"),
+                    new HostedFileContent("file_123abc"),
+                    new DataContent(new byte[] { 1, 2, 3, 4, 5 }, "application/pdf"),
+                ])
+            {
+                MessageId = "msg-001"
+            };
+
+        // Act
+        RecordValue result = message.ToRecord();
+        DataValue resultValue = result.ToDataValue();
+        ChatMessage? messageCopy = resultValue.ToChatMessage();
+
+        // Assert
+        Assert.NotNull(messageCopy);
+        Assert.Equal(message.Role, messageCopy.Role);
+        Assert.Equal(message.MessageId, messageCopy.MessageId);
+        Assert.Equal(message.Contents.Count, messageCopy.Contents.Count);
+        foreach (AIContent contentCopy in messageCopy.Contents)
+        {
+            AIContent sourceContent = Assert.Single(message.Contents, c => c.GetType() == contentCopy.GetType());
+            Assert.Equivalent(sourceContent, contentCopy);
+        }
+    }
+
+    [Fact]
+    public void RoundTripChatMessageAsTable()
+    {
+        // Arrange
+        ChatMessage message =
+            new(ChatRole.User,
+                [
+                    new TextContent("Test message"),
+                    new UriContent("https://example.com/image.jpg", "image/jpeg"),
+                    new HostedFileContent("file_123abc"),
+                    new DataContent(new byte[] { 1, 2, 3, 4, 5 }, "application/pdf"),
+                ])
+            {
+                MessageId = "msg-001"
+            };
+
+        IEnumerable<ChatMessage> messages = [message];
+
+        // Act
+        TableValue result = messages.ToTable();
+        TableDataValue resultValue = result.ToTable();
+        ChatMessage[] messagesCopy = resultValue.ToChatMessages().ToArray();
+
+        // Assert
+        Assert.NotNull(messagesCopy);
+        ChatMessage messageCopy = Assert.Single(messagesCopy);
+        Assert.Equal(message.Role, messageCopy.Role);
+        Assert.Equal(message.MessageId, messageCopy.MessageId);
+        Assert.Equal(message.Contents.Count, messageCopy.Contents.Count);
+        foreach (AIContent contentCopy in messageCopy.Contents)
+        {
+            AIContent sourceContent = Assert.Single(message.Contents, c => c.GetType() == contentCopy.GetType());
+            Assert.Equivalent(sourceContent, contentCopy);
+        }
+    }
 }