Skip to content

Commit 084262a

Browse files
westey-malliscode
authored andcommitted
.NET: Add github actions workflow for verify-samples (microsoft#5034)
* Add github actions workflow for verify-samples * Make workflow run as part of PR (for now) * Update workflow to remove pr trigger * Address PR comments
1 parent 5709f95 commit 084262a

5 files changed

Lines changed: 237 additions & 1 deletion

File tree

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
#
2+
# Runs the .NET sample verification tool, which builds and executes sample projects
3+
# and verifies their output using deterministic checks and AI-powered verification.
4+
#
5+
# Results are displayed as a GitHub Job Summary and the CSV report is uploaded as an artifact.
6+
#
7+
8+
name: dotnet-verify-samples
9+
10+
on:
11+
workflow_dispatch:
12+
inputs:
13+
category:
14+
description: "Sample category to run (blank for all)"
15+
required: false
16+
type: choice
17+
options:
18+
- ""
19+
- "01-get-started"
20+
- "02-agents"
21+
- "03-workflows"
22+
parallelism:
23+
description: "Max parallel sample runs"
24+
required: false
25+
default: "8"
26+
type: string
27+
schedule:
28+
- cron: "0 6 * * 1-5" # Weekdays at 6:00 UTC
29+
30+
concurrency:
31+
group: ${{ github.workflow }}-${{ github.ref }}
32+
cancel-in-progress: true
33+
34+
permissions:
35+
contents: read
36+
id-token: write
37+
38+
jobs:
39+
verify-samples:
40+
runs-on: ubuntu-latest
41+
environment: 'integration'
42+
timeout-minutes: 90
43+
steps:
44+
- uses: actions/checkout@v6
45+
with:
46+
persist-credentials: false
47+
sparse-checkout: |
48+
.
49+
.github
50+
dotnet
51+
workflow-samples
52+
53+
- name: Setup dotnet
54+
uses: actions/setup-dotnet@v5.2.0
55+
with:
56+
global-json-file: ${{ github.workspace }}/dotnet/global.json
57+
58+
- name: Azure CLI Login
59+
if: github.event_name != 'pull_request'
60+
uses: azure/login@v2
61+
with:
62+
client-id: ${{ secrets.AZURE_CLIENT_ID }}
63+
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
64+
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
65+
66+
- name: Run verify-samples
67+
id: verify
68+
working-directory: dotnet
69+
shell: bash
70+
run: |
71+
CATEGORY_ARG=""
72+
if [ -n "$CATEGORY_INPUT" ]; then
73+
CATEGORY_ARG="--category $CATEGORY_INPUT"
74+
fi
75+
76+
dotnet run --project eng/verify-samples -- \
77+
$CATEGORY_ARG \
78+
--parallel "$PARALLELISM" \
79+
--md results.md \
80+
--csv results.csv \
81+
--log results.log
82+
env:
83+
CATEGORY_INPUT: ${{ github.event.inputs.category || '' }}
84+
PARALLELISM: ${{ github.event.inputs.parallelism || '8' }}
85+
# OpenAI Models
86+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
87+
OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
88+
OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
89+
# Azure OpenAI Models
90+
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
91+
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
92+
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
93+
# Azure AI Foundry
94+
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
95+
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
96+
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
97+
98+
- name: Write Job Summary
99+
if: always()
100+
working-directory: dotnet
101+
shell: bash
102+
run: |
103+
if [ -f results.md ]; then
104+
cat results.md >> "$GITHUB_STEP_SUMMARY"
105+
else
106+
echo "⚠️ No results.md generated — verify-samples may have failed to start." >> "$GITHUB_STEP_SUMMARY"
107+
fi
108+
109+
- name: Upload results
110+
if: always()
111+
uses: actions/upload-artifact@v7
112+
with:
113+
name: verify-samples-results
114+
path: |
115+
dotnet/results.csv
116+
dotnet/results.log
117+
if-no-files-found: warn
118+
119+
- name: Fail if samples failed
120+
if: always() && steps.verify.outcome == 'failure'
121+
shell: bash
122+
run: exit 1

dotnet/.github/skills/verify-samples-tool/SKILL.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ dotnet run --project eng/verify-samples -- Agent_Step02_StructuredOutput Agent_S
2525
dotnet run --project eng/verify-samples -- --parallel 8 --log results.log
2626

2727
# Combine options
28-
dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv
28+
dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv --md results.md
2929
```
3030

3131
### Required Environment Variables
@@ -40,6 +40,7 @@ Individual samples require their own env vars (e.g., `AZURE_AI_PROJECT_ENDPOINT`
4040

4141
- `--log results.log` — detailed per-sample log with stdout/stderr, AI reasoning, and a summary
4242
- `--csv results.csv` — tabular summary with Sample, ProjectPath, Status, FailedChecks, and Failures columns
43+
- `--md results.md` — Markdown summary with results table and collapsible failure details (suitable for GitHub PR comments)
4344

4445
## Sample Categories
4546

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.Text;
4+
5+
namespace VerifySamples;
6+
7+
/// <summary>
8+
/// Writes a Markdown summary of sample verification results.
9+
/// </summary>
10+
internal static class MarkdownResultWriter
11+
{
12+
/// <summary>
13+
/// Writes the results to a Markdown file at the specified path.
14+
/// </summary>
15+
public static async Task WriteAsync(
16+
string path,
17+
IReadOnlyList<VerificationResult> orderedResults,
18+
IReadOnlyList<(string Name, string Reason)> skipped,
19+
TimeSpan elapsed)
20+
{
21+
var passCount = orderedResults.Count(r => r.Passed);
22+
var failCount = orderedResults.Count(r => !r.Passed);
23+
24+
var sb = new StringBuilder();
25+
sb.AppendLine("# Sample Verification Results");
26+
sb.AppendLine();
27+
sb.AppendLine($"**{passCount} passed, {failCount} failed, {skipped.Count} skipped** | Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}");
28+
sb.AppendLine();
29+
30+
// Results table
31+
sb.AppendLine("| Sample | Status | Failed Checks | Failures |");
32+
sb.AppendLine("|--------|--------|---------------|----------|");
33+
34+
foreach (var result in orderedResults)
35+
{
36+
var status = result.Passed ? "✅ PASSED" : "❌ FAILED";
37+
var failedChecks = result.Failures.Count;
38+
var failures = MdEscape(string.Join("; ", result.Failures));
39+
sb.AppendLine($"| {MdEscape(result.SampleName)} | {status} | {failedChecks} | {failures} |");
40+
}
41+
42+
foreach (var (name, reason) in skipped)
43+
{
44+
sb.AppendLine($"| {MdEscape(name)} | ⏭️ SKIPPED | 0 | {MdEscape(reason)} |");
45+
}
46+
47+
// Collapsible AI reasoning details for failures
48+
var failures2 = orderedResults.Where(r => !r.Passed && !string.IsNullOrEmpty(r.AIReasoning)).ToList();
49+
if (failures2.Count > 0)
50+
{
51+
sb.AppendLine();
52+
sb.AppendLine("## Failure Details");
53+
sb.AppendLine();
54+
55+
foreach (var result in failures2)
56+
{
57+
sb.AppendLine($"<details><summary><strong>{HtmlEscape(result.SampleName)}</strong></summary>");
58+
sb.AppendLine();
59+
if (result.Failures.Count > 0)
60+
{
61+
foreach (var failure in result.Failures)
62+
{
63+
sb.AppendLine($"- {MdEscape(failure)}");
64+
}
65+
66+
sb.AppendLine();
67+
}
68+
69+
sb.AppendLine("**AI Reasoning:**");
70+
sb.AppendLine();
71+
sb.AppendLine("```");
72+
sb.AppendLine(result.AIReasoning);
73+
sb.AppendLine("```");
74+
sb.AppendLine();
75+
sb.AppendLine("</details>");
76+
sb.AppendLine();
77+
}
78+
}
79+
80+
await File.WriteAllTextAsync(path, sb.ToString());
81+
}
82+
83+
/// <summary>
84+
/// Escapes pipe characters and newlines for use inside Markdown table cells.
85+
/// </summary>
86+
private static string MdEscape(string value)
87+
{
88+
return value.Replace("|", "\\|").Replace("\n", " ").Replace("\r", "");
89+
}
90+
91+
/// <summary>
92+
/// Escapes HTML special characters for use inside HTML tags.
93+
/// </summary>
94+
private static string HtmlEscape(string value)
95+
{
96+
return value.Replace("&", "&amp;").Replace("<", "&lt;").Replace(">", "&gt;").Replace("\"", "&quot;");
97+
}
98+
}

dotnet/eng/verify-samples/Program.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// dotnet run -- --parallel 16 # Run up to 16 samples concurrently
1414
// dotnet run -- --log results.log # Write sequential log to file
1515
// dotnet run -- --csv results.csv # Write CSV summary to file
16+
// dotnet run -- --md results.md # Write Markdown summary to file
1617
//
1718
// Required environment variables (for AI-powered samples):
1819
// AZURE_OPENAI_ENDPOINT
@@ -90,6 +91,13 @@
9091
Console.WriteLine($"CSV written to: {options.CsvFilePath}");
9192
}
9293

94+
// Write Markdown summary
95+
if (options.MarkdownFilePath is not null)
96+
{
97+
await MarkdownResultWriter.WriteAsync(options.MarkdownFilePath, orderedResults, run.Skipped, stopwatch.Elapsed);
98+
Console.WriteLine($"Markdown written to: {options.MarkdownFilePath}");
99+
}
100+
93101
return orderedResults.Any(r => !r.Passed) ? 1 : 0;
94102
}
95103
finally

dotnet/eng/verify-samples/VerifyOptions.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ internal sealed class VerifyOptions
1717
/// </summary>
1818
public string? CsvFilePath { get; init; }
1919

20+
/// <summary>
21+
/// Path to write a Markdown summary file, or <c>null</c> to skip.
22+
/// </summary>
23+
public string? MarkdownFilePath { get; init; }
24+
2025
/// <summary>
2126
/// Path to write a sequential log file, or <c>null</c> to skip.
2227
/// </summary>
@@ -49,6 +54,7 @@ internal sealed class VerifyOptions
4954
var categoryFilter = ExtractArg(argList, "--category");
5055
var logFilePath = ExtractArg(argList, "--log");
5156
var csvFilePath = ExtractArg(argList, "--csv");
57+
var markdownFilePath = ExtractArg(argList, "--md");
5258

5359
int maxParallelism = 8;
5460
var parallelArg = ExtractArg(argList, "--parallel");
@@ -98,6 +104,7 @@ internal sealed class VerifyOptions
98104
MaxParallelism = maxParallelism,
99105
LogFilePath = logFilePath,
100106
CsvFilePath = csvFilePath,
107+
MarkdownFilePath = markdownFilePath,
101108
Samples = samples,
102109
};
103110
}

0 commit comments

Comments
 (0)