diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index 37b28874c58..25d9f1d7121 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"2f052ffbae60d063e6fd03882cfaf94cebaed434d16cd31ffdb258f705a95e3c","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"8e1b45ac0e2c883e76a948893cb57390962ce64083cf2bf9212463384ccdf65e","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.41"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.41"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.41"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.41"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.6","digest":"sha256:2bb8eef86006a4c5963c55616a9c51c32f27bfdecb023b8aa6f91f6718d9171c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.6@sha256:2bb8eef86006a4c5963c55616a9c51c32f27bfdecb023b8aa6f91f6718d9171c"},{"image":"ghcr.io/github/github-mcp-server:v1.0.3","digest":"sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.3@sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959"},{"image":"node:lts-alpine","digest":"sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f","pinned_image":"node:lts-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f"}]} # ___ _ _ # / _ \ | | (_) @@ -191,20 +191,20 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_27bb211012917eaf_EOF' + cat << 'GH_AW_PROMPT_b79b9a648c212332_EOF' - GH_AW_PROMPT_27bb211012917eaf_EOF + GH_AW_PROMPT_b79b9a648c212332_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_27bb211012917eaf_EOF' + cat << 'GH_AW_PROMPT_b79b9a648c212332_EOF' Tools: add_comment, submit_pull_request_review, missing_tool, missing_data, noop - GH_AW_PROMPT_27bb211012917eaf_EOF + GH_AW_PROMPT_b79b9a648c212332_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_27bb211012917eaf_EOF' + cat << 'GH_AW_PROMPT_b79b9a648c212332_EOF' The following GitHub context information is available for this workflow: {{#if __GH_AW_GITHUB_ACTOR__ }} @@ -233,13 +233,13 @@ jobs: {{/if}} - GH_AW_PROMPT_27bb211012917eaf_EOF + GH_AW_PROMPT_b79b9a648c212332_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_27bb211012917eaf_EOF' + cat << 'GH_AW_PROMPT_b79b9a648c212332_EOF' {{#runtime-import .github/workflows/shared/reporting.md}} {{#runtime-import .github/workflows/test-quality-sentinel.md}} - GH_AW_PROMPT_27bb211012917eaf_EOF + GH_AW_PROMPT_b79b9a648c212332_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -389,10 +389,38 @@ jobs: env: GH_TOKEN: ${{ github.token }} - env: + GH_AW_GITHUB_EVENT_PULL_REQUEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.pull_request.number }} name: Pre-fetch PR data - run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/agent\n\n# PR metadata\ngh pr view \"$PR_NUMBER\" \\\n --json files,additions,deletions,baseRefName,headRefName \\\n > /tmp/gh-aw/agent/pr-meta.json\n\n# List of changed test files\ngh pr diff \"$PR_NUMBER\" \\\n --name-only | grep -E '(_test\\.go|\\.test\\.cjs|\\.test\\.js)$' \\\n > /tmp/gh-aw/agent/test-files.txt || true\n\n# Diff for test files only (empty file is fine if no test files changed)\nif [ -s /tmp/gh-aw/agent/test-files.txt ]; then\n # shellcheck disable=SC2046\n gh pr diff \"$PR_NUMBER\" \\\n -- $(tr '\\n' ' ' < /tmp/gh-aw/agent/test-files.txt) \\\n > /tmp/gh-aw/agent/test-diff.txt 2>/dev/null || true\nelse\n touch /tmp/gh-aw/agent/test-diff.txt\nfi\n\necho \"Pre-fetched $(grep -c . /tmp/gh-aw/agent/test-files.txt || echo 0) test files\"\n" + run: | + set -euo pipefail + mkdir -p /tmp/gh-aw/agent + + # PR metadata + gh pr view "$PR_NUMBER" \ + --json files,additions,deletions,baseRefName,headRefName \ + > /tmp/gh-aw/agent/pr-meta.json + + # List of changed test files + gh pr diff "$PR_NUMBER" \ + --name-only | grep -E '(_test\.go|\.test\.cjs|\.test\.js)$' \ + > /tmp/gh-aw/agent/test-files.txt || true + + # Diff for test files only (empty file is fine if no test files changed) + if [ -s /tmp/gh-aw/agent/test-files.txt ]; then + # shellcheck disable=SC2046 + gh pr diff "$PR_NUMBER" \ + -- $(tr '\n' ' ' < /tmp/gh-aw/agent/test-files.txt) \ + > /tmp/gh-aw/agent/test-diff.txt 2>/dev/null || true + else + touch /tmp/gh-aw/agent/test-diff.txt + fi + + git diff "$GH_AW_GITHUB_EVENT_PULL_REQUEST_BASE_SHA...HEAD" --numstat \ + > /tmp/gh-aw/agent/diff-numstat.txt 2>/dev/null || true + + echo "Pre-fetched $(grep -c . /tmp/gh-aw/agent/test-files.txt || echo 0) test files" - name: Configure Git credentials env: @@ -460,9 +488,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_73bf4b770d190d6c_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_1428044c636c8116_EOF' {"add_comment":{"hide_older_comments":true,"max":1},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{},"submit_pull_request_review":{"max":1}} - GH_AW_SAFE_OUTPUTS_CONFIG_73bf4b770d190d6c_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_1428044c636c8116_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -669,7 +697,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_7a565ef53b4595e7_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_caf174e4ed5be36d_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "safeoutputs": { @@ -694,7 +722,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_7a565ef53b4595e7_EOF + GH_AW_MCP_CONFIG_caf174e4ed5be36d_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true diff --git a/.github/workflows/test-quality-sentinel.md b/.github/workflows/test-quality-sentinel.md index b639d0cca5e..90275d8f3a3 100644 --- a/.github/workflows/test-quality-sentinel.md +++ b/.github/workflows/test-quality-sentinel.md @@ -53,6 +53,9 @@ steps: touch /tmp/gh-aw/agent/test-diff.txt fi + git diff "${{ github.event.pull_request.base.sha }}...HEAD" --numstat \ + > /tmp/gh-aw/agent/diff-numstat.txt 2>/dev/null || true + echo "Pre-fetched $(grep -c . /tmp/gh-aw/agent/test-files.txt || echo 0) test files" safe-outputs: add-comment: @@ -71,6 +74,7 @@ imports: - shared/reporting.md features: copilot-requests: true + inline-agents: true --- # Test Quality Sentinel ๐Ÿงช @@ -103,6 +107,9 @@ cat /tmp/gh-aw/agent/test-files.txt # Diff for test files only cat /tmp/gh-aw/agent/test-diff.txt + +# Numstat for all changed files +cat /tmp/gh-aw/agent/diff-numstat.txt ``` Then identify all **new and modified test files** in the diff: @@ -155,24 +162,10 @@ For each changed test file, run structural checks using available tools. ### 3a. Go โ€” `Test*` functions -Analyze Go test functions using grep and awk on the diff. This codebase uses **both** stdlib assertions (`t.Errorf`, `t.Fatalf`, `t.Error`) **and** testify (`assert.*`, `require.*`). The project guideline is **no mock libraries** โ€” tests must interact with real components; any use of `gomock`, `testify/mock`, or `EXPECT()` in Go is itself a red flag. - -```bash -# Count assertions, error checks, table-driven subtests, and any forbidden mock calls per Test* function -git diff ${{ github.event.pull_request.base.sha }}...HEAD -- '*_test.go' | awk ' -/^\+func Test/ { - if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "table_driven=" table_driven, "forbidden_mocks=" forbidden_mocks - match($0, /func (Test[^(]+)/, arr); test_name=arr[1]; assertions=0; errors=0; table_driven=0; forbidden_mocks=0 -} -test_name && /^\+.*(assert\.|require\.)/ { assertions++ } -test_name && /^\+.*t\.(Error|Errorf|Fatal|Fatalf)\(/ { assertions++; errors++ } -test_name && /^\+.*(assert\.Error|require\.Error|assert\.NoError|require\.NoError)/ { errors++ } -test_name && /^\+.*t\.Run\(/ { table_driven++ } -test_name && /^\+.*(gomock\.|testify\/mock|\.EXPECT\(\)|\.On\(|\.Return\()/ { forbidden_mocks++ } -test_name && /^\+\}$/ { print test_name, "assertions=" assertions, "errors=" errors, "table_driven=" table_driven, "forbidden_mocks=" forbidden_mocks; test_name="" } -END { if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "table_driven=" table_driven, "forbidden_mocks=" forbidden_mocks } -' -``` +Use the `go-test-analyzer` agent to extract per-function assertion counts, error coverage, +table-driven usage, and forbidden mock calls from the pre-fetched test diff. It also checks +for missing `//go:build` tags in newly added Go test files. Use its table and build-tag findings +as input to Step 4. Key signals for Go tests in this codebase: - **Assertions (accepted forms)**: @@ -185,22 +178,9 @@ Key signals for Go tests in this codebase: ### 3b. JavaScript โ€” vitest `test()` / `it()` blocks -This codebase uses **vitest** (not jest). Mock helpers come from vitest: `vi.fn()`, `vi.spyOn()`, `vi.mock()`. Primary test file extension is `.test.cjs`; scripts tests use `.test.js`. - -```bash -# Count expect() assertions, error matchers, and vi.* mock calls per test block -git diff ${{ github.event.pull_request.base.sha }}...HEAD -- '*.test.cjs' '*.test.js' | awk ' -/^\+(it|test)\(/ { - if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "mocks=" mocks - match($0, /(it|test)\(["'"'"']([^"'"'"']+)/, arr); test_name=arr[2]; assertions=0; errors=0; mocks=0 -} -test_name && /^\+.*expect\(/ { assertions++ } -test_name && /^\+.*(\.toThrow|\.rejects|\.toThrowError)/ { errors++ } -test_name && /^\+.*(vi\.mock|vi\.spyOn|vi\.fn)/ { mocks++ } -test_name && /^\+\}\)/ { print test_name, "assertions=" assertions, "errors=" errors, "mocks=" mocks; test_name="" } -END { if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "mocks=" mocks } -' -``` +Use the `js-test-analyzer` agent to extract per-test assertion counts, error matcher usage, +and `vi.*` mock calls from the pre-fetched test diff. Covers both `.test.cjs` (primary) and +`.test.js` (scripts) formats. Use its table as input to Step 4. Key signals for JavaScript tests in this codebase: - **Assertions**: `expect(...)` calls with vitest matchers (`.toBe`, `.toEqual`, `.toMatchObject`, `.toContain`, `.toBeNull`, etc.) @@ -254,8 +234,7 @@ Calculate the test inflation ratio for each changed test file: ```bash # Count lines added to test files vs. production files -git diff ${{ github.event.pull_request.base.sha }}...HEAD --stat | grep -E "test|spec" || echo "no test stat" -git diff ${{ github.event.pull_request.base.sha }}...HEAD --numstat +cat /tmp/gh-aw/agent/diff-numstat.txt ``` For each **Go and JavaScript** test file, find the corresponding production file and compare the ratio of lines added: @@ -475,3 +454,73 @@ After posting the comment, submit a pull request review based on the verdict: 2. In the PR comment (Step 7), add a note such as: "โš ๏ธ Sampling applied โ€” analyzed the first 50 of N test functions. Prioritized newly added tests." - Keep individual test analysis concise โ€” 2โ€“3 sentences per test in the flagged section. - Use `
` tags for per-test tables with more than 10 rows. + +## agent: `go-test-analyzer` +--- +description: Run awk analysis on Go test diff and return per-function stats plus missing build tags +model: small +--- +Read the pre-fetched test diff and extract per-function Go test stats: + +```bash +cat /tmp/gh-aw/agent/test-diff.txt | awk ' +/^\+func Test/ { + if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "table_driven=" table_driven, "forbidden_mocks=" forbidden_mocks + match($0, /func (Test[^(]+)/, arr); test_name=arr[1]; assertions=0; errors=0; table_driven=0; forbidden_mocks=0 +} +test_name && /^\+.*(assert\.|require\.)/ { assertions++ } +test_name && /^\+.*t\.(Error|Errorf|Fatal|Fatalf)\(/ { assertions++; errors++ } +test_name && /^\+.*(assert\.Error|require\.Error|assert\.NoError|require\.NoError)/ { errors++ } +test_name && /^\+.*t\.Run\(/ { table_driven++ } +test_name && /^\+.*(gomock\.|testify\/mock|\.EXPECT\(\)|\.On\(|\.Return\()/ { forbidden_mocks++ } +test_name && /^\+\}$/ { print test_name, "assertions=" assertions, "errors=" errors, "table_driven=" table_driven, "forbidden_mocks=" forbidden_mocks; test_name="" } +END { if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "table_driven=" table_driven, "forbidden_mocks=" forbidden_mocks } +' +``` + +Also check for newly added Go test files missing the mandatory build tag: + +```bash +git diff ${{ github.event.pull_request.base.sha }}...HEAD --diff-filter=A --name-only | grep '_test\.go$' | while read f; do + if ! head -1 "$f" | grep -qE '^//go:build'; then + echo "MISSING BUILD TAG: $f" + fi +done +``` + +Return: +1. A markdown table with this exact header: + `| Test Function | Assertions | Error Checks | Table-Driven Subtests | Forbidden Mock Calls |` + Example row: + `| TestCompile | 4 | 2 | 1 | 0 |` +2. A `Missing Build Tags` section listing any `MISSING BUILD TAG: ` lines, or `None.` +3. If no Go test functions are in the diff, return: `No Go test functions found in diff.` + +## agent: `js-test-analyzer` +--- +description: Run awk analysis on JavaScript vitest diff and return per-test stats +model: small +--- +Read the pre-fetched test diff and extract per-test JavaScript vitest stats: + +```bash +cat /tmp/gh-aw/agent/test-diff.txt | awk ' +/^\+(it|test)\(/ { + if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "mocks=" mocks + match($0, /(it|test)\(["'"'"']([^"'"'"']+)/, arr); test_name=arr[2]; assertions=0; errors=0; mocks=0 +} +test_name && /^\+.*expect\(/ { assertions++ } +test_name && /^\+.*(\.toThrow|\.rejects|\.toThrowError)/ { errors++ } +test_name && /^\+.*(vi\.mock|vi\.spyOn|vi\.fn)/ { mocks++ } +test_name && /^\+\}\)/ { print test_name, "assertions=" assertions, "errors=" errors, "mocks=" mocks; test_name="" } +END { if (test_name) print test_name, "assertions=" assertions, "errors=" errors, "mocks=" mocks } +' +``` + +Return a markdown table with this exact header: +`| Test Name | Assertions | Error Matchers | vi.* Mock Calls |` + +Example row: +`| should_validate_input | 3 | 1 | 0 |` + +If no JavaScript test blocks are in the diff, return: `No JavaScript test blocks found in diff.`