diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 92524ea17..178fcd17a 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -14,56 +14,54 @@ jobs: - name: Check out code uses: actions/checkout@v6 with: - # Fetch full history to access merge-base fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v6 + - name: Run MCP Conformance Test + uses: SamMorrowDrums/mcp-conformance-action@v2 with: - go-version-file: "go.mod" + setup_go: "true" + install_command: go mod download + start_command: go run ./cmd/github-mcp-server stdio + env_vars: | + GITHUB_PERSONAL_ACCESS_TOKEN=test-token + configurations: | + [ + {"name": "default", "args": ""}, + {"name": "read-only", "args": "--read-only"}, + {"name": "dynamic-toolsets", "args": "--dynamic-toolsets"}, + {"name": "read-only+dynamic", "args": "--read-only --dynamic-toolsets"}, + {"name": "toolsets-repos", "args": "--toolsets=repos"}, + {"name": "toolsets-issues", "args": "--toolsets=issues"}, + {"name": "toolsets-pull_requests", "args": "--toolsets=pull_requests"}, + {"name": "toolsets-repos,issues", "args": "--toolsets=repos,issues"}, + {"name": "toolsets-all", "args": "--toolsets=all"}, + {"name": "tools-get_me", "args": "--tools=get_me"}, + {"name": "tools-get_me,list_issues", "args": "--tools=get_me,list_issues"}, + {"name": "toolsets-repos+read-only", "args": "--toolsets=repos --read-only"}, + {"name": "toolsets-all+dynamic", "args": "--toolsets=all --dynamic-toolsets"}, + {"name": "toolsets-repos+dynamic", "args": "--toolsets=repos --dynamic-toolsets"}, + {"name": "toolsets-repos,issues+dynamic", "args": "--toolsets=repos,issues --dynamic-toolsets"}, + { + "name": "dynamic-tool-calls", + "args": "--dynamic-toolsets", + "custom_messages": [ + {"id": 10, "name": "list_toolsets_before", "message": {"jsonrpc": "2.0", "id": 10, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}}, + {"id": 11, "name": "get_toolset_tools", "message": {"jsonrpc": "2.0", "id": 11, "method": "tools/call", "params": {"name": "get_toolset_tools", "arguments": {"toolset": "repos"}}}}, + {"id": 12, "name": "enable_toolset", "message": {"jsonrpc": "2.0", "id": 12, "method": "tools/call", "params": {"name": "enable_toolset", "arguments": {"toolset": "repos"}}}}, + {"id": 13, "name": "list_toolsets_after", "message": {"jsonrpc": "2.0", "id": 13, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}} + ] + } + ] - - name: Download dependencies - run: go mod download - - - name: Run conformance test - id: conformance - run: | - # Run conformance test, capture stdout for summary - script/conformance-test > conformance-summary.txt 2>&1 || true - - # Output the summary - cat conformance-summary.txt - - # Check result - if grep -q "RESULT: ALL TESTS PASSED" conformance-summary.txt; then - echo "status=passed" >> $GITHUB_OUTPUT - else - echo "status=differences" >> $GITHUB_OUTPUT - fi - - - name: Generate Job Summary + - name: Add interpretation note + if: always() run: | - # Add the full markdown report to the job summary - echo "# MCP Server Conformance Report" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Comparing PR branch against merge-base with \`origin/main\`" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Extract and append the report content (skip the header since we added our own) - tail -n +5 conformance-report/CONFORMANCE_REPORT.md >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY echo "---" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - # Add interpretation note - if [ "${{ steps.conformance.outputs.status }}" = "passed" ]; then - echo "✅ **All conformance tests passed** - No behavioral differences detected." >> $GITHUB_STEP_SUMMARY - else - echo "⚠️ **Differences detected** - Review the diffs above to ensure changes are intentional." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY - echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY - echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY - echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY - fi + echo "ℹ️ **Note:** Differences may be intentional improvements." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY + echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY + echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY + echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY