From 98bebab4b9335d5f20b4a0833b3cead70e832f6c Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 13:32:25 +0100 Subject: [PATCH 01/18] ci: use mcp-conformance-action for conformance tests Replace custom script/conformance-test with the reusable mcp-conformance-action GitHub Action. This provides: - Standardized MCP conformance testing across all MCP servers - Automatic comparison between PR branch and merge-base - Support for multiple configurations with different flags - Custom message support for dynamic toolset testing - Detailed conformance reports with timing comparisons - Artifact upload for test results All 16 original test configurations are preserved including the dynamic tool call tests for toolset management. Action: https://github.com/marketplace/actions/mcp-conformance-test Related: #1826 --- .github/workflows/conformance.yml | 91 +++++++++++++++---------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 92524ea17..4d6d1e7de 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -12,58 +12,57 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - # Fetch full history to access merge-base fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v6 + - name: Run MCP Conformance Test + uses: SamMorrowDrums/mcp-conformance-action@v1 with: - go-version-file: "go.mod" + setup_go: "true" + go_version_file: go.mod + install_command: go mod download + server_command: go run ./cmd/github-mcp-server stdio + server_env: | + GITHUB_PERSONAL_ACCESS_TOKEN=test-token + configurations: | + [ + {"name": "default", "args": ""}, + {"name": "read-only", "args": "--read-only"}, + {"name": "dynamic-toolsets", "args": "--dynamic-toolsets"}, + {"name": "read-only+dynamic", "args": "--read-only --dynamic-toolsets"}, + {"name": "toolsets-repos", "args": "--toolsets=repos"}, + {"name": "toolsets-issues", "args": "--toolsets=issues"}, + {"name": "toolsets-pull_requests", "args": "--toolsets=pull_requests"}, + {"name": "toolsets-repos,issues", "args": "--toolsets=repos,issues"}, + {"name": "toolsets-all", "args": "--toolsets=all"}, + {"name": "tools-get_me", "args": "--tools=get_me"}, + {"name": "tools-get_me,list_issues", "args": "--tools=get_me,list_issues"}, + {"name": "toolsets-repos+read-only", "args": "--toolsets=repos --read-only"}, + {"name": "toolsets-all+dynamic", "args": "--toolsets=all --dynamic-toolsets"}, + {"name": "toolsets-repos+dynamic", "args": "--toolsets=repos --dynamic-toolsets"}, + {"name": "toolsets-repos,issues+dynamic", "args": "--toolsets=repos,issues --dynamic-toolsets"}, + { + "name": "dynamic-tool-calls", + "args": "--dynamic-toolsets", + "custom_messages": [ + {"id": 10, "name": "list_toolsets_before", "message": {"jsonrpc": "2.0", "id": 10, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}}, + {"id": 11, "name": "get_toolset_tools", "message": {"jsonrpc": "2.0", "id": 11, "method": "tools/call", "params": {"name": "get_toolset_tools", "arguments": {"toolset": "repos"}}}}, + {"id": 12, "name": "enable_toolset", "message": {"jsonrpc": "2.0", "id": 12, "method": "tools/call", "params": {"name": "enable_toolset", "arguments": {"toolset": "repos"}}}}, + {"id": 13, "name": "list_toolsets_after", "message": {"jsonrpc": "2.0", "id": 13, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}} + ] + } + ] - - name: Download dependencies - run: go mod download - - - name: Run conformance test - id: conformance - run: | - # Run conformance test, capture stdout for summary - script/conformance-test > conformance-summary.txt 2>&1 || true - - # Output the summary - cat conformance-summary.txt - - # Check result - if grep -q "RESULT: ALL TESTS PASSED" conformance-summary.txt; then - echo "status=passed" >> $GITHUB_OUTPUT - else - echo "status=differences" >> $GITHUB_OUTPUT - fi - - - name: Generate Job Summary + - name: Add interpretation note + if: always() run: | - # Add the full markdown report to the job summary - echo "# MCP Server Conformance Report" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Comparing PR branch against merge-base with \`origin/main\`" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Extract and append the report content (skip the header since we added our own) - tail -n +5 conformance-report/CONFORMANCE_REPORT.md >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY echo "---" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - # Add interpretation note - if [ "${{ steps.conformance.outputs.status }}" = "passed" ]; then - echo "✅ **All conformance tests passed** - No behavioral differences detected." >> $GITHUB_STEP_SUMMARY - else - echo "⚠️ **Differences detected** - Review the diffs above to ensure changes are intentional." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY - echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY - echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY - echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY - fi + echo "ℹ️ **Note:** Differences may be intentional improvements." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY + echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY + echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY + echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY From 7bdb5c59619d3cc8a162c289735afb8e756d60d2 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 14:30:57 +0100 Subject: [PATCH 02/18] fix: use correct action input names --- .github/workflows/conformance.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 4d6d1e7de..6618c6e91 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -20,10 +20,9 @@ jobs: uses: SamMorrowDrums/mcp-conformance-action@v1 with: setup_go: "true" - go_version_file: go.mod install_command: go mod download - server_command: go run ./cmd/github-mcp-server stdio - server_env: | + start_command: go run ./cmd/github-mcp-server stdio + env_vars: | GITHUB_PERSONAL_ACCESS_TOKEN=test-token configurations: | [ From cccd7d831f906467bd23dcd1e015371440a3e8fd Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 14:36:35 +0100 Subject: [PATCH 03/18] fix: address review feedback - Use checkout@v6 for consistency with other workflows - Pin mcp-conformance-action to SHA for security --- .github/workflows/conformance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6618c6e91..66ede1988 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -12,12 +12,12 @@ jobs: steps: - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@v1 + uses: SamMorrowDrums/mcp-conformance-action@b5c958a5a80d5708b68dec3b69a8c1bb8a5189df # v1 with: setup_go: "true" install_command: go mod download From 0320a366f8c3f95dfb4b1da8e88f86713563cdeb Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 14:41:05 +0100 Subject: [PATCH 04/18] fix: update action SHA to include subshell fix --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 66ede1988..9c08dbe5e 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@b5c958a5a80d5708b68dec3b69a8c1bb8a5189df # v1 + uses: SamMorrowDrums/mcp-conformance-action@6377b0dbb4525c86562355d2035aff751d0083ea # v1 with: setup_go: "true" install_command: go mod download From 0b907200f1334e81fadea6f730aec971038a816a Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 14:44:34 +0100 Subject: [PATCH 05/18] fix: update action SHA to include stderr logging --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 9c08dbe5e..f3332aec8 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@6377b0dbb4525c86562355d2035aff751d0083ea # v1 + uses: SamMorrowDrums/mcp-conformance-action@e431538d426a6668adbb5bd8f79798bbee192200 # v1 with: setup_go: "true" install_command: go mod download From f07d13a2ed977d6a15e7546cb37a2dd9fdb32f72 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 14:45:44 +0100 Subject: [PATCH 06/18] chore: use immutable semver tag v1.4.0 --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f3332aec8..5af35b694 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@e431538d426a6668adbb5bd8f79798bbee192200 # v1 + uses: SamMorrowDrums/mcp-conformance-action@e431538d426a6668adbb5bd8f79798bbee192200 # v1.4.0 with: setup_go: "true" install_command: go mod download From 85d7abc18c3645bb2aa4626bf77e89e471430853 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 14:53:03 +0100 Subject: [PATCH 07/18] chore: update mcp-conformance-action to v1.4.1 --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 5af35b694..be01b06be 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@e431538d426a6668adbb5bd8f79798bbee192200 # v1.4.0 + uses: SamMorrowDrums/mcp-conformance-action@9778f25e4227644224e58320e9f9c0f6d79ba687 # v1.4.1 with: setup_go: "true" install_command: go mod download From 3699703bb18d053d108466458d401a26b073dfa8 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 14:57:28 +0100 Subject: [PATCH 08/18] chore: update mcp-conformance-action to v1.4.2 Fixes step summary size limit issue by excluding full stderr logs from the markdown report. Logs are still available in the uploaded artifact. --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index be01b06be..1797f0b8b 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@9778f25e4227644224e58320e9f9c0f6d79ba687 # v1.4.1 + uses: SamMorrowDrums/mcp-conformance-action@288f0373655a2952a785c45dcf3bafc8f6cd592b # v1.4.2 with: setup_go: "true" install_command: go mod download From f3684158d915d6680921b9fdbae98dbccdecf785 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 15:01:59 +0100 Subject: [PATCH 09/18] chore: update mcp-conformance-action to v1.4.3 Removes embedded JSON schemas from report to stay under GitHub's 1MB step summary limit. Full data is still available in the artifact. --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 1797f0b8b..f289c36e4 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@288f0373655a2952a785c45dcf3bafc8f6cd592b # v1.4.2 + uses: SamMorrowDrums/mcp-conformance-action@3a3e7108e20d87c071081d45848b4969d380fcee # v1.4.3 with: setup_go: "true" install_command: go mod download From 12820be77674b0322dbcf9435d4a4525f5c3c4da Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 15:09:09 +0100 Subject: [PATCH 10/18] chore: update mcp-conformance-action to v1.4.4 Removes stderr log capture entirely for cleaner output and smaller artifacts. --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f289c36e4..47903067d 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@3a3e7108e20d87c071081d45848b4969d380fcee # v1.4.3 + uses: SamMorrowDrums/mcp-conformance-action@d1cc7dd57d8a7e93e32e7abcf6ef74a812f2bcf6 # v1.4.4 with: setup_go: "true" install_command: go mod download From 433513dffa46a4aecf1644622fc42b54fdc7cec5 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 15:24:25 +0100 Subject: [PATCH 11/18] chore: update mcp-conformance-action to v1.4.6 Uses named pipes for reliable server startup detection instead of a blind sleep, fixing intermittent failures on slow startup. --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 47903067d..6e4761830 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@d1cc7dd57d8a7e93e32e7abcf6ef74a812f2bcf6 # v1.4.4 + uses: SamMorrowDrums/mcp-conformance-action@1ee2bba49a2c6e6a4f0e6711506b5038d559a1c1 # v1.4.6 with: setup_go: "true" install_command: go mod download From 45e9828c33c8fe7e96a3d1e2073b69f81177c785 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 21:54:36 +0100 Subject: [PATCH 12/18] ci: update to mcp-conformance-action@v2.0.0 --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6e4761830..20d952929 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@1ee2bba49a2c6e6a4f0e6711506b5038d559a1c1 # v1.4.6 + uses: SamMorrowDrums/mcp-conformance-action@3ab61b5ceacc9f3345c26bd42fe4b981fb796309 # v2.0.0 with: setup_go: "true" install_command: go mod download From 21d3a467f8ee62969448dbf7ee2ddbd9d14f857a Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 23:39:28 +0100 Subject: [PATCH 13/18] chore: update conformance action to v2.0.1 with transport fix --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 20d952929..f30750687 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@3ab61b5ceacc9f3345c26bd42fe4b981fb796309 # v2.0.0 + uses: SamMorrowDrums/mcp-conformance-action@2fe7a3c344b27c60732f524ca4dd99dcfe953c1d # v2.0.1 with: setup_go: "true" install_command: go mod download From 6afbe7a189f74df70f579ff4405ed7801acb3c7c Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Fri, 16 Jan 2026 23:47:43 +0100 Subject: [PATCH 14/18] chore: update conformance action to v2.0.2 with Zod schema fix Fixes custom message validation error: 'v3Schema.safeParse is not a function' --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f30750687..8e3283c06 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@2fe7a3c344b27c60732f524ca4dd99dcfe953c1d # v2.0.1 + uses: SamMorrowDrums/mcp-conformance-action@616529364beb374fa4da7f81f26932fa5a70ce97 # v2.0.2 with: setup_go: "true" install_command: go mod download From eefdecb8bd8ca928c413ec9d02a4dcbb12a740bb Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Sat, 17 Jan 2026 09:24:11 +0100 Subject: [PATCH 15/18] ci: upgrade mcp-conformance-action to v2.2.0 v2.2.0 adds automatic HTTP server lifecycle management and improved documentation for transport configuration. --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 8e3283c06..510efb40e 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@616529364beb374fa4da7f81f26932fa5a70ce97 # v2.0.2 + uses: SamMorrowDrums/mcp-conformance-action@25cdd5c265f1ac00ff4d9098b344864932167054 # v2.2.0 with: setup_go: "true" install_command: go mod download From c81355835fdd3585fcdca928ba2156f545334c06 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Sat, 17 Jan 2026 09:35:33 +0100 Subject: [PATCH 16/18] chore: update mcp-conformance-action to v2.3.0 --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 510efb40e..b82eeb0f5 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@25cdd5c265f1ac00ff4d9098b344864932167054 # v2.2.0 + uses: SamMorrowDrums/mcp-conformance-action@v2.3.0 with: setup_go: "true" install_command: go mod download From c3074cc615cb20b2671fe3eb6e516b4b8bd92d70 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Sat, 17 Jan 2026 15:29:37 +0100 Subject: [PATCH 17/18] chore: update mcp-conformance-action to v2.3.2 --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index b82eeb0f5..f18fb2198 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@v2.3.0 + uses: SamMorrowDrums/mcp-conformance-action@v2.3.2 with: setup_go: "true" install_command: go mod download From f0d35bedbf4d461dfc491ac6b58f0a82b89f89a0 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Sun, 18 Jan 2026 01:00:07 +0100 Subject: [PATCH 18/18] chore: update mcp-conformance-action to v2 for security fixes --- .github/workflows/conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f18fb2198..178fcd17a 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - name: Run MCP Conformance Test - uses: SamMorrowDrums/mcp-conformance-action@v2.3.2 + uses: SamMorrowDrums/mcp-conformance-action@v2 with: setup_go: "true" install_command: go mod download