python-sdk/.github/workflows/conformance.yml at main · modelcontextprotocol/python-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
name: Conformance Tests

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:

concurrency:
  group: conformance-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  # Pinned conformance harness package spec (passed verbatim to `npx --yes`).
  # Bump deliberately and reconcile the three
  # .github/actions/conformance/expected-failures*.yml files in the same change.
  CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"

# Each job runs three `--suite all` legs, one per baseline file. The two
# `--spec-version <rev>` legs run every scenario the pinned harness marks
# applicable at that spec revision, at that revision's wire (stateful
# handshake for 2025-11-25, stateless per-request _meta for 2026-07-28). Each
# is a superset of the frozen requirement set that `conformance tier-check`
# scores for that revision (`npx $CONFORMANCE_PKG list --requirements <rev>`
# prints it), so an expected-failures.<rev>.yml entry for a scenario in that
# set means tier-check reports the SDK below 100% for <rev>. The bare leg runs
# every shipped scenario at whichever wire the harness defaults it to; it is
# the only leg that reaches extension-tagged scenarios (tasks-*, auth/dpop*,
# ...), which never match a --spec-version filter. For everything else it
# overlaps one of the pinned legs, so a failure it also sees needs its own
# expected-failures.yml entry besides the pinned file's.

jobs:
  server-conformance:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          enable-cache: true
          version: 0.12.24
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: 24
      - run: uv sync --frozen --all-extras --package mcp-everything-server
      - name: Run server conformance (all suite, 2025-11-25 wire)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --spec-version 2025-11-25
          --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml
          --output-dir conformance-results/server-2025-11-25
      - name: Run server conformance (all suite, 2026-07-28 wire)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
          --output-dir conformance-results/server-2026-07-28
      - name: Run server conformance (all suite, default wire)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/server-all
      - name: Upload conformance results
        # The log has only summary counts; per-check data is in checks.json.
        # always(), not !cancelled(): a timeout-minutes kill is a cancellation,
        # and that is when the partial results matter most.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: server-conformance-results
          path: conformance-results/
          if-no-files-found: ignore

  client-conformance:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          enable-cache: true
          version: 0.12.24
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: 24
      # --compile-bytecode: without it, ~40 concurrently spawned interpreters
      # race to byte-compile site-packages during the timing-sensitive window.
      - run: uv sync --frozen --all-extras --package mcp --compile-bytecode
      - name: Pre-compile bytecode (editable sources)
        run: uv run --frozen python -m compileall -q src .github/actions/conformance
      # The harness runs a leg's scenarios via unbounded Promise.all; under
      # that concurrency the slowest one (sse-retry, which has a real-time SSE
      # reconnect wait) needs more than the 30s default budget.
      # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40
      # concurrent spawns; run-client.sh re-runs unexpected failures solo.
      - name: Run client conformance (all suite, 2025-11-25 wire)
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --spec-version 2025-11-25
          --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml
          --output-dir conformance-results/client-2025-11-25
      - name: Run client conformance (all suite, 2026-07-28 wire)
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
          --output-dir conformance-results/client-2026-07-28
      - name: Run client conformance (all suite, default wire)
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/client-all
      - name: Upload conformance results
        # The log has only summary counts; per-check data is in checks.json.
        # always(), not !cancelled(): a timeout-minutes kill is a cancellation,
        # and that is when the partial results matter most.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: client-conformance-results
          path: conformance-results/
          if-no-files-found: ignore