Skip to content

gh-136728: Refactor build.yml CI config and multissltests.py #143940

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
hugovk wants to merge 20 commits into
base: main
Choose a base branch
from
+199 −190
Draft

gh-136728: Refactor build.yml CI config and multissltests.py #143940

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
a104f8a
Adapt @hugovk's proof-of-concept CI definition refactor
WillChilds-Klein Jul 14, 2025
3fcbe0d
Preliminary refactor of multissltests.py, TODO migrate to classes
WillChilds-Klein Jul 14, 2025
5d8ec9a
Fix aws-lc/awslc lib name discrepancy
WillChilds-Klein Jul 14, 2025
991c6b2
Migrate AbstractBuilder to abc
WillChilds-Klein Jul 14, 2025
7b51499
Fix imports
WillChilds-Klein Jul 14, 2025
6638127
Complete ABC refactor
WillChilds-Klein Jul 16, 2025
4e0a8ca
Colorize parser
WillChilds-Klein Jul 16, 2025
1a90e0c
Adjust compatibility comment
WillChilds-Klein Jul 16, 2025
1fcb49f
📜🤖 Added by blurb_it.
blurb-it[bot] Jul 16, 2025
dd969fb
Revert "📜🤖 Added by blurb_it."
WillChilds-Klein Jul 16, 2025
1edce1c
Merge branch 'main' into refactor-ssl-tests
WillChilds-Klein Jul 16, 2025
3343120
Include old versions when using default versions
WillChilds-Klein Jul 16, 2025
82f1154
Merge remote-tracking branch 'upstream/main' into 3.15-refactor-ssl-t…
hugovk Jan 16, 2026
0560203
Apply suggestions from code review
hugovk Jan 16, 2026
08be1f2
Bump GitHub Actions
hugovk Jan 16, 2026
b2f353c
Merge remote-tracking branch 'upstream/main' into 3.15-refactor-ssl-t…
hugovk Jan 16, 2026
31325c5
Retain multiline
hugovk Jan 16, 2026
94a826d
Use extend instead of += [...]
hugovk Jan 16, 2026
9113833
Remove 3.3 block, use long options
hugovk Jan 16, 2026
942434a
Apply suggestions from code review
hugovk Jan 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 40 additions & 88 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,78 +248,32 @@ jobs:
free-threading: ${{ matrix.free-threading }}
os: ${{ matrix.os }}

build-ubuntu-ssltests-openssl:
name: 'Ubuntu SSL tests with OpenSSL'
runs-on: ${{ matrix.os }}
build-ubuntu-ssltests:
name: 'Ubuntu SSL tests'
runs-on: ubuntu-24.04
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-ubuntu == 'true'
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04]
# Keep 1.1.1w in our list despite it being upstream EOL and otherwise
# unsupported as it most resembles other 1.1.1-work-a-like ssl APIs
# supported by important vendors such as AWS-LC.
openssl_ver: [1.1.1w, 3.0.18, 3.2.6, 3.3.5, 3.4.3, 3.5.4]
include:
# Keep 1.1.1w in our list despite it being upstream EOL and otherwise
# unsupported as it most resembles other 1.1.1-work-a-like ssl APIs
# supported by important vendors such as AWS-LC.
- { ssl: openssl, ssl_ver: 1.1.1w }
- { ssl: openssl, ssl_ver: 3.0.18 }
- { ssl: openssl, ssl_ver: 3.2.6 }
- { ssl: openssl, ssl_ver: 3.3.5 }
- { ssl: openssl, ssl_ver: 3.4.3 }
- { ssl: openssl, ssl_ver: 3.5.4 }
- { ssl: awslc, ssl_ver: 1.55.0 }
# See Tools/ssl/make_ssl_data.py for notes on adding a new version
env:
OPENSSL_VER: ${{ matrix.openssl_ver }}
MULTISSL_DIR: ${{ github.workspace }}/multissl
OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Runner image version
run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install dependencies
run: sudo ./.github/workflows/posix-deps-apt.sh
- name: Configure OpenSSL env vars
run: |
echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV"
echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV"
- name: 'Restore OpenSSL build'
id: cache-openssl
uses: actions/cache@v5
with:
path: ./multissl/openssl/${{ env.OPENSSL_VER }}
key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
- name: Install OpenSSL
if: steps.cache-openssl.outputs.cache-hit != 'true'
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
- name: Configure CPython
run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR"
- name: Build CPython
run: make -j4
- name: Display build info
run: make pythoninfo
- name: SSL tests
run: ./python Lib/test/ssltests.py

build-ubuntu-ssltests-awslc:
name: 'Ubuntu SSL tests with AWS-LC'
runs-on: ${{ matrix.os }}
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-ubuntu == 'true'
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04]
awslc_ver: [1.55.0]
env:
AWSLC_VER: ${{ matrix.awslc_ver}}
SSL_VER: ${{ matrix.ssl_ver }}
MULTISSL_DIR: ${{ github.workspace }}/multissl
OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}
LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib
SSL_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssl }}/${{ matrix.ssl_ver }}
LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssl }}/${{ matrix.ssl_ver }}/lib
steps:
- uses: actions/checkout@v6
with:
Expand All @@ -333,39 +287,40 @@ jobs:
- name: Configure SSL lib env vars
run: |
echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV"
echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV"
- name: 'Restore AWS-LC build'
id: cache-aws-lc
echo "SSL_DIR=${GITHUB_WORKSPACE}/multissl/${{ matrix.ssl }}/${SSL_VER}" >> "$GITHUB_ENV"
echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/${{ matrix.ssl }}/${SSL_VER}/lib" >> "$GITHUB_ENV"
- name: 'Restore SSL build'
id: cache-ssl
uses: actions/cache@v5
with:
path: ./multissl/aws-lc/${{ matrix.awslc_ver }}
key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }}
- name: Install AWS-LC
if: steps.cache-aws-lc.outputs.cache-hit != 'true'
path: ./multissl/${{ env.SSL }}/${{ env.SSL_VER }}
key: ${{ env.IMAGE_OS_VERSION }}-multissl-${{ env.SSL }}-${{ env.SSL_VER }}
- name: Install SSL
if: steps.cache-ssl.outputs.cache-hit != 'true'
run: |
python3 Tools/ssl/multissltests.py \
--steps=library \
--base-directory "$MULTISSL_DIR" \
--awslc ${{ matrix.awslc_ver }} \
--ssl ${{ matrix.ssl }} \
--ssl-versions ${{ matrix.ssl_ver }} \
--system Linux
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
- name: Configure CPython
run: |
Copy link
Member

@webknjaz webknjaz Jan 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

>- makes EOL escaping redundant while allowing for nice line separation of long/logically self-contained components:

Suggested change
run: |
env:
CONFIGURE_TRAILING_ARGS: >-
${{
matrix.ssl != 'openssl'
&& '--with-builtin-hashlib-hashes=blake2 --with-ssl-default-suites=openssl'
|| ''
}}
run: >-

Copy link
Member

@picnixz picnixz Jan 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer \ becauseI find them easier to read. I find the >- harder to read honestly.

./configure CFLAGS="-fdiagnostics-format=json" \
--config-cache \
--enable-slower-safety \
--with-pydebug \
--with-openssl="$OPENSSL_DIR" \
--with-builtin-hashlib-hashes=blake2 \
--with-ssl-default-suites=openssl
CMD=(./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$SSL_DIR")
if [ "${{ matrix.ssl }}" = "openssl" ]; then
"${CMD[@]}"
else
"${CMD[@]}" --with-builtin-hashlib-hashes=blake2 --with-ssl-default-suites=openssl
fi
Comment on lines +312 to +317
Copy link
Member

@webknjaz webknjaz Jan 16, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
CMD=(./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$SSL_DIR")
if [ "${{ matrix.ssl }}" = "openssl" ]; then
"${CMD[@]}"
else
"${CMD[@]}" --with-builtin-hashlib-hashes=blake2 --with-ssl-default-suites=openssl
fi
./configure
CFLAGS="-fdiagnostics-format=json"
--config-cache
--enable-slower-safety
--with-pydebug
--with-openssl="$SSL_DIR"
${CONFIGURE_TRAILING_ARGS}

Copy link
Member

@picnixz picnixz Jan 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer the if/else approach. I do not like the notion of 'trailing' args.

- name: Build CPython
run: make -j
run: make -j4
- name: Display build info
run: make pythoninfo
- name: Verify python is linked to AWS-LC
if: matrix.ssl == 'aws-lc'
run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC
- name: SSL tests
run: ./python Lib/test/ssltests.py
Expand Down Expand Up @@ -451,7 +406,7 @@ jobs:
key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
- name: Install OpenSSL
if: steps.cache-openssl.outputs.cache-hit != 'true'
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --ssl 'openssl' --ssl-versions "$OPENSSL_VER" --system Linux
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
Expand Down Expand Up @@ -569,7 +524,7 @@ jobs:
key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
- name: Install OpenSSL
if: steps.cache-openssl.outputs.cache-hit != 'true'
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --ssl 'openssl' --ssl-versions "$OPENSSL_VER" --system Linux
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
Expand Down Expand Up @@ -695,8 +650,7 @@ jobs:
- build-windows-msi
- build-macos
- build-ubuntu
- build-ubuntu-ssltests-awslc
- build-ubuntu-ssltests-openssl
- build-ubuntu-ssltests
- build-ios
- build-wasi
- test-hypothesis
Expand All @@ -713,8 +667,7 @@ jobs:
allowed-failures: >-
build-android,
build-windows-msi,
build-ubuntu-ssltests-awslc,
build-ubuntu-ssltests-openssl,
build-ubuntu-ssltests,
test-hypothesis,
cifuzz,
allowed-skips: >-
Expand Down Expand Up @@ -745,8 +698,7 @@ jobs:
!fromJSON(needs.build-context.outputs.run-ubuntu)
&& '
build-ubuntu,
build-ubuntu-ssltests-awslc,
build-ubuntu-ssltests-openssl,
build-ubuntu-ssltests,
test-hypothesis,
build-asan,
build-san,
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/reusable-ubuntu.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ jobs:
key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
- name: Install OpenSSL
if: steps.cache-openssl.outputs.cache-hit != 'true'
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --ssl openssl --ssl-versions "$OPENSSL_VER" --system Linux
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
Expand Down
Loading
Loading