Skip to content

Commit d456108

Browse files
colesburycolesbury
authored
Support building free-threaded CPython (#319)
* Support building free-threaded CPython Add support for Python's free threading build mode where the global interpreter lock is disabled. The packages are marked using a suffix on the architecture, like 'x64-freethreaded' or 'arm64-freethreaded'. * Match '-freethreaded' in arch * Use type 'string' instead of 'str' * On Linux, only delete Python installations with the same architecture. This matches the macOS behavior and allows users to install both the free-threading and default builds at the same time.
1 parent e550a75 commit d456108

14 files changed

+176
-39
lines changed

‎.github/workflows/build-python-packages.yml‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ on:
1212
required: true
1313
type: boolean
1414
default: false
15+
THREADING_BUILD_MODES:
16+
description: 'CPython threading build modes'
17+
required: true
18+
type: string
19+
default: 'default,freethreaded'
1520
PLATFORMS:
1621
description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
1722
required: true
@@ -40,32 +45,42 @@ jobs:
4045
id: generate-matrix
4146
run: |
4247
[String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim()
48+
[String[]]$buildModes = "${{ inputs.threading_build_modes || 'default' }}".Split(",").Trim()
4349
$matrix = @()
4450
4551
foreach ($configuration in $configurations) {
46-
$parts = $configuration.Split("_")
47-
$os = $parts[0]
48-
$arch = if ($parts[1]) {$parts[1]} else {"x64"}
49-
switch -wildcard ($os) {
50-
"*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
51-
"*macos*" { $platform = 'darwin' }
52-
"*windows*" { $platform = 'win32' }
53-
}
54-
55-
if ($configuration -eq "ubuntu-22.04_arm64") {
56-
$os = "setup-actions-ubuntu-arm64-2-core"
57-
}
58-
elseif ($configuration -eq "ubuntu-24.04_arm64") {
59-
$os = "setup-actions-ubuntu24-arm64-2-core"
60-
}
61-
elseif ($configuration -eq "windows-2019_arm64") {
62-
$os = "setup-actions-windows-arm64-4-core"
63-
}
64-
65-
$matrix += @{
66-
'platform' = $platform
67-
'os' = $os
68-
'arch' = $arch
52+
foreach ($buildMode in $buildModes) {
53+
$parts = $configuration.Split("_")
54+
$os = $parts[0]
55+
$arch = if ($parts[1]) {$parts[1]} else {"x64"}
56+
switch -wildcard ($os) {
57+
"*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
58+
"*macos*" { $platform = 'darwin' }
59+
"*windows*" { $platform = 'win32' }
60+
}
61+
62+
if ($configuration -eq "ubuntu-22.04_arm64") {
63+
$os = "setup-actions-ubuntu-arm64-2-core"
64+
}
65+
elseif ($configuration -eq "ubuntu-24.04_arm64") {
66+
$os = "setup-actions-ubuntu24-arm64-2-core"
67+
}
68+
elseif ($configuration -eq "windows-2019_arm64") {
69+
$os = "setup-actions-windows-arm64-4-core"
70+
}
71+
72+
if ($buildMode -eq "freethreaded") {
73+
if ([semver]"${{ inputs.VERSION }}" -lt [semver]"3.13.0") {
74+
continue;
75+
}
76+
$arch += "-freethreaded"
77+
}
78+
79+
$matrix += @{
80+
'platform' = $platform
81+
'os' = $os
82+
'arch' = $arch
83+
}
6984
}
7085
}
7186
echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
@@ -205,6 +220,9 @@ jobs:
205220
python-version: ${{ env.VERSION }}
206221
architecture: ${{ matrix.arch }}
207222

223+
- name: Python version
224+
run: python -VVV
225+
208226
- name: Verbose sysconfig dump
209227
if: runner.os == 'Linux' || runner.os == 'macOS'
210228
run: python ./sources/python-config-output.py

‎builders/macos-python-builder.psm1‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,37 @@ class macOSPythonBuilder : NixPythonBuilder {
151151
return $pkgLocation
152152
}
153153

154+
[string] GetFrameworkName() {
155+
<#
156+
.SYNOPSIS
157+
Get the Python installation Package name.
158+
#>
159+
160+
if ($this.IsFreeThreaded()) {
161+
return "PythonT.framework"
162+
} else {
163+
return "Python.framework"
164+
}
165+
}
166+
167+
[string] GetPkgChoices() {
168+
<#
169+
.SYNOPSIS
170+
Reads the configuration XML file for the Python installer
171+
#>
172+
173+
$config = if ($this.IsFreeThreaded()) { "freethreaded" } else { "default" }
174+
$choicesFile = Join-Path $PSScriptRoot "../config/macos-pkg-choices-$($config).xml"
175+
$choicesTemplate = Get-Content -Path $choicesFile -Raw
176+
177+
$variablesToReplace = @{
178+
"{{__VERSION_MAJOR_MINOR__}}" = "$($this.Version.Major).$($this.Version.Minor)";
179+
}
180+
181+
$variablesToReplace.keys | ForEach-Object { $choicesTemplate = $choicesTemplate.Replace($_, $variablesToReplace[$_]) }
182+
return $choicesTemplate
183+
}
184+
154185
[void] CreateInstallationScriptPkg() {
155186
<#
156187
.SYNOPSIS
@@ -165,6 +196,8 @@ class macOSPythonBuilder : NixPythonBuilder {
165196
"{{__VERSION_FULL__}}" = $this.Version;
166197
"{{__PKG_NAME__}}" = $this.GetPkgName();
167198
"{{__ARCH__}}" = $this.Architecture;
199+
"{{__FRAMEWORK_NAME__}}" = $this.GetFrameworkName();
200+
"{{__PKG_CHOICES__}}" = $this.GetPkgChoices();
168201
}
169202

170203
$variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }

‎builders/nix-python-builder.psm1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ class NixPythonBuilder : PythonBuilder {
115115
Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
116116
$buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
117117

118-
Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
118+
Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
119119
Execute-Command -Command "make install" -ErrorAction Continue
120120

121121
Write-Debug "Done; Make log location: $buildOutputLocation"

‎builders/python-builder.psm1‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,24 @@ class PythonBuilder {
9494
return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
9595
}
9696

97+
[string] GetHardwareArchitecture() {
98+
<#
99+
.SYNOPSIS
100+
The hardware architecture (x64, arm64) without any Python free threading suffix.
101+
#>
102+
103+
return $this.Architecture.Replace("-freethreaded", "")
104+
}
105+
106+
[bool] IsFreeThreaded() {
107+
<#
108+
.SYNOPSIS
109+
Check if Python version is free threaded.
110+
#>
111+
112+
return $this.Architecture.EndsWith("-freethreaded")
113+
}
114+
97115
[void] PreparePythonToolcacheLocation() {
98116
<#
99117
.SYNOPSIS

‎builders/ubuntu-python-builder.psm1‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@ class UbuntuPythonBuilder : NixPythonBuilder {
3737
$configureString += " --enable-shared"
3838
$configureString += " --enable-optimizations"
3939

40+
if ($this.IsFreeThreaded()) {
41+
if ($this.Version -lt "3.13.0") {
42+
Write-Host "Python versions lower than 3.13.0 do not support free threading"
43+
exit 1
44+
}
45+
$configureString += " --disable-gil"
46+
}
47+
4048
### Compile with support of loadable sqlite extensions.
4149
### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
4250
$configureString += " --enable-loadable-sqlite-extensions"

‎builders/win-python-builder.psm1‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@ class WinPythonBuilder : PythonBuilder {
5454
#>
5555

5656
$ArchitectureExtension = ""
57-
if ($this.Architecture -eq "x64") {
57+
if ($this.GetHardwareArchitecture() -eq "x64") {
5858
if ($this.Version -ge "3.5") {
5959
$ArchitectureExtension = "-amd64"
6060
} else {
6161
$ArchitectureExtension = ".amd64"
6262
}
63-
}elseif ($this.Architecture -eq "arm64") {
63+
} elseif ($this.GetHardwareArchitecture() -eq "arm64") {
6464
$ArchitectureExtension = "-arm64"
6565
}
6666

@@ -113,6 +113,7 @@ class WinPythonBuilder : PythonBuilder {
113113

114114
$variablesToReplace = @{
115115
"{{__ARCHITECTURE__}}" = $this.Architecture;
116+
"{{__HARDWARE_ARCHITECTURE__}}" = $this.GetHardwareArchitecture();
116117
"{{__VERSION__}}" = $this.Version;
117118
"{{__PYTHON_EXEC_NAME__}}" = $pythonExecName
118119
}

‎config/macos-pkg-choices-default.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3+
<plist version="1.0">
4+
<array>
5+
<dict>
6+
</dict>
7+
</array>
8+
</plist>

‎config/macos-pkg-choices-freethreaded.xml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3+
<plist version="1.0">
4+
<array>
5+
<dict>
6+
<key>attributeSetting</key>
7+
<integer>1</integer>
8+
<key>choiceAttribute</key>
9+
<string>selected</string>
10+
<key>choiceIdentifier</key>
11+
<string>org.python.Python.PythonTFramework-{{__VERSION_MAJOR_MINOR__}}</string>
12+
</dict>
13+
</array>
14+
</plist>

‎config/python-manifest-config.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
2+
"regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+(-freethreaded)?)",
33
"groups": {
44
"arch": 4,
55
"platform": 2,

‎installers/macos-pkg-setup-template.sh‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ set -e
22

33
PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
44
PYTHON_PKG_NAME="{{__PKG_NAME__}}"
5+
PYTHON_FRAMEWORK_NAME="{{__FRAMEWORK_NAME__}}"
6+
PYTHON_PKG_CHOICES=$(cat << 'EOF'
7+
{{__PKG_CHOICES__}}
8+
EOF
9+
)
510
ARCH="{{__ARCH__}}"
611
MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
712
MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
@@ -20,7 +25,7 @@ fi
2025
PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
2126
PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
2227
PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
23-
PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
28+
PYTHON_FRAMEWORK_PATH="/Library/Frameworks/${PYTHON_FRAMEWORK_NAME}/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
2429
PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
2530

2631
echo "Check if Python hostedtoolcache folder exist..."
@@ -38,8 +43,11 @@ else
3843
done
3944
fi
4045

46+
PYTHON_PKG_CHOICES_FILES=$(mktemp)
47+
echo "$PYTHON_PKG_CHOICES" > $PYTHON_PKG_CHOICES_FILES
48+
4149
echo "Install Python binaries from prebuilt package"
42-
sudo installer -pkg $PYTHON_PKG_NAME -target /
50+
sudo installer -pkg $PYTHON_PKG_NAME -applyChoiceChangesXML $PYTHON_PKG_CHOICES_FILES -target /
4351

4452
echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
4553
echo "Create Python $PYTHON_FULL_VERSION folder"
@@ -53,7 +61,9 @@ ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
5361

5462
echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
5563
ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
64+
chmod +x python
5665

66+
# Note that bin is a symlink so referencing .. from bin will not work as expected
5767
cd bin/
5868

5969
# This symlink already exists if Python version with the same major.minor version is installed,
@@ -62,11 +72,15 @@ if [ ! -f $PYTHON_MAJOR_MINOR ]; then
6272
ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
6373
fi
6474

75+
if [ ! -f $PYTHON_MAJOR ]; then
76+
ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR
77+
fi
78+
6579
if [ ! -f python ]; then
6680
ln -s $PYTHON_MAJOR_DOT_MINOR python
6781
fi
6882

69-
chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
83+
chmod +x $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
7084

7185
echo "Upgrading pip..."
7286
export PIP_ROOT_USER_ACTION=ignore

0 commit comments

Comments
 (0)