Skip to content

Commit 9ff87c5

Browse files
authored
Split the install-time fix-up out of gather into fixup.py (#48)
`gather --patch` and `--origin` replaced the placeholder prefix at build time, standing in for what an installer would do. This extracts the "install step" into a standalone fixup script that can be included in the distro. `gather` now leaves the placeholder in, so the gathered tree isn't tied to a location. `fixup.py` replaces it once the tree is where it will live. It runs on any Python 3.12+ other than the one in the tree, and fails if no text file in the tree has the placeholder One deliberate difference from `--patch`: `fixup.py` only rewrites text files. Binaries are left alone, so their signatures stay valid, and it lists the ones that still contain the placeholder (today the interpreter and the `.pyc` files) `test` gathers, moves the tree, runs `fixup.py` and then checks it. `--no-fixup` skips that step, which is what a build that needs no install-time step would have to pass. CI runs `test`, `test --no-fixup` and, on Linux, `test --relative-rpath`. The Build artifacts workflow uploads one tarball per platform with the tree and `fixup.py`, instead of trees patched for the runner's paths. On a moved tree, `fixup.py` turns `runtime.paths` and `metadata.build-details` from FAIL to PASS on both platforms. `metadata.prefix-leaks` still fails (the interpreter has the placeholder compiled in), and so does `developer.python-config` on macOS (a shebang can't hold a path with spaces).
1 parent 1278647 commit 9ff87c5

4 files changed

Lines changed: 186 additions & 130 deletions

File tree

‎.github/workflows/reusable-build.yml‎

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,20 @@ jobs:
8080
working-directory: cpython
8181
run: python ../prebuilt-cpython/relocatable.py make
8282

83-
- name: Test (patch)
83+
- name: Test
8484
working-directory: cpython
8585
continue-on-error: true
86-
run: python ../prebuilt-cpython/relocatable.py test --strategy patch --format json > ../results-patch.json
87-
- name: Test (origin)
86+
run: python ../prebuilt-cpython/relocatable.py test --format json > ../results.json
87+
# `--relative-rpath` only changes ELF files.
88+
- name: Test (relative rpath)
89+
if: runner.os == 'Linux'
90+
working-directory: cpython
91+
continue-on-error: true
92+
run: python ../prebuilt-cpython/relocatable.py test --relative-rpath --format json > ../results-relative-rpath.json
93+
- name: Test (no fixup)
8894
working-directory: cpython
8995
continue-on-error: true
90-
run: python ../prebuilt-cpython/relocatable.py test --strategy origin --format json > ../results-origin.json
96+
run: python ../prebuilt-cpython/relocatable.py test --no-fixup --format json > ../results-no-fixup.json
9197

9298
- name: Summarize
9399
shell: python
@@ -97,11 +103,14 @@ jobs:
97103
colours = {"PASS": "🟢", "FAIL": "🔴", "SKIP": "🟡"}
98104
99105
lines = []
100-
for strategy in ("patch", "origin"):
106+
tables = {"test": "results.json", "test --no-fixup": "results-no-fixup.json"}
107+
if os.environ["RUNNER_OS"] == "Linux":
108+
tables["test --relative-rpath"] = "results-relative-rpath.json"
109+
for command, filename in tables.items():
101110
# Fails the job if the harness crashed instead of reporting.
102-
with open(f"results-{strategy}.json") as file:
111+
with open(filename) as file:
103112
results = json.load(file)
104-
lines += [f"### `--strategy {strategy}`", "", "| Check | Status | Summary |", "|---|---|---|"]
113+
lines += [f"### `{command}`", "", "| Check | Status | Summary |", "|---|---|---|"]
105114
lines += [f"| `{r['name']}` | {colours[r['status']]} {r['status']} | {r['summary']} |" for r in results]
106115
lines.append("")
107116
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
@@ -110,31 +119,23 @@ jobs:
110119
if: ${{ !cancelled() }}
111120
with:
112121
name: relocatability-${{ matrix.os }}
113-
path: results-*.json
122+
path: results*.json
114123
if-no-files-found: ignore
115124

116-
# `gather --origin` is Linux only. The tarballs keep the symlinks and
117-
# file modes that the artifact zip drops.
125+
# fixup.py goes with the tree. The tarball keeps the symlinks and file
126+
# modes that the artifact zip drops.
118127
- name: Gather
119128
if: inputs.artifacts
120129
working-directory: cpython
121130
env:
122131
NAME: ${{ steps.commit.outputs.name }}
123-
STRATEGIES: ${{ runner.os == 'Linux' && 'patch origin' || 'patch' }}
124132
run: |
125-
for strategy in $STRATEGIES; do
126-
python ../prebuilt-cpython/relocatable.py gather --output "$GITHUB_WORKSPACE/python" "--$strategy"
127-
tar -C "$GITHUB_WORKSPACE" -czf "$GITHUB_WORKSPACE/$NAME-$strategy.tar.gz" python
128-
done
133+
python ../prebuilt-cpython/relocatable.py gather --output "$GITHUB_WORKSPACE/$NAME/python"
134+
cp ../prebuilt-cpython/fixup.py "$GITHUB_WORKSPACE/$NAME/"
135+
tar -C "$GITHUB_WORKSPACE" -czf "$GITHUB_WORKSPACE/$NAME.tar.gz" "$NAME"
129136
- uses: actions/upload-artifact@v7
130137
if: inputs.artifacts
131138
with:
132-
name: ${{ steps.commit.outputs.name }}-patch
133-
path: ${{ steps.commit.outputs.name }}-patch.tar.gz
134-
if-no-files-found: error
135-
- uses: actions/upload-artifact@v7
136-
if: inputs.artifacts && runner.os == 'Linux'
137-
with:
138-
name: ${{ steps.commit.outputs.name }}-origin
139-
path: ${{ steps.commit.outputs.name }}-origin.tar.gz
139+
name: ${{ steps.commit.outputs.name }}
140+
path: ${{ steps.commit.outputs.name }}.tar.gz
140141
if-no-files-found: error

‎README.md‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ It's broken up into subcommands to configure, make, gather, and test a relocatab
3030
```shell
3131
../prebuilt-cpython/relocatable.py configure
3232
../prebuilt-cpython/relocatable.py make
33-
../prebuilt-cpython/relocatable.py gather <patching approach>
33+
../prebuilt-cpython/relocatable.py gather
3434
../prebuilt-cpython/relocatable.py test
3535
```
3636

@@ -45,16 +45,21 @@ Some stdlib extension modules need third-party libraries that macOS doesn't prov
4545

4646
`deps` downloads each source archive from [cpython-source-deps](https://github.com/python/cpython-source-deps), checks it, and installs only static libraries into deps/install, so the extension modules don't link against anything outside the installation. OpenSSL still has compiled-in paths for its configuration and modules under deps/install. Use `--deps-dir DIR` to put them somewhere else.
4747

48-
The key thing to note is which patching approach to specify with `relocatable.py gather`. There are:
48+
`configure` uses a placeholder `--prefix`. `gather` runs CPython's top-level `make install ENSUREPIP=no` into a staging `DESTDIR` and moves the installed prefix into place, so the gathered tree contains everything a normal install provides (such as `build-details.json` and the unversioned `python3` links). The tree still has the placeholder in it, so it isn't tied to a location and can be packed up as is.
4949

50-
- `--patch` which rewrites all found paths of the placeholder `--prefix` argument to the target install directory (which defaults to dist/); this is meant to follow the approach used by conda.
51-
- `--origin` which uses `patchelf` to do a more targeted path rewrite upfront and then to patch any remaining paths; this is meant to follow the approach used by python-build-standalone.
50+
Replacing the placeholder is a separate step, done at install time. Once the tree is where it will live, run `fixup.py` on it with any Python 3.12+ other than the one in the tree:
5251

53-
Either way, `gather` runs CPython's top-level `make install ENSUREPIP=no` into a staging `DESTDIR` and moves the installed prefix into place, so the gathered tree contains everything a normal install provides (such as `build-details.json` and the unversioned `python3` links). Only the placeholder prefix is rewritten, and patched Mach-O files are re-signed ad hoc, since macOS refuses to run binaries whose code signature no longer matches.
52+
```shell
53+
python3 ../prebuilt-cpython/fixup.py dist/
54+
```
55+
56+
`fixup.py` replaces the placeholder with the tree's location in text files only and lists the binary files that still contain it. It never modifies a binary, so signatures stay valid. It fails if no text file in the tree has the placeholder, which is the case for a tree it has already been run on.
57+
58+
`gather --relative-rpath` also uses `patchelf` to give the installed ELF files an `$ORIGIN`-relative RPATH, so they load libraries from inside the tree wherever it ends up; this follows what python-build-standalone does at build time. It's Linux only.
5459

5560
### Testing a relocated build
5661

57-
`relocatable.py test` checks whether a build still works after it has been moved. It gathers the build into a temporary directory (dist/ is left alone), renames that directory to a path of a different length with spaces in it, and runs these checks without patching anything again:
62+
`relocatable.py test` checks whether a build still works after it has been moved. It gathers the build into a temporary directory (dist/ is left alone), renames that directory to a path of a different length with spaces in it, runs `fixup.py` on it, and runs these checks:
5863

5964
- The interpreter starts
6065
- `sys` and `sysconfig` paths point inside the moved tree
@@ -74,11 +79,18 @@ Each check prints `PASS`, `FAIL`, or `SKIP` (in color on a terminal, unless `NO_
7479
../prebuilt-cpython/relocatable.py test
7580
```
7681

77-
Use `--strategy origin` to gather with `--origin` (the default is `--patch`), `--full` to also run `python -m test --fast-ci`, `--work-dir DIR` to keep the temporary files (including `gather.log`, the `make install` output), and `--format json` for JSON output.
82+
Use `--relative-rpath` to gather with `--relative-rpath`, `--no-fixup` to check the moved tree without running `fixup.py` (what a build that needs no install-time step would have to pass), `--full` to also run `python -m test --fast-ci`, `--work-dir DIR` to keep the temporary files (including `gather.log`, the `make install` output), and `--format json` for JSON output.
7883

79-
The Relocatability GitHub Actions workflow runs `test` with both strategies against CPython `main` on Linux and macOS, daily and for each pull request. Each run's summary lists the results, and the JSON is attached to the run.
84+
The Relocatability GitHub Actions workflow runs `test` in each of those modes against CPython `main` on Linux and macOS, daily and for each pull request. Each run's summary lists the results, and the JSON is attached to the run.
85+
86+
The Build artifacts workflow is run by hand and also attaches the build to the run: a tarball per platform with the gathered tree in `python/` and `fixup.py` next to it. To try one, unpack it, move `python/` to where it should live, and run `fixup.py` on it:
87+
88+
```shell
89+
tar -xf cpython-<commit>-<os>-<arch>.tar.gz
90+
python3 cpython-<commit>-<os>-<arch>/fixup.py cpython-<commit>-<os>-<arch>/python
91+
```
8092

81-
If the build wasn't configured by `relocatable.py configure`, its `--prefix` is left alone and treated as the old location. If that directory still exists, the `distribution.isolation` check fails, because the moved build might be using files from it without anyone noticing.
93+
If the build wasn't configured by `relocatable.py configure`, `fixup.py` isn't run, and its `--prefix` is treated as the old location. If that directory still exists, the `distribution.isolation` check fails, because the moved build might be using files from it without anyone noticing.
8294

8395
## Project Representatives
8496
These are people representing projects that have experience with prebuilt binaries for CPython.

‎fixup.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
"""Set the prefix of an unpacked build made by `relocatable.py gather`.
2+
3+
The build was configured with a placeholder prefix. This script replaces the
4+
placeholder with the tree's actual location in the text files that contain
5+
it. Binary files are never modified.
6+
7+
Run it once, after unpacking the tree where it is going to live, with any
8+
Python 3.12 or newer other than the one in the tree:
9+
10+
python3 fixup.py PATH/TO/TREE
11+
12+
It fails if no text file in the tree has the placeholder, which is the case
13+
for a tree it has already been run on.
14+
"""
15+
16+
# /// script
17+
# requires-python = ">=3.12"
18+
# ///
19+
20+
import argparse
21+
import os
22+
import pathlib
23+
import sys
24+
25+
PLACEHOLDER = "/the/knights/who/say/ni".ljust(255, "i")
26+
27+
28+
def fixup(tree: pathlib.Path) -> tuple[list[pathlib.Path], list[pathlib.Path]]:
29+
"""Replace the placeholder prefix in the text files of *tree*.
30+
31+
Return the text files that were rewritten and the binary files that
32+
still contain the placeholder.
33+
"""
34+
tree = tree.resolve(strict=True)
35+
if unsafe := set(os.fspath(tree)) & set("'\"\\\n"):
36+
# The prefix ends up inside Python string literals in _sysconfigdata.
37+
raise ValueError(f"{tree} contains {''.join(sorted(unsafe))!r}")
38+
placeholder = os.fsencode(PLACEHOLDER)
39+
prefix = os.fsencode(tree)
40+
text = {}
41+
binary = []
42+
for root, _, filenames in tree.walk():
43+
for filename in filenames:
44+
path = root / filename
45+
if path.is_symlink():
46+
continue
47+
contents = path.read_bytes()
48+
if placeholder not in contents:
49+
continue
50+
if b"\0" in contents:
51+
binary.append(path)
52+
else:
53+
text[path] = contents
54+
if not text:
55+
raise ValueError(
56+
f"no text file in {tree} has the placeholder prefix; "
57+
"it has already been fixed up or wasn't built by relocatable.py"
58+
)
59+
for path, contents in text.items():
60+
path.write_bytes(contents.replace(placeholder, prefix))
61+
return sorted(text), sorted(binary)
62+
63+
64+
def main(args: list[str] | None = None) -> int:
65+
parser = argparse.ArgumentParser(description=__doc__.partition("\n")[0])
66+
parser.add_argument("tree", type=pathlib.Path, help="the unpacked installation")
67+
namespace = parser.parse_args(args)
68+
try:
69+
text, binary = fixup(namespace.tree)
70+
except (OSError, ValueError) as exc:
71+
print(f"error: {exc}", file=sys.stderr)
72+
return 1
73+
tree = namespace.tree.resolve()
74+
print(f"Set the prefix to {tree} in {len(text)} text files:")
75+
for path in text:
76+
print(f" {path.relative_to(tree)}")
77+
if binary:
78+
print(f"Left {len(binary)} binary files that contain the placeholder alone:")
79+
# A .pyc file records where its source was when it was compiled.
80+
if pyc := sum(path.suffix == ".pyc" for path in binary):
81+
print(f" {pyc} .pyc files")
82+
for path in binary:
83+
if path.suffix != ".pyc":
84+
print(f" {path.relative_to(tree)}")
85+
return 0
86+
87+
88+
if __name__ == "__main__":
89+
sys.exit(main())

0 commit comments

Comments
 (0)