diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..48142fd --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,34 @@ +name: "Test" + +on: + push: + branches: + - "main" + +jobs: + test-linux: + name: "Test on ${{ matrix.config.os-name }}" + runs-on: "${{ matrix.config.runner }}" + strategy: + matrix: + config: + - os-name: "Linux" + runner: "ubuntu-latest" + test-label: "ci-test-linux" + - os-name: "macOS" + runner: "macos-latest" + test-label: "ci-test-macos" + - os-name: "Windows" + runner: "windows-latest" + test-label: "ci-test-windows" + fail-fast: false + + steps: + - name: "Use it!" + id: "finder" + uses: "kurtmckee/setup-python-version-detector@main" + + - name: "Print it!" + shell: "bash" + run: | + echo '${{ steps.finder.outputs.python-identifiers }}' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..ae54e3e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +ci: + autoupdate_schedule: "monthly" + +repos: + - repo: "https://github.com/pre-commit/pre-commit-hooks" + rev: "v4.5.0" + hooks: + - id: "trailing-whitespace" + - id: "end-of-file-fixer" + - id: "check-yaml" + - id: "check-added-large-files" + + - repo: "https://github.com/python-jsonschema/check-jsonschema" + rev: "0.27.0" + hooks: + - id: "check-github-workflows" + - id: "check-dependabot" + + - repo: "local" + hooks: + - id: "sync-identify-code" + name: "Synchronize identify.py source code into 'finder.sh'" + language: "python" + entry: "python src/python_cache_buster/sync_identify_code.py" + files: "^src/python_cache_buster/identify.py$" + + - id: "sync-finder-code" + name: "Synchronize finder source code into 'action.yml'" + language: "python" + entry: "python src/python_cache_buster/sync_finder_code.py" + files: "^src/python_cache_buster/finder.*$" diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..3171e9e --- /dev/null +++ b/README.rst @@ -0,0 +1,6 @@ +Identify Python interpreters +############################ + +*Robust cache-busting based on Python implementations, architectures, and versions.* + +---- diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 0000000..8762a8e --- /dev/null +++ b/TODO.rst @@ -0,0 +1,8 @@ +* Add shellcheck pre-commit hook for finder.sh +* Add a Powershell syntax checker for finder.ps1 +* Expand README +* Create ``releases`` branch for tagging and such +* Add pre-commit hook to check README against latest tagged SHA +* Rename repo...maybe ``python-cache-buster`` +* Add test suite that can run locally +* Add test suite that verifies expected outputs in CI diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..9b9bfd9 --- /dev/null +++ b/action.yml @@ -0,0 +1,143 @@ +name: "Identify installed Python interpreters" +description: | + Find installed Python interpreters and output them as "python-identifiers". + This can be useful for cache busting of tox, virtual environments, + and other items that are sensitive to changes in Python minor versions. + + Only Python interpreters named "python" in the $PATH will be found. + +outputs: + python-identifiers: + description: | + A string of sorted Python identifiers. + + In most cases the identifiers will be paths, + but for system Pythons, 'sysconfig.get_config_var("EXT_SUFFIX")' will be included. + + The string will be separated by OS-specific PATH separators; + ":" is used on Linux and macOS, and ";" is used on Windows. + value: "${{ steps.final-step.outputs.python-identifiers }}" + +runs: + using: "composite" + steps: + - name: "Find Pythons on Linux / macOS" + id: "linux" + if: "runner.os != 'windows'" + shell: "bash" + # Do not edit the 'run' code below. + # It is copied from 'finder.sh' by a pre-commit hook. + # START: finder.sh + run: | + python_code=$( + cat <<'identify.py_SOURCE_CODE' + from __future__ import print_function + + import sysconfig + + + def main(): + ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") + if ext_suffix is not None: + print(ext_suffix) + else: + # Python 2.7 on GitHub macOS runners + import platform + print( + "." + platform.python_implementation().lower(), + sysconfig.get_config_var("py_version_nodot"), + sysconfig.get_config_var("MACHDEP"), + sep="-" + ) + + + if __name__ == "__main__": + main() + identify.py_SOURCE_CODE + ) + + # Search paths in $PATH for Python interpreters. + IFS=: read -r -a all_paths <<< "$PATH" + + # Find Python interpreters. + paths=() + for path in "${all_paths[@]}"; do + # Interpreters in RUNNER_TOOL_CACHE have directory names that include: + # + # * The implementation (like "Python" or "PyPy") + # * A version (like "3.10.12") + # * The architecture (like "x64") + # + # In such cases, the path can be used as the identifier. + if [[ "${path/#${RUNNER_TOOL_CACHE}/}" != "${path}" ]]; then + # Check for bin/python first; + # this results in duplicate paths which are later removed. + if [[ -x "${path}/bin/python" ]]; then + paths+=("${path}/bin") + elif [[ -x "${path}/python" ]]; then + paths+=("${path}") + fi + else + # System Pythons (e.g. /usr/bin/python) have nothing unique in the path. + # In such cases, it's necessary to run the executable to get something unique. + if [[ -x "${path}/python" ]]; then + paths+=("$(echo "${python_code}" | "${path}/python" -)") + fi + fi + done + + # Sort the paths, ensure each path is unique, and create the output result. + result="$( + echo "${paths[*]}" \ + | tr ' ' '\n' \ + | sort \ + | uniq \ + | tr '\n' ':' + )" + + # Trim trailing colons. + result="${result%:}" + + # Output path information. + echo "python-identifiers=${result}" > "$GITHUB_OUTPUT" + # END: finder.sh + + - name: "Find Pythons on Windows" + id: "windows" + if: "runner.os == 'windows'" + shell: "powershell" + # Do not edit the 'run' code below. + # It is copied from 'finder.ps1' by a pre-commit hook. + # START: finder.ps1 + run: | + $true | Out-Null # No-op to prevent YAML syntax errors. + # Search paths in $PATH for Python interpreters. + $all_paths = $env:PATH -split ";" + + # Find Python interpreters. + $paths = @() + foreach ($path in $all_paths) { + # Only consider paths in RUNNER_TOOL_CACHE. + if ($path.StartsWith($env:RUNNER_TOOL_CACHE)) { + if (Test-Path "$path\python.exe") { + $paths += $path + } + } + } + + # Sort the paths, ensure each path is unique, and create the output result. + $result = ( + $paths ` + | Sort-Object ` + | Get-Unique + ) -join ";" + + # Output path information. + Write-Output "python-identifiers=$result" > "$env:GITHUB_OUTPUT" + # END: finder.ps1 + + - name: "Output" + id: "final-step" + shell: "bash" + run: | + echo 'python-identifiers=${{ steps.linux.outputs.python-identifiers }}${{ steps.windows.outputs.python-identifiers }}' > "$GITHUB_OUTPUT" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..485c298 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[tool.poetry] +name = "python_cache_buster" +version = "1.0.0" +description = "" +authors = ["Kurt McKee "] +license = "MIT" +readme = "README.rst" + +[tool.poetry.scripts] +identify = "python_cache_buster.identify:main" +sync-finder-code = "python_cache_buster.sync_finder_code:main" +sync-identify-code = "python_cache_buster.sync_identify_code:main" + +[tool.poetry.dependencies] +python = ">=3.8" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/src/python_cache_buster/__init__.py b/src/python_cache_buster/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/python_cache_buster/finder.ps1 b/src/python_cache_buster/finder.ps1 new file mode 100644 index 0000000..3cd97be --- /dev/null +++ b/src/python_cache_buster/finder.ps1 @@ -0,0 +1,23 @@ +# Search paths in $PATH for Python interpreters. +$all_paths = $env:PATH -split ";" + +# Find Python interpreters. +$paths = @() +foreach ($path in $all_paths) { + # Only consider paths in RUNNER_TOOL_CACHE. + if ($path.StartsWith($env:RUNNER_TOOL_CACHE)) { + if (Test-Path "$path\python.exe") { + $paths += $path + } + } +} + +# Sort the paths, ensure each path is unique, and create the output result. +$result = ( + $paths ` + | Sort-Object ` + | Get-Unique +) -join ";" + +# Output path information. +Write-Output "python-identifiers=$result" diff --git a/src/python_cache_buster/finder.sh b/src/python_cache_buster/finder.sh new file mode 100644 index 0000000..5375bbb --- /dev/null +++ b/src/python_cache_buster/finder.sh @@ -0,0 +1,71 @@ +python_code=$( +cat <<'identify.py_SOURCE_CODE' +from __future__ import print_function + +import sysconfig + + +def main(): + ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") + if ext_suffix is not None: + print(ext_suffix) + else: + # Python 2.7 on GitHub macOS runners + import platform + print( + "." + platform.python_implementation().lower(), + sysconfig.get_config_var("py_version_nodot"), + sysconfig.get_config_var("MACHDEP"), + sep="-" + ) + + +if __name__ == "__main__": + main() +identify.py_SOURCE_CODE +) + +# Search paths in $PATH for Python interpreters. +IFS=: read -r -a all_paths <<< "$PATH" + +# Find Python interpreters. +paths=() +for path in "${all_paths[@]}"; do + # Interpreters in RUNNER_TOOL_CACHE have directory names that include: + # + # * The implementation (like "Python" or "PyPy") + # * A version (like "3.10.12") + # * The architecture (like "x64") + # + # In such cases, the path can be used as the identifier. + if [[ "${path/#${RUNNER_TOOL_CACHE}/}" != "${path}" ]]; then + # Check for bin/python first; + # this results in duplicate paths which are later removed. + if [[ -x "${path}/bin/python" ]]; then + paths+=("${path}/bin") + elif [[ -x "${path}/python" ]]; then + paths+=("${path}") + fi + else + # System Pythons (e.g. /usr/bin/python) have nothing unique in the path. + # In such cases, it's necessary to run the executable to get something unique. + if [[ -x "${path}/python" ]]; then + paths+=("$(echo "${python_code}" | "${path}/python" -)") + fi + fi +done + +# Sort the paths, ensure each path is unique, and create the output result. +result="$( + echo "${paths[*]}" \ + | tr ' ' '\n' \ + | sort \ + | uniq \ + | tr '\n' ':' +)" + +# Trim trailing colons. +result="${result%:}" + +# Output path information. +echo "python-identifiers=${result}" diff --git a/src/python_cache_buster/identify.py b/src/python_cache_buster/identify.py new file mode 100644 index 0000000..afe601a --- /dev/null +++ b/src/python_cache_buster/identify.py @@ -0,0 +1,22 @@ +from __future__ import print_function + +import sysconfig + + +def main(): + ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") + if ext_suffix is not None: + print(ext_suffix) + else: + # Python 2.7 on GitHub macOS runners + import platform + print( + "." + platform.python_implementation().lower(), + sysconfig.get_config_var("py_version_nodot"), + sysconfig.get_config_var("MACHDEP"), + sep="-" + ) + + +if __name__ == "__main__": + main() diff --git a/src/python_cache_buster/sync_finder_code.py b/src/python_cache_buster/sync_finder_code.py new file mode 100644 index 0000000..f14152f --- /dev/null +++ b/src/python_cache_buster/sync_finder_code.py @@ -0,0 +1,67 @@ +import sys +import pathlib +import textwrap + +FILES_NOT_MODIFIED = 0 +FILES_MODIFIED = 1 + +FINDER_SH = (pathlib.Path(__file__).parent / "finder.sh").resolve() +FINDER_PS1 = (pathlib.Path(__file__).parent / "finder.ps1").resolve() +ACTION_YML = (pathlib.Path(__file__).parent / "../../action.yml").resolve() + + +def sync(path: pathlib.Path) -> int: + # Verify the paths are acceptable. + if path != FINDER_SH and path != FINDER_PS1: + print(f"{path} is not an acceptable file path.") + return FILES_NOT_MODIFIED + + code = path.read_text().strip() + + # Prevent YAML syntax errors caused by shared comment syntax. + if code.lstrip().startswith("#"): + if path == FINDER_SH: + code = "true > /dev/null # No-op to prevent YAML syntax errors.\n" + code + else: # path == FINDER_PS1 + code = "$true | Out-Null # No-op to prevent YAML syntax errors.\n" + code + + # Redirect outputs to GITHUB_OUTPUT. + if path == FINDER_SH: + code += ' > "$GITHUB_OUTPUT"' + else: # path == FINDER_PS1 + code += ' > "$env:GITHUB_OUTPUT"' + + # Determine the text boundaries of the source code in 'action.yml'. + yaml = ACTION_YML.read_text() + start_line = f"# START: {path.name}\n" + end_line = f"# END: {path.name}" + start = yaml.find(start_line) + len(start_line) + end = yaml.rfind("\n", start, yaml.find(end_line, start)) + indent = yaml.find("run:", start) - start + + # Prepare the code for injection. + block = textwrap.indent("run: |\n" + textwrap.indent(code, " " * 2), " " * indent) + + # Inject the source code and determine if 'action.yml' needs to be overwritten. + new_yaml = yaml[:start] + block + yaml[end:] + if new_yaml != yaml: + ACTION_YML.write_text(new_yaml) + return FILES_MODIFIED + + return FILES_NOT_MODIFIED + + +def main(): + if len(sys.argv) < 2: + print("The path to a modified 'finder.sh' or 'finder.ps1' must be provided.") + sys.exit(FILES_NOT_MODIFIED) + + rc = FILES_NOT_MODIFIED + for argument in sys.argv[1:]: + rc |= sync(pathlib.Path(argument).resolve()) + + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/src/python_cache_buster/sync_identify_code.py b/src/python_cache_buster/sync_identify_code.py new file mode 100644 index 0000000..6154d38 --- /dev/null +++ b/src/python_cache_buster/sync_identify_code.py @@ -0,0 +1,44 @@ +import sys +import pathlib + +FILES_NOT_MODIFIED = 0 +FILES_MODIFIED = 1 + +IDENTIFY_PY = (pathlib.Path(__file__).parent / "identify.py").resolve() +FINDER_SH = (pathlib.Path(__file__).parent / "finder.sh").resolve() + + +def sync(path: pathlib.Path) -> int: + # Verify the paths are acceptable. + if path != IDENTIFY_PY: + print(f"{path} is not an acceptable file path.") + return FILES_NOT_MODIFIED + + code = path.read_text().strip() + + # Determine the text boundaries of the source code in 'action.yml'. + sh = FINDER_SH.read_text() + tag = f"{IDENTIFY_PY.name}_SOURCE_CODE" + start = sh.find("\n", sh.find(tag)) + 1 + end = sh.rfind("\n", start, sh.find(tag, start)) + + # Inject the source code and determine if 'action.yml' needs to be overwritten. + new_sh = sh[:start] + code + sh[end:] + if new_sh != sh: + FINDER_SH.write_text(new_sh) + return FILES_MODIFIED + + return FILES_NOT_MODIFIED + + +def main(): + if len(sys.argv) < 2: + print(f"The path to a modified '{IDENTIFY_PY.name}' must be provided.") + sys.exit(FILES_NOT_MODIFIED) + + rc = sync(pathlib.Path(sys.argv[-1]).resolve()) + sys.exit(rc) + + +if __name__ == "__main__": + main()