From 5f280273a163fa8f2f4798c43a4dd2415aa0a9a9 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Fri, 3 Jan 2025 23:37:26 -0500 Subject: [PATCH 01/14] Trial 1 of adding 3.13 support to CI --- .github/workflows/python-package.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7ecd8f9..609b3c4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', pypy-3.10] + python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13', pypy-3.10] impl: [cpython, cffi] purity: [pure, with-pylmdb-mods] @@ -243,7 +243,7 @@ jobs: # We publish a subset of the targets we test matrix: os: [macos-latest, windows-latest] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', pypy-3.10] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', pypy-3.10] impl: [cpython, cffi] purity: [with-pylmdb-mods] @@ -264,6 +264,8 @@ jobs: impl: cffi - python-version: '3.12' impl: cffi + - python-version: '3.13' + impl: cffi include: # Ubuntu artifacts apply to all python versions From 93c0cc4a29595847b9d0067b98f242b63df8a263 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Fri, 3 Jan 2025 23:50:38 -0500 Subject: [PATCH 02/14] Bump version used of upload-artifact action --- .github/workflows/python-package.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 609b3c4..a79145d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -96,7 +96,7 @@ jobs: pytest - name: Save wheel - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: "${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.impl }}-${{ matrix.purity }}.whl" path: dist/lmdb-*.whl @@ -112,7 +112,7 @@ jobs: if: >- matrix.python-version == '3.10' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: path: vers.txt name: vers.txt @@ -124,7 +124,7 @@ jobs: matrix.python-version == '3.10' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: path: dist/lmdb*.tar.gz name: source @@ -148,7 +148,7 @@ jobs: if: >- matrix.python-version == '3.10' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: path: dist/lmdb*manylinux*.whl name: manylinux @@ -200,19 +200,19 @@ jobs: if \[ ${{ matrix.PYTHON }} == cp310-cp310 \] && \[ ${{ runner.os }} == 'Linux' \] && \[ ${{ matrix.purity }} == 'with-pylmdb-mods' \] && \[ ${{ matrix.impl }} == 'cpython' \] ; then /opt/python/${{ matrix.PYTHON }}/bin/python -c \"import lmdb; print(lmdb.__version__, end='')\" > vers.txt; fi;" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: "${{ runner.os }}-${{ matrix.PYTHON }}-${{ matrix.impl }}-${{ matrix.purity }}.whl" path: dist/lmdb-*.whl - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: >- matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: path: vers.txt name: vers.txt - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: >- matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' @@ -227,7 +227,7 @@ jobs: python-versions: >- cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 build-requirements: 'patch-ng' - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: >- matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' @@ -277,21 +277,21 @@ jobs: steps: - name: Download source if: matrix.os == 'ubuntu-20.04' - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: source path: dist - name: Download manylinux artifact if: matrix.os == 'ubuntu-20.04' - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: manylinux path: dist - name: Download non-Linux wheel if: matrix.os != 'ubuntu-20.04' - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: "${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.impl }}-${{ matrix.purity }}.whl" path: dist From 229daf0c8d9be521d6ad64b3b7bd4e2466da2252 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Fri, 3 Jan 2025 23:58:41 -0500 Subject: [PATCH 03/14] For aarch64 CI, change setuptools install order --- .github/workflows/python-package.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a79145d..53909d8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -173,8 +173,9 @@ jobs: chown -R $(whoami) /github/home/.cache; python --version; /opt/python/${{ matrix.PYTHON }}/bin/python -m venv .venv; + .venv/bin/pip install -U pip setuptools; yum install -y libffi-devel; - .venv/bin/pip install -U pip setuptools wheel cffi six; + .venv/bin/pip install -U pip wheel cffi six; if \[ ${{ matrix.impl }} == cpython \] ; then echo LMDB_FORCE_CPYTHON=1 else From 1622c8032637edc93795281ec73f33bef7c81857 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 00:05:57 -0500 Subject: [PATCH 04/14] Second attempt at fixing aarch64 setuptools --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 53909d8..c17340b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -173,9 +173,8 @@ jobs: chown -R $(whoami) /github/home/.cache; python --version; /opt/python/${{ matrix.PYTHON }}/bin/python -m venv .venv; - .venv/bin/pip install -U pip setuptools; yum install -y libffi-devel; - .venv/bin/pip install -U pip wheel cffi six; + .venv/bin/pip install -U pip setuptools wheel cffi six; if \[ ${{ matrix.impl }} == cpython \] ; then echo LMDB_FORCE_CPYTHON=1 else @@ -191,6 +190,7 @@ jobs: echo \"Windows: Envs are cpython=$Env:LMDB_FORCE_CPYTHON cffi=$Env:LMDB_FORCE_CFFI pure=$Env:LMDB_PURE system=$Env:LMDB_FORCE_SYSTEM\"; .venv/bin/pip install setuptools flake8 pytest patch-ng; + /opt/python/${{ matrix.PYTHON }}/bin/python -m pip install setuptools; /opt/python/${{ matrix.PYTHON }}/bin/python setup.py develop bdist_wheel; ls dist; /opt/python/${{ matrix.PYTHON }}/bin/python -m pip install pytest; From 4ef2c305be73323512e3f829dd6102d5f8a24e8a Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 00:48:31 -0500 Subject: [PATCH 05/14] Add some missing 3.13 entries --- .github/workflows/python-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c17340b..f509d3c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -136,7 +136,7 @@ jobs: uses: RalfG/python-wheels-manylinux-build@v0.7.1 with: python-versions: >- - cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 + cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313 build-requirements: 'patch-ng' - name: What do we have @@ -160,7 +160,7 @@ jobs: matrix: impl: [cpython, cffi] purity: [pure, with-pylmdb-mods] - PYTHON: ["cp37-cp37m" , "cp38-cp38" , "cp39-cp39", "cp310-cp310", "cp311-cp311", 'cp312-cp312'] + PYTHON: ["cp37-cp37m" , "cp38-cp38" , "cp39-cp39", "cp310-cp310", "cp311-cp311", 'cp312-cp312', 'cp313-cp313'] steps: - uses: actions/checkout@v2 - run: | @@ -226,7 +226,7 @@ jobs: matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: python-versions: >- - cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 + cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313 build-requirements: 'patch-ng' - uses: actions/upload-artifact@v4 if: >- From 4d8f134f355905e3f420136072c103ca02a20204 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Wed, 4 Sep 2024 19:09:04 +0200 Subject: [PATCH 06/14] Fix build with Python 3.13+ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid using PyObject_AsReadBuffer, do what PyObject_AsReadBuffer does. Note that this is not safe, and it has never been safe, but the code does *exactly* what it used to do (at least on Python 3.8+). Fixes https://github.com/jnwatson/py-lmdb/issues/362 Co-Authored-By: Miro Hrončok --- lmdb/cpython.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lmdb/cpython.c b/lmdb/cpython.c index 3030883..81fdb06 100644 --- a/lmdb/cpython.c +++ b/lmdb/cpython.c @@ -545,9 +545,21 @@ val_from_buffer(MDB_val *val, PyObject *buf) type_error("Won't implicitly convert Unicode to bytes; use .encode()"); return -1; } +#if PY_VERSION_HEX < 0x030d0000 return PyObject_AsReadBuffer(buf, (const void **) &val->mv_data, (Py_ssize_t *) &val->mv_size); +#else + Py_buffer view; + int ret; + ret = PyObject_GetBuffer(buf, &view, PyBUF_SIMPLE); + if(ret == 0) { + val->mv_data = view.buf; + val->mv_size = view.len; + PyBuffer_Release(&view); + } + return ret; +#endif } /* ------------------- */ From 9657bfa641e985a099b45fe7beef843f1d20f662 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 13:40:13 -0500 Subject: [PATCH 07/14] Try the new trusted publisher thing --- .github/workflows/python-package.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f509d3c..a57bd02 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -305,12 +305,11 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} + # password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository-url: https://test.pypi.org/legacy/ skip-existing: true - name: Publish distribution 📦 to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} + # with: password: ${{ secrets.PYPI_API_TOKEN }} From ebe16593eab37fc010073400785f86796592a31c Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 14:04:39 -0500 Subject: [PATCH 08/14] Add metadata to indicate 3.13 support --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6d8709c..b1fc004 100644 --- a/setup.py +++ b/setup.py @@ -215,6 +215,7 @@ def grep_version(): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Database", "Topic :: Database :: Database Engines/Servers", ], From 28d26588e11ef8e8df310d759045a452c1a3272b Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 14:11:03 -0500 Subject: [PATCH 09/14] Experiment for aarch uploading not working. Add perm for publishing --- .github/workflows/python-package.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a57bd02..a635169 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -212,7 +212,7 @@ jobs: matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: path: vers.txt - name: vers.txt + name: vers.txt.aarch64 - uses: actions/upload-artifact@v4 if: >- matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && @@ -237,6 +237,8 @@ jobs: name: manylinux publish: + permissions: + id-token: write # This is required for requesting the JWT needs: build # N.B. the host running twine to upload is distinct from the target image runs-on: ubuntu-20.04 From 289840a9fb5d95241748c004276bc90118bb0d66 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 14:45:45 -0500 Subject: [PATCH 10/14] Eliminate aarch upload source, 1 instance of pytest --- .github/workflows/python-package.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a635169..b3f3941 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -194,13 +194,11 @@ jobs: /opt/python/${{ matrix.PYTHON }}/bin/python setup.py develop bdist_wheel; ls dist; /opt/python/${{ matrix.PYTHON }}/bin/python -m pip install pytest; - if \[ ${{ matrix.PYTHON }} != cp36-cp36 \] && \[ ${{ matrix.PYTHON }} != cp38-cp38 \] ; then - echo \"Envs are cpython=$LMDB_FORCE_CPYTHON cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM\"; + echo \"Envs are cpython=$LMDB_FORCE_CPYTHON cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM\"; + if \[ ${{ matrix.PYTHON }} != cp36-cp36 \] && \[ ${{ matrix.PYTHON }} != cp38-cp38 \] && \[ ${{ matrix.PYTHON }} != cp311-cp311 \] ; then /opt/python/${{ matrix.PYTHON }}/bin/python -m pytest; fi; - if \[ ${{ matrix.PYTHON }} == cp310-cp310 \] && \[ ${{ runner.os }} == 'Linux' \] && \[ ${{ matrix.purity }} == 'with-pylmdb-mods' \] && \[ ${{ matrix.impl }} == 'cpython' \] ; then - /opt/python/${{ matrix.PYTHON }}/bin/python -c \"import lmdb; print(lmdb.__version__, end='')\" > vers.txt; - fi;" + /opt/python/${{ matrix.PYTHON }}/bin/python -c \"import lmdb; print(lmdb.__version__, end='')\" > vers.txt; - uses: actions/upload-artifact@v4 with: name: "${{ runner.os }}-${{ matrix.PYTHON }}-${{ matrix.impl }}-${{ matrix.purity }}.whl" @@ -213,13 +211,6 @@ jobs: with: path: vers.txt name: vers.txt.aarch64 - - uses: actions/upload-artifact@v4 - if: >- - matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && - matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' - with: - path: dist/lmdb*.tar.gz - name: source - uses: RalfG/python-wheels-manylinux-build@v0.7.1-manylinux2014_aarch64 if: >- matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && From 9e7c6ab91a428df303a1d24feb50a07636c3f38d Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 18:37:53 -0600 Subject: [PATCH 11/14] Change version to test publishing works --- lmdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmdb/__init__.py b/lmdb/__init__.py index 3d91af9..e328aa6 100644 --- a/lmdb/__init__.py +++ b/lmdb/__init__.py @@ -50,4 +50,4 @@ def _reading_docs(): from lmdb.cffi import __all__ from lmdb.cffi import __doc__ -__version__ = '1.5.1' +__version__ = '1.5.1.dev0' From 5aacb182a5c9cac28e9b6fad7ca4d3e9ba077d55 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 21:42:56 -0600 Subject: [PATCH 12/14] Fix aarch64 stomping on non aarch64 manylinux on upload --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b3f3941..22c10ec 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -225,7 +225,7 @@ jobs: matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: path: dist/lmdb*manylinux*.whl - name: manylinux + name: manylinux.aarch64 publish: permissions: From 75c01c29573112f8272fe5167f82ebcebb236047 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 22:30:04 -0600 Subject: [PATCH 13/14] Publish aarch64 wheels --- .github/workflows/python-package.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 22c10ec..d39fcda 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -276,13 +276,20 @@ jobs: name: source path: dist - - name: Download manylinux artifact + - name: Download manylinux artifacts if: matrix.os == 'ubuntu-20.04' uses: actions/download-artifact@v4 with: name: manylinux path: dist + - name: Download aarch64 manylinux artifacts + if: matrix.os == 'ubuntu-20.04' + uses: actions/download-artifact@v4 + with: + name: manylinux.aarch64 + path: dist + - name: Download non-Linux wheel if: matrix.os != 'ubuntu-20.04' uses: actions/download-artifact@v4 From 92c0d1a81b91cbfe0cf106dcfe1a5d6a3924fe20 Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Sat, 4 Jan 2025 22:46:00 -0600 Subject: [PATCH 14/14] Add dependency from publish to build_aarch64 --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d39fcda..15e98e1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -229,8 +229,8 @@ jobs: publish: permissions: - id-token: write # This is required for requesting the JWT - needs: build + id-token: write # This is required for passwordless publishing to PyPI + needs: [build, build_aarch64] # N.B. the host running twine to upload is distinct from the target image runs-on: ubuntu-20.04 strategy: