Skip to content
This repository has been archived by the owner on Dec 7, 2023. It is now read-only.

Commit

Permalink
Fix CI for manylinux build of python package (#7)
Browse files Browse the repository at this point in the history
* add dockerfile for centos build

* add tox

* modified:   .github/workflows/centos_builder.yml

* modified:   .github/workflows/main.yml

* modified:   ../.github/workflows/main.yml

* modified:   ../.github/workflows/centos_builder.yml

* test for ci [skip ci]

* add dockerfile for manylinux

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* deleted:    .github/workflows/test.yml

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* [skip ci]

* "versionup: 0.4.9 -> 0.4.10dev0"

* [skip ci]

* Revert "[skip ci]"

This reverts commit 9f9b8ca.

* Revert ""versionup: 0.4.9 -> 0.4.10dev0""

This reverts commit f37ed99.

* Revert "[skip ci]"

This reverts commit da1c2e2.

* [skip ci]

* "versionup: 0.4.9 -> 0.4.10dev1"

* [skip ci]

* "versionup: 0.4.10dev1 -> 0.4.10-dev2"

* refactor ci

* "versionup: 0.4.10-dev2 -> 0.4.10-dev3"

* refactor ci

* "versionup: 0.4.10-dev3 -> 0.4.10-dev4"

* refactor ci

* "versionup: 0.4.10-dev4 -> 0.4.10-dev5"

* add pypi test ci

* [skip ci]

* [skip ci]

* [skip ci]

* "versionup: 0.4.10-dev5 -> 0.4.10-dev6"

* [skip ci]

* [skip ci]

* fix ci

* "versionup: 0.4.10-dev6 -> 0.4.10-dev7"

* modified:   ../.github/workflows/test_pypi_manylinux.yml

* fix ci

* modified:   ../../python/README.md

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* [run pip test] [skip ci]

* "versionup: 0.4.10-dev7 -> 0.4.10"
  • Loading branch information
tamuhey authored Mar 9, 2020
1 parent 56211f0 commit 8ce82c2
Show file tree
Hide file tree
Showing 15 changed files with 265 additions and 122 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/target/
60 changes: 45 additions & 15 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Test
name: Test and Deploy

on:
push:
Expand All @@ -7,11 +7,12 @@ on:

jobs:
test:
if: contains(github.event.head_commit.message, '[skip ci]') == false
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: [3.5, 3.6, 3.7, 3.8]
os: [ubuntu-latest, macos-latest, windows-latest]
os: [macos-latest, windows-latest, ubuntu-latest]
steps:
- uses: actions/checkout@v1
- name: Set up Python ${{ matrix.python-version }}
Expand Down Expand Up @@ -46,7 +47,7 @@ jobs:
working-directory: python
run: |
python -m pip install --upgrade pip
pip install poetry
pip install poetry maturin
poetry install
- name: Build python package
Expand All @@ -72,14 +73,15 @@ jobs:
cargo login ${{ secrets.CRATES_PASS }}
cargo publish
publish-python:
publish-python-wheels:
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/python/')
needs: test
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: [3.5, 3.6, 3.7, 3.8]
os: [ubuntu-latest, macos-latest, windows-latest]
# ubuntu wheel is built in `manylinux_build.yml`
os: [macos-latest, windows-latest]

steps:
- uses: actions/checkout@v1
Expand All @@ -95,23 +97,51 @@ jobs:
working-directory: python
run: |
python -m pip install --upgrade pip
pip install maturin
pip install maturin twine
- name: Publish (without sdist)
if: matrix.os != 'ubuntu-latest' || matrix.python-version != '3.7'
- name: Build
working-directory: python
run: maturin build --no-sdist --release --strip -i python

- name: Publish test pypi
working-directory: python
run: twine upload target/wheels/*whl --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TEST_PYPI_USER }} -p ${{ secrets.TEST_PYPI_PASS }}

- name: Publish pypi
working-directory: python
run: twine upload target/wheels/*whl -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASS }}

publish-python-sdist:
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/python/')
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: 3.7
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly
override: true

- name: Install publishment tool
working-directory: python
run: |
maturin publish -i python --no-sdist --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TEST_PYPI_USER }} -p ${{ secrets.TEST_PYPI_PASS }}
python -m pip install --upgrade pip
pip install maturin twine
- name: Build sdist
working-directory: python
run: maturin sdist

- name: Publish
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.7'
- name: Publish test pypi
working-directory: python
run: |
maturin publish -i python --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TEST_PYPI_USER }} -p ${{ secrets.TEST_PYPI_PASS }}
twine upload target/wheels/*.tar.gz --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TEST_PYPI_USER }} -p ${{ secrets.TEST_PYPI_PASS }}
- name: Publish pypi package
- name: Publish pypi
working-directory: python
run: |
pip install twine
twine upload -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASS }} target/wheels/*
twine upload target/wheels/*.tar.gz -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASS }}
72 changes: 72 additions & 0 deletions .github/workflows/manylinux_build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: build manylinux

on:
push:

jobs:
build:
if: contains(github.event.head_commit.message, '[skip ci]') == false
runs-on: ubuntu-latest
strategy:
matrix:
python-version:
- 3.5
- 3.6
- 3.7
- 3.8
container:
image: quay.io/pypa/manylinux1_x86_64:2020-03-07-9c5ba95
env:
PATH: /root/.cargo/bin:/root/.local/bin:/opt/python/cp36-cp36m/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/python/cp35-cp35m/bin:/opt/python/cp36-cp36m/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin
options: --user root
env:
HOME: /root
PYTHON: python${{ matrix.python-version }}
steps:
- uses: actions/checkout@v1
- name: Install rust
run: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain nightly --profile minimal -y
- name: Test rust lib
run: cargo test

- name: Install dependencies with pip
working-directory: python
run: |
$PYTHON -m pip install --upgrade pip
$PYTHON -m venv .venv
$PYTHON -m pip install poetry maturin
poetry install
poetry run which python
- name: Build python package
working-directory: python
run: poetry run maturin develop

- name: Test with pytest
working-directory: python
run: poetry run pytest tests

- name: Install publishment tool
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/python/')
working-directory: python
run: $PYTHON -m pip install twine auditwheel

- name: Build
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/python/')
working-directory: python
run: |
maturin build --no-sdist --release --strip -i $PYTHON
find target/ -type f -name "*whl" -exec $PYTHON -m auditwheel repair {} \;
- name: Publish test pypi
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/python/')
working-directory: python
run: |
twine upload target/wheels/*whl --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TEST_PYPI_USER }} -p ${{ secrets.TEST_PYPI_PASS }}
- name: Publish pypi
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/python/')
working-directory: python
run: |
twine upload target/wheels/*whl -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASS }}
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ name: check-pypi
on:
schedule:
- cron: '0 0 * * *'
push:

jobs:
build:
if: contains(github.event.head_commit.message, '[run pip test]') == true || github.event_name == 'schedule'
runs-on: ${{ matrix.os }}
strategy:
matrix:
Expand All @@ -16,7 +18,11 @@ jobs:
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
- name: Pip install
run: |
python -m pip install --upgrade pip
pip install pytokenizations --pre
- run: |
import tokenizations
tokenizations.get_alignments(["a"],["a"])
shell: python
25 changes: 25 additions & 0 deletions .github/workflows/test_pypi_manylinux.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: check-pypi-manylinux

on:
schedule:
- cron: '0 0 * * *'
push:

jobs:
build:
if: contains(github.event.head_commit.message, '[run pip test]') == true || github.event_name == 'schedule'
runs-on: ubuntu-latest
strategy:
matrix:
container:
- centos
- manylinux
python-version: [3.5, 3.6, 3.7, 3.8]
env:
DOCKERFILE: dockerfiles/ci/${{ matrix.container }}/Dockerfile
PYTHON: python${{ matrix.python-version }}
steps:
- uses: actions/checkout@v1
- run: docker login -u tamuhey -p ${{ secrets.GITHUB_TOKEN }} docker.pkg.github.com
- run: docker build -f $DOCKERFILE -t img .
- run: docker run img sh -c "$PYTHON -m pip install pytokenizations --pre && $PYTHON -c 'import tokenizations; print(tokenizations.get_alignments([\"a\"], [\"a\"]))'"
31 changes: 2 additions & 29 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,8 @@
[![pypi](https://img.shields.io/pypi/v/pytokenizations.svg)](https://pypi.org/project/pytokenizations/)
[![Actions Status](https://github.com/tamuhey/tokenizations/workflows/Test/badge.svg)](https://github.com/tamuhey/tokenizations/actions)

## Installation

```bash
$ pip install pytokenizations
```

## Usage

Get an alignment map for two different tokenizations:

```python
import tokenizations
tokens_a = ["New York"]
tokens_b = ["New", "York"]
a2b = [[0, 1]]
b2a = [[0], [0]]
assert tokenizations.get_alignments(tokens_a, tokens_b) == (a2b, b2a)
```

`a2b[i]` is tokens_a list representing the alignment from `tokens_a` to `tokens_b`.
You can get the alignments for "dirty" tokens:

```python
tokens_a = ["げん", ""]
tokens_b = ["けんこ"] # all accents are dropped (が -> か, ご -> こ)
a2b = [[0], [0]]
b2a = [[0, 1]]
assert tokenizations.get_alignments(tokens_a, tokens_b) == (a2b, b2a)
```
Rust document: [docs.rs](https://docs.rs/tokenizations/0.2.2/tokenizations/)
Python document: [python/README.md](./python/README.md)

## Algorithm

Expand Down
12 changes: 12 additions & 0 deletions dockerfiles/centos/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
FROM centos:7
ENV HOME /root
ENV PATH $HOME/.pyenv/bin:$HOME/.pyenv/shims:$HOME/.cargo/bin:$HOME/.local/bin:$PATH
RUN yum update -y && yum install -y git gcc make zlib-devel && \
curl https://pyenv.run | bash
# pyenv prequisits. see https://github.com/pyenv/pyenv/wiki/common-build-problems
RUN yum install @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils -y
RUN pyenv install 3.8.2 && \
pyenv install 3.7.6 && \
pyenv install 3.6.10 && \
pyenv install 3.5.4 && \
pyenv global 3.8.2 3.7.6 3.6.10 3.5.4
1 change: 1 addition & 0 deletions dockerfiles/ci/centos/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
FROM docker.pkg.github.com/tamuhey/tokenizations/centos7-python:0
2 changes: 2 additions & 0 deletions dockerfiles/ci/manylinux/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
FROM quay.io/pypa/manylinux1_x86_64:2020-03-07-9c5ba95
ENV PATH /root/.cargo/bin:/root/.local/bin:/opt/python/cp36-cp36m/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/python/cp35-cp35m/bin:/opt/python/cp36-cp36m/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin
2 changes: 1 addition & 1 deletion python/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "pytokenizations"
version = "0.4.9"
version = "0.4.10"
authors = ["Yohei Tamura <[email protected]>"]
edition = "2018"

Expand Down
42 changes: 42 additions & 0 deletions python/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
## Installation

```bash
$ pip install pytokenizations
```

### Install from source

This library uses [maturin](https://github.com/PyO3/maturin) to build.

```console
$ git clone https://github.com/tamuhey/tokenizations
$ cd python
$ pip install maturin
$ maturin build
```

Now wheel is built in `python/target/wheels` directory. You can install it with `pip install *whl`.

## Usage

Get an alignment map for two different tokenizations:

```python
import tokenizations
tokens_a = ["New York"]
tokens_b = ["New", "York"]
a2b = [[0, 1]]
b2a = [[0], [0]]
assert tokenizations.get_alignments(tokens_a, tokens_b) == (a2b, b2a)
```

`a2b[i]` is tokens_a list representing the alignment from `tokens_a` to `tokens_b`.
You can get the alignments for "dirty" tokens:

```python
tokens_a = ["げん", ""]
tokens_b = ["けんこ"] # all accents are dropped (が -> か, ご -> こ)
a2b = [[0], [0]]
b2a = [[0, 1]]
assert tokenizations.get_alignments(tokens_a, tokens_b) == (a2b, b2a)
```
Loading

0 comments on commit 8ce82c2

Please sign in to comment.