Skip to content

Commit 7c7005c

Browse files
tmitmi
committed
Switch eccodes python to wheelchains
1 parent bae4f34 commit 7c7005c

File tree

6 files changed

+102
-256
lines changed

6 files changed

+102
-256
lines changed

‎.github/workflows/build-wheel-linux.yml

+23-124
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
# nor does it submit to any jurisdiction.
88

99

10-
name: Build Linux
10+
name: Build Python Wheel for Linux
1111

1212
on:
1313
# Trigger the workflow manually
@@ -16,137 +16,36 @@ on:
1616
# Allow to be called from another workflow
1717
workflow_call: ~
1818

19+
# TODO automation trigger
1920
# repository_dispatch:
2021
# types: [eccodes-updated]
2122

22-
push:
23-
tags-ignore:
24-
- '**'
25-
paths:
26-
- 'scripts/common.sh'
27-
- 'scripts/select-python-linux.sh'
28-
- 'scripts/wheel-linux.sh'
29-
- 'scripts/build-linux.sh'
30-
- 'scripts/test-linux.sh'
31-
- 'scripts/copy-licences.py'
32-
- '.github/workflows/build-wheel-linux.yml'
33-
34-
# to allow the action to run on the manylinux docker image based on CentOS 7
35-
env:
36-
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
37-
3823
jobs:
39-
4024
build:
41-
42-
# if: false # for temporarily disabling for debugging
43-
44-
runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
45-
container: dockcross/manylinux_2_28-x64:latest
46-
47-
name: Build manylinux_2_28-x64
48-
49-
steps:
50-
- uses: actions/checkout@v4
51-
52-
- run: ./scripts/build-linux.sh
53-
54-
################################################################
55-
- run: ./scripts/wheel-linux.sh 3.8
56-
- uses: actions/upload-artifact@v4
57-
name: Upload wheel 3.8
58-
with:
59-
name: wheel-manylinux2014-3.8
60-
path: wheelhouse/*.whl
61-
62-
# ################################################################
63-
- run: ./scripts/wheel-linux.sh 3.9
64-
- uses: actions/upload-artifact@v4
65-
name: Upload wheel 3.9
66-
with:
67-
name: wheel-manylinux2014-3.9
68-
path: wheelhouse/*.whl
69-
70-
# ################################################################
71-
- run: ./scripts/wheel-linux.sh 3.10
72-
- uses: actions/upload-artifact@v4
73-
name: Upload wheel 3.10
74-
with:
75-
name: wheel-manylinux2014-3.10
76-
path: wheelhouse/*.whl
77-
78-
# ################################################################
79-
- run: ./scripts/wheel-linux.sh 3.11
80-
- uses: actions/upload-artifact@v4
81-
name: Upload wheel 3.11
82-
with:
83-
name: wheel-manylinux2014-3.11
84-
path: wheelhouse/*.whl
85-
86-
# ################################################################
87-
- run: ./scripts/wheel-linux.sh 3.12
88-
- uses: actions/upload-artifact@v4
89-
name: Upload wheel 3.12
90-
with:
91-
name: wheel-manylinux2014-3.12
92-
path: wheelhouse/*.whl
93-
94-
test:
95-
96-
needs: build
97-
98-
strategy:
99-
fail-fast: false
100-
matrix: # We don't test 3.6, as it is not supported anymore by github actions
101-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
102-
25+
name: Build manylinux_2_28
10326
runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
104-
105-
name: Test with ${{ matrix.python-version }}
106-
27+
# TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix?
28+
container:
29+
image: eccr.ecmwf.int/wheelmaker/2_28:latest
30+
credentials:
31+
username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
32+
password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
10733
steps:
108-
109-
- uses: actions/checkout@v4
110-
111-
- uses: actions/download-artifact@v4
112-
with:
113-
name: wheel-manylinux2014-${{ matrix.python-version }}
114-
115-
- run: ./scripts/test-linux.sh ${{ matrix.python-version }}
116-
117-
118-
deploy:
119-
120-
if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }}
121-
122-
strategy:
123-
fail-fast: false
124-
matrix:
125-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
126-
127-
needs: [test, build]
128-
129-
name: Deploy wheel ${{ matrix.python-version }}
130-
131-
runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
132-
133-
134-
steps:
135-
136-
- run: mkdir artifact-${{ matrix.python-version }}
137-
138-
- uses: actions/checkout@v4
139-
140-
- uses: actions/download-artifact@v4
141-
with:
142-
name: wheel-manylinux2014-${{ matrix.python-version }}
143-
path: artifact-${{ matrix.python-version }}
144-
34+
# TODO convert this to be matrix-friendly. Note it's a bit tricky since
35+
# we'd ideally not reexecute the compile step multiple times, but it
36+
# (non-essentially) depends on a matrix-based step
37+
# NOTE we dont use action checkout because it doesnt cleanup after itself correctly
38+
- run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj
39+
- run: cd /proj && /buildscripts/prepare_deps.sh ./buildconfig 3.11
14540
- run: |
146-
source ./scripts/select-python-linux.sh 3.10
147-
pip3 install twine
148-
ls -l artifact-${{ matrix.python-version }}/*.whl
149-
twine upload artifact-${{ matrix.python-version }}/*.whl
41+
cd /proj
42+
export PYTHONPATH=/buildscripts/; export LIBDIR=/tmp/prereqs/eccodeslib/lib64; export INCDIR=/tmp/prereqs/eccodeslib/include
43+
uv run --python python3.11 python -m build --installer uv --wheel .
44+
- run: mkdir -p /build/wheel && mv /proj/dist/*whl /build/wheel
45+
- run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /build/wheel/*whl
46+
- run: cd /proj && /buildscripts/upload-pypi.sh /build/wheel/*whl
15047
env:
15148
TWINE_USERNAME: __token__
15249
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
50+
# NOTE temporary thing until all the mess gets cleared
51+
- run: rm -rf ./* ./.git ./.github

‎MANIFEST.in

-20
This file was deleted.

‎buildconfig

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# (C) Copyright 2024- ECMWF.
2+
#
3+
# This software is licensed under the terms of the Apache Licence Version 2.0
4+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
5+
# In applying this licence, ECMWF does not waive the privileges and immunities
6+
# granted to it by virtue of its status as an intergovernmental organisation
7+
# nor does it submit to any jurisdiction.
8+
9+
# NOTE since eccodes-python uses only parts of the Wheelmaker tooling, this file
10+
# is only correspondingly partial. Namely, `compile` and `wheel-linux` options
11+
# are not present, and only `prepare_deps` and `test-wheel` invocation is assumed
12+
13+
DEPENDENCIES='["eccodeslib"]'
14+
NAME="eccodes"

‎gribapi/bindings.py

+10-21
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,6 @@
2727

2828
LOG = logging.getLogger(__name__)
2929

30-
_MAP = {
31-
"grib_api": "eccodes",
32-
"gribapi": "eccodes",
33-
}
34-
3530
EXTENSIONS = {
3631
"darwin": ".dylib",
3732
"win32": ".dll",
@@ -43,16 +38,13 @@
4338
LOG.addHandler(logging.StreamHandler())
4439

4540

46-
def _lookup(name):
47-
return _MAP.get(name, name)
48-
49-
50-
def find_binary_libs(name):
51-
name = _lookup(name)
41+
def _find_eccodes_custom() -> str|None:
42+
# TODO deprecate this method in favour of findlibs only
43+
name = "eccodes"
5244
env_var = "ECCODES_PYTHON_USE_FINDLIBS"
5345
if int(os.environ.get(env_var, "0")):
5446
LOG.debug(f"{name} lib search: {env_var} set, so using findlibs")
55-
47+
return None
5648
else:
5749
LOG.debug(f"{name} lib search: trying to find binary wheel")
5850
here = os.path.dirname(__file__)
@@ -90,17 +82,14 @@ def find_binary_libs(name):
9082
LOG.debug(
9183
f"{name} lib search: did not find library from wheel; try to find as separate lib"
9284
)
85+
return None
9386

94-
# if did not find the binary wheel, or the env var is set, fall back to findlibs
95-
import findlibs
96-
97-
foundlib = findlibs.find(name)
98-
LOG.debug(f"{name} lib search: findlibs returned {foundlib}")
99-
return foundlib
100-
101-
102-
library_path = find_binary_libs("eccodes")
10387

88+
library_path = _find_eccodes_custom()
89+
if library_path is None:
90+
import findlibs
91+
library_path = findlibs.find("eccodes")
92+
LOG.debug(f"eccodes lib search: findlibs returned {library_path}")
10493
if library_path is None:
10594
raise RuntimeError("Cannot find the ecCodes library")
10695

‎setup.cfg

+20
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,23 @@
1+
[metadata]
2+
description = "eccodes"
3+
long_description = file: README.rst
4+
long_description_content_type = text/rst
5+
author = "European Centre for Medium-Range Weather Forecasts (ECMWF)"
6+
author_email = "[email protected]"
7+
url = "https://github.com/ecmwf/eccodes-python"
8+
keywords = ecCodes, GRIB, BUFR
9+
classifiers =
10+
Development Status :: 4 - Beta
11+
Intended Audience :: Developers
12+
License :: OSI Approved :: Apache Software License
13+
Programming Language :: Python :: 3.8
14+
Programming Language :: Python :: 3.9
15+
Programming Language :: Python :: 3.10
16+
Programming Language :: Python :: 3.11
17+
Programming Language :: Python :: Implementation :: CPython
18+
Programming Language :: Python :: Implementation :: PyPy
19+
Operating System :: OS Independent
20+
121
[aliases]
222
test = pytest
323

0 commit comments

Comments
 (0)