Skip to content

Commit 7178ce2

Browse files
oliverwm1oliverwm1
authored
Updated public release (#5)
* Delete existing files * Copy full-model code from 028c085e * Restore README.md from ace repo * Update README.md to add future docs and fix example link * Update full-model refs to ace/ACE * Remove refs to internal compute systems from README * Update version to 0.2.0 * Delete GPU CI test because it wont run on this repo * Remove Documentation section from README * Delete internal stuff from Makefile * Remove internal data processing configs/scripts * Also delete test_config.py
1 parent 50d49b8 commit 7178ce2

File tree

227 files changed

+26605
-4426
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

227 files changed

+26605
-4426
lines changed

‎.github/workflows/docs.yaml

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: docs
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
11+
jobs:
12+
build:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v3
16+
- uses: actions/setup-python@v4
17+
with:
18+
python-version: "3.10"
19+
- uses: actions/cache@v4
20+
with:
21+
path: ${{ env.pythonLocation }}
22+
key: ${{ env.pythonLocation }}-${{ hashFiles('fme/requirements.txt') }}-${{ hashFiles('fme/docs/requirements.txt') }}-${{ hashFiles('fme/constraints.txt') }}
23+
- name: Install dependencies
24+
run: |
25+
python -m pip install uv==0.2.5
26+
uv pip install --system -c constraints.txt -e fme[docs]
27+
- name: Build docs
28+
run: |
29+
cd fme/docs && make doctest html
30+
- name: Deploy to GitHub Pages
31+
uses: peaceiris/actions-gh-pages@v3
32+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
33+
with:
34+
publish_branch: gh-pages
35+
github_token: ${{ secrets.GITHUB_TOKEN }}
36+
publish_dir: fme/docs/_build/
37+
force_orphan: true

‎.github/workflows/fme-unit-tests.yaml

-33
This file was deleted.

‎.github/workflows/tests.yaml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
11+
jobs:
12+
cpu:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v3
16+
- uses: actions/setup-python@v4
17+
with:
18+
python-version: "3.10"
19+
- uses: actions/cache@v4
20+
with:
21+
path: ${{ env.pythonLocation }}
22+
key: ${{ env.pythonLocation }}-${{ hashFiles('fme/requirements.txt') }}-${{ hashFiles('fme/dev-requirements.txt') }}-${{ hashFiles('fme/constraints.txt') }}
23+
- name: Install dependencies
24+
run: |
25+
python -m pip install uv==0.2.5
26+
uv pip install --system -c constraints.txt -e fme[dev]
27+
- name: Run pytest
28+
run: |
29+
make test

‎.gitignore

+7
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
fme/docs/available_modules.rst
2+
fme/docs/_build
3+
14
.vscode
25

36
# Byte-compiled / optimized / DLL files
@@ -72,6 +75,7 @@ instance/
7275

7376
# Sphinx documentation
7477
docs/_build/
78+
.DS_Store
7579

7680
# PyBuilder
7781
target/
@@ -129,3 +133,6 @@ dmypy.json
129133

130134
# Pyre type checker
131135
.pyre/
136+
137+
# scratch directory for testing
138+
scratch/

‎.pre-commit-config.yaml

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,3 @@
1-
exclude: |
2-
(?x)^(
3-
models/fcn-mip/.+ |
4-
models/FourCastNet/.+
5-
)$
61
repos:
72
- repo: https://github.com/psf/black
83
rev: 23.3.0
@@ -24,7 +19,7 @@ repos:
2419
name: flake8 __init__.py files
2520
files: "__init__.py"
2621
# ignore unused import error in __init__.py files
27-
args: ["--ignore=F401,E203", --config, setup.cfg]
22+
args: ["--ignore=F401,E203,W503", --config, setup.cfg]
2823
- repo: https://github.com/pre-commit/mirrors-mypy
2924
rev: v1.2.0
3025
hooks:

‎Makefile

+5-24
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,18 @@
11
VERSION ?= $(shell git rev-parse --short HEAD)
22
IMAGE ?= fme
33
ENVIRONMENT_NAME ?= fme
4-
USERNAME ?= $(shell beaker account whoami --format=json | jq -r '.[0].name')
54

65
build_docker_image:
76
docker build -f docker/Dockerfile -t $(IMAGE):$(VERSION) .
87

9-
build_beaker_image: build_docker_image
10-
beaker image create --name $(IMAGE)-$(VERSION) $(IMAGE):$(VERSION)
11-
12-
build_podman_image:
13-
podman-hpc build -f docker/Dockerfile -t $(IMAGE):$(VERSION) .
14-
15-
migrate_podman_image: build_podman_image
16-
podman-hpc migrate $(IMAGE):$(VERSION)
17-
188
enter_docker_image: build_docker_image
199
docker run -it --rm $(IMAGE):$(VERSION) bash
2010

21-
launch_beaker_session:
22-
./launch-beaker-session.sh $(USERNAME)/$(IMAGE)-$(VERSION)
23-
24-
install_local_packages:
25-
./install_local_packages.sh
26-
27-
install_dependencies:
28-
./install_dependencies.sh
29-
3011
# recommended to deactivate current conda environment before running this
3112
create_environment:
32-
conda create -n $(ENVIRONMENT_NAME) python=3.8 pip
33-
conda run -n $(ENVIRONMENT_NAME) ./install_dependencies.sh
34-
conda run -n $(ENVIRONMENT_NAME) ./install_local_packages.sh
13+
conda create -n $(ENVIRONMENT_NAME) python=3.10 pip
14+
conda run --no-capture-output -n $(ENVIRONMENT_NAME) python -m pip install uv==0.2.5
15+
conda run --no-capture-output -n $(ENVIRONMENT_NAME) uv pip install -c constraints.txt -e fme[dev]
3516

36-
test_fme_unit_tests:
37-
pytest -m "not requires_gpu" --durations 10 fme/
17+
test:
18+
pytest --durations 20 .

‎README.md

+2-20
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
# ACE: AI2 Climate Emulator
22
This repo contains the inference code accompanying "ACE: A fast, skillful learned global atmospheric model for climate prediction" ([arxiv:2310.02074](https://arxiv.org/abs/2310.02074)).
33

4-
## DISCLAIMER
5-
This is rapidly changing research software. We make no guarantees of maintaining backwards compatibility.
6-
74
## Quickstart
85

96
### 1. Install
@@ -35,27 +32,12 @@ enough data to span the desired prediction period. The checkpoint is also availa
3532
`gs://ai2cm-public-requester-pays/2023-11-29-ai2-climate-emulator-v1/checkpoints/ace_ckpt.tar`.
3633

3734
### 3. Update configuration and run
38-
Update the paths in the [example config](examples/config-inference.yaml). Then in the
35+
Update the paths in the [example config](fme/docs/inference-config.yaml). Then in the
3936
`fme` conda environment, run inference with:
4037
```
41-
python -m fme.fcn_training.inference.inference examples/config-inference.yaml
38+
python -m fme.ace.inference fme/docs/inference-config.yaml
4239
```
4340

44-
## Configuration options
45-
See the `InferenceConfig` class in [this file](fme/fme/fcn_training/inference/inference.py) for
46-
description of configuration options. The [example config](examples/config-inference.yaml)
47-
shows some useful defaults for performing a 400-step simulation (100 days, with the 6-hour time step).
48-
49-
## Performance
50-
While inference can be performed without a GPU, it may be very slow in that case. In addition,
51-
I/O performance is critical for fast inference due to loading of forcing data and target data
52-
during inference.
53-
54-
## Analyzing output
55-
Various climate performance metrics are computed online by the inference code. These can be viewed via
56-
[wandb](https://wandb.ai) by setting `logging.log_to_wandb` to true and updating `logging.entity`
57-
to your wandb entity. Additionally, raw output data is saved to netCDF by the inference code.
58-
5941
## Available datasets
6042
Two versions of the dataset described in [arxiv:2310.02074](https://arxiv.org/abs/2310.02074)
6143
are available:

‎constraints.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
torch==2.1.2 # minor version matches torch in Docker image

‎docker/Dockerfile

+7-9
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,21 @@
1-
FROM nvcr.io/nvidia/pytorch:22.08-py3
1+
FROM nvcr.io/nvidia/pytorch:23.08-py3
22

33
ENV FME_DIR=/full-model
44
ENV DGLBACKEND=pytorch
55

6-
# Install gcloud
6+
# Install gcloud- used for monthly netcdf data processing script
77
# https://cloud.google.com/sdk/docs/install#deb
88
RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | \
99
tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
1010
apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && apt-get install google-cloud-cli -y
1111

1212
# install python deps
13-
COPY requirements_except_torch.txt /tmp/requirements.txt
14-
COPY requirements_no_deps.txt /tmp/requirements_no_deps.txt
13+
COPY fme/requirements.txt /tmp/requirements.txt
1514
RUN python3 -m pip install -r /tmp/requirements.txt
16-
RUN python3 -m pip install --no-deps -r /tmp/requirements_no_deps.txt
1715

18-
# copy local code
16+
# copy local code and install
1917
COPY fme ${FME_DIR}/fme
20-
COPY install_local_packages.sh ${FME_DIR}/install_local_packages.sh
18+
RUN cd $FME_DIR && pip install --no-deps -e fme
2119

22-
# install packages in full-model repo
23-
RUN cd $FME_DIR && ./install_local_packages.sh
20+
# copy after install so editing scripts does not trigger reinstall
21+
COPY scripts ${FME_DIR}/scripts

‎examples/config-inference.yaml

-17
This file was deleted.

‎fme/docs/Makefile

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Minimal makefile for Sphinx documentation
2+
#
3+
4+
# You can set these variables from the command line.
5+
SPHINXOPTS =
6+
SPHINXBUILD = python -msphinx
7+
SPHINXPROJ = fme
8+
SOURCEDIR = .
9+
BUILDDIR = _build
10+
11+
# Put it first so that "make" without argument is like "make help".
12+
help:
13+
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14+
15+
.PHONY: help Makefile
16+
17+
# Catch-all target: route all unknown targets to Sphinx using the new
18+
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19+
%: Makefile
20+
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

‎fme/docs/api.rst

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
.. _API Reference:
2+
3+
=============
4+
API Reference
5+
=============
6+
7+
fme
8+
===
9+
10+
.. automodule:: fme
11+
:members:
12+
13+
fme.ace
14+
=======
15+
16+
.. automodule:: fme.ace
17+
:members:

0 commit comments

Comments
 (0)