From 80427a51dda0edd78c931077ca69213c444887aa Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Thu, 6 Jul 2023 09:17:56 -0400 Subject: [PATCH] more formatting cleanup (#1015) * more formatting cleanup Signed-off-by: Niels Bantilan * more cleanup Signed-off-by: Niels Bantilan * use requirements.in for mlflow plugin example Signed-off-by: Niels Bantilan --------- Signed-off-by: Niels Bantilan Co-authored-by: Kevin Su --- .../athena_plugin/athena_plugin/athena.py | 4 +- examples/mlflow_plugin/Dockerfile | 3 +- examples/mlflow_plugin/requirements.in | 3 + examples/mlflow_plugin/requirements.txt | 411 ++++++++++++++++++ .../pytorch_single_node_and_gpu.py | 2 +- 5 files changed, 419 insertions(+), 4 deletions(-) create mode 100644 examples/mlflow_plugin/requirements.in create mode 100644 examples/mlflow_plugin/requirements.txt diff --git a/examples/athena_plugin/athena_plugin/athena.py b/examples/athena_plugin/athena_plugin/athena.py index 859f1739f..6fb163f31 100644 --- a/examples/athena_plugin/athena_plugin/athena.py +++ b/examples/athena_plugin/athena_plugin/athena.py @@ -86,11 +86,11 @@ def no_io_wf(): SELECT * FROM vaccinations where iso_code like '{{ .Inputs.iso_code }}' """, # While we define a generic schema as the output here, Flyte supports more strongly typed schemas to provide - # better compile-time checks for task compatibility. Refer to :py:class:`flytekit.FlyteSchema` for more details + # better compile-time checks for task compatibility. Refer to {py:class}`flytekit.FlyteSchema` for more details output_schema_type=FlyteSchema, # Cache the output data so we don't have to re-run the query in future workflow iterations # should we decide to change how we manipulate data downstream. - # For more information about caching, visit :ref:`Task Caching ` + # For more information about caching, visit {ref}`Task Caching ` cache=True, cache_version="1.0", ) diff --git a/examples/mlflow_plugin/Dockerfile b/examples/mlflow_plugin/Dockerfile index cdea38a93..fa39c8f4a 100644 --- a/examples/mlflow_plugin/Dockerfile +++ b/examples/mlflow_plugin/Dockerfile @@ -24,7 +24,8 @@ RUN python3 -m venv ${VENV} ENV PATH="${VENV}/bin:$PATH" # Install Python dependencies -RUN pip install flytekitplugins-mlflow tensorflow protobuf==3.20.1 +COPY requirements.txt /root +RUN pip install -r /root/requirements.txt # Copy the actual code COPY . /root/ diff --git a/examples/mlflow_plugin/requirements.in b/examples/mlflow_plugin/requirements.in new file mode 100644 index 000000000..c5f8e5ea3 --- /dev/null +++ b/examples/mlflow_plugin/requirements.in @@ -0,0 +1,3 @@ +flytekitplugins-mlflow +tensorflow +protobuf==3.20.1 diff --git a/examples/mlflow_plugin/requirements.txt b/examples/mlflow_plugin/requirements.txt new file mode 100644 index 000000000..da24d689b --- /dev/null +++ b/examples/mlflow_plugin/requirements.txt @@ -0,0 +1,411 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --resolver=backtracking requirements.in +# +absl-py==1.4.0 + # via + # tensorboard + # tensorflow +alembic==1.11.1 + # via mlflow +arrow==1.2.3 + # via jinja2-time +astunparse==1.6.3 + # via tensorflow +binaryornot==0.4.4 + # via cookiecutter +blinker==1.6.2 + # via flask +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests +cffi==1.15.1 + # via cryptography +chardet==5.1.0 + # via binaryornot +charset-normalizer==3.1.0 + # via requests +click==8.1.3 + # via + # cookiecutter + # databricks-cli + # flask + # flytekit + # mlflow +cloudpickle==2.2.1 + # via + # flytekit + # mlflow +contourpy==1.1.0 + # via matplotlib +cookiecutter==2.1.1 + # via flytekit +croniter==1.4.1 + # via flytekit +cryptography==41.0.1 + # via pyopenssl +cycler==0.11.0 + # via matplotlib +databricks-cli==0.17.7 + # via mlflow +dataclasses-json==0.5.9 + # via flytekit +decorator==5.1.1 + # via retry +deprecated==1.2.14 + # via flytekit +diskcache==5.6.1 + # via flytekit +docker==6.1.3 + # via + # flytekit + # mlflow +docker-image-py==0.1.12 + # via flytekit +docstring-parser==0.15 + # via flytekit +entrypoints==0.4 + # via mlflow +flask==2.3.2 + # via mlflow +flatbuffers==1.12 + # via tensorflow +flyteidl==1.2.11 + # via flytekit +flytekit==1.2.11 + # via flytekitplugins-mlflow +flytekitplugins-mlflow==1.7.0 + # via -r requirements.in +fonttools==4.40.0 + # via matplotlib +gast==0.4.0 + # via tensorflow +gitdb==4.0.10 + # via gitpython +gitpython==3.1.31 + # via + # flytekit + # mlflow +google-auth==2.21.0 + # via + # google-auth-oauthlib + # kubernetes + # tensorboard +google-auth-oauthlib==0.4.6 + # via tensorboard +google-pasta==0.2.0 + # via tensorflow +googleapis-common-protos==1.56.4 + # via + # flyteidl + # grpcio-status +greenlet==2.0.2 + # via sqlalchemy +grpcio==1.48.2 + # via + # flytekit + # grpcio-status + # tensorboard + # tensorflow +grpcio-status==1.48.2 + # via flytekit +gunicorn==20.1.0 + # via mlflow +h5py==3.9.0 + # via tensorflow +idna==3.4 + # via requests +importlib-metadata==6.7.0 + # via + # alembic + # flask + # flytekit + # keyring + # markdown + # mlflow +importlib-resources==5.12.0 + # via + # alembic + # keyring + # matplotlib +itsdangerous==2.1.2 + # via flask +jaraco-classes==3.2.3 + # via keyring +jinja2==3.1.2 + # via + # cookiecutter + # flask + # jinja2-time + # mlflow +jinja2-time==0.2.0 + # via cookiecutter +joblib==1.3.1 + # via + # flytekit + # scikit-learn +keras==2.9.0 + # via tensorflow +keras-preprocessing==1.1.2 + # via tensorflow +keyring==24.2.0 + # via flytekit +kiwisolver==1.4.4 + # via matplotlib +kubernetes==26.1.0 + # via flytekit +libclang==16.0.0 + # via tensorflow +mako==1.2.4 + # via alembic +markdown==3.4.3 + # via + # mlflow + # tensorboard +markupsafe==2.1.3 + # via + # jinja2 + # mako + # werkzeug +marshmallow==3.19.0 + # via + # dataclasses-json + # marshmallow-enum + # marshmallow-jsonschema +marshmallow-enum==1.5.1 + # via dataclasses-json +marshmallow-jsonschema==0.13.0 + # via flytekit +matplotlib==3.7.2 + # via mlflow +mlflow==2.4.1 + # via flytekitplugins-mlflow +more-itertools==9.1.0 + # via jaraco-classes +mypy-extensions==1.0.0 + # via typing-inspect +natsort==8.4.0 + # via flytekit +numpy==1.23.5 + # via + # contourpy + # flytekit + # h5py + # keras-preprocessing + # matplotlib + # mlflow + # opt-einsum + # pandas + # pyarrow + # scikit-learn + # scipy + # tensorboard + # tensorflow +oauthlib==3.2.2 + # via + # databricks-cli + # requests-oauthlib +opt-einsum==3.3.0 + # via tensorflow +packaging==23.1 + # via + # docker + # marshmallow + # matplotlib + # mlflow + # plotly + # tensorflow +pandas==1.5.3 + # via + # flytekit + # mlflow +pillow==10.0.0 + # via matplotlib +plotly==5.15.0 + # via flytekitplugins-mlflow +protobuf==3.20.1 + # via + # -r requirements.in + # flyteidl + # flytekit + # googleapis-common-protos + # grpcio-status + # mlflow + # protoc-gen-swagger + # tensorboard + # tensorflow +protoc-gen-swagger==0.1.0 + # via flyteidl +py==1.11.0 + # via retry +pyarrow==10.0.1 + # via + # flytekit + # mlflow +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pycparser==2.21 + # via cffi +pyjwt==2.7.0 + # via databricks-cli +pyopenssl==23.2.0 + # via flytekit +pyparsing==3.0.9 + # via matplotlib +python-dateutil==2.8.2 + # via + # arrow + # croniter + # flytekit + # kubernetes + # matplotlib + # pandas +python-json-logger==2.0.7 + # via flytekit +python-slugify==8.0.1 + # via cookiecutter +pytimeparse==1.1.8 + # via flytekit +pytz==2023.3 + # via + # flytekit + # mlflow + # pandas +pyyaml==6.0 + # via + # cookiecutter + # flytekit + # kubernetes + # mlflow + # responses +querystring-parser==1.2.4 + # via mlflow +regex==2023.6.3 + # via docker-image-py +requests==2.31.0 + # via + # cookiecutter + # databricks-cli + # docker + # flytekit + # kubernetes + # mlflow + # requests-oauthlib + # responses + # tensorboard +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 + # via flytekit +retry==0.9.2 + # via flytekit +rsa==4.9 + # via google-auth +scikit-learn==1.3.0 + # via mlflow +scipy==1.10.1 + # via + # mlflow + # scikit-learn +six==1.16.0 + # via + # astunparse + # databricks-cli + # google-auth + # google-pasta + # grpcio + # keras-preprocessing + # kubernetes + # python-dateutil + # querystring-parser + # tensorflow +smmap==5.0.0 + # via gitdb +sortedcontainers==2.4.0 + # via flytekit +sqlalchemy==2.0.18 + # via + # alembic + # mlflow +sqlparse==0.4.4 + # via mlflow +statsd==3.3.0 + # via flytekit +tabulate==0.9.0 + # via databricks-cli +tenacity==8.2.2 + # via plotly +tensorboard==2.9.0 + # via tensorflow +tensorboard-data-server==0.6.1 + # via tensorboard +tensorboard-plugin-wit==1.8.1 + # via tensorboard +tensorflow==2.9.0 + # via -r requirements.in +tensorflow-estimator==2.9.0 + # via tensorflow +tensorflow-io-gcs-filesystem==0.32.0 + # via tensorflow +termcolor==2.3.0 + # via tensorflow +text-unidecode==1.3 + # via python-slugify +threadpoolctl==3.1.0 + # via scikit-learn +types-pyyaml==6.0.12.10 + # via responses +typing-extensions==4.7.1 + # via + # alembic + # flytekit + # sqlalchemy + # tensorflow + # typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json +urllib3==1.26.16 + # via + # databricks-cli + # docker + # flytekit + # google-auth + # kubernetes + # requests + # responses +websocket-client==1.6.1 + # via + # docker + # kubernetes +werkzeug==2.3.6 + # via + # flask + # tensorboard +wheel==0.40.0 + # via + # astunparse + # flytekit + # tensorboard +wrapt==1.15.0 + # via + # deprecated + # flytekit + # tensorflow +zipp==3.15.0 + # via + # importlib-metadata + # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py b/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py index ac1a78b82..0d9382fdb 100644 --- a/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py +++ b/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py @@ -318,7 +318,7 @@ def pytorch_mnist_task(hp: Hyperparameters) -> TrainingOutputs: train(model, device, training_data_loader, optimizer, epoch, hp.log_interval) accuracies.append(test(model, device, test_data_loader)) - # after training the model, we can simply save it to disk and return it from the Flyte task as a :py:class:`flytekit.types.file.FlyteFile` + # after training the model, we can simply save it to disk and return it from the Flyte task as a {py:class}`flytekit.types.file.FlyteFile` # type, which is the ``PythonPickledFile``. ``PythonPickledFile`` is simply a decorator on the ``FlyteFile`` that records the format # of the serialized model as ``pickled`` model_file = "mnist_cnn.pt"