From ecf99b31bda269505b032db14bb55c1076535b4d Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Fri, 14 Jun 2024 12:13:55 +0000 Subject: [PATCH 01/14] vscode settings to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a0a48dc3a..cace748f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Ours *.DS_Store* *.idea/* +.vscode/* .env dataset lightning_logs/ From c619892c5d56c6e03fe6044bec359d3cc66312de Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Fri, 14 Jun 2024 13:12:20 +0000 Subject: [PATCH 02/14] extra arguments for pip install --- ci/Dockerfile | 4 ++++ ci/requirements_loggers.txt | 4 ++++ ci/requirements_nlp.txt | 1 + ci/requirements_optional.txt | 5 ----- setup.py | 13 +++++++++++++ 5 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 ci/requirements_loggers.txt create mode 100644 ci/requirements_nlp.txt diff --git a/ci/Dockerfile b/ci/Dockerfile index 7557d4e50..6ecf15bba 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -30,9 +30,13 @@ RUN pip install Cython==0.29.28 COPY ci/requirements.txt requirements.txt COPY ci/requirements_tests.txt requirements_tests.txt COPY ci/requirements_optional.txt requirements_optional.txt +COPY ci/requirements_loggers.txt requirements_loggers.txt +COPY ci/requirements_nlp.txt requirements_nlp.txt RUN pip install --no-cache-dir -r requirements.txt ${EXTRA_INDEX} RUN pip install --no-cache-dir -r requirements_optional.txt RUN pip install --no-cache-dir -r requirements_tests.txt +RUN pip install --no-cache-dir -r requirements_loggers.txt +RUN pip install --no-cache-dir -r requirements_nlp.txt WORKDIR /oml-lib COPY . /oml-lib diff --git a/ci/requirements_loggers.txt b/ci/requirements_loggers.txt new file mode 100644 index 000000000..3d831fea0 --- /dev/null +++ b/ci/requirements_loggers.txt @@ -0,0 +1,4 @@ +neptune>=1.0.0, <1.10.1 +wandb>=0.15.4 +mlflow>=2.0.0 +clearml>=1.5.0 diff --git a/ci/requirements_nlp.txt b/ci/requirements_nlp.txt new file mode 100644 index 000000000..c74c2f5dd --- /dev/null +++ b/ci/requirements_nlp.txt @@ -0,0 +1 @@ +transformers>=4.0 diff --git a/ci/requirements_optional.txt b/ci/requirements_optional.txt index b33f00cbf..1291952fe 100644 --- a/ci/requirements_optional.txt +++ b/ci/requirements_optional.txt @@ -1,7 +1,2 @@ grad-cam==1.4.6 jupyter>=1.0.0 -neptune>=1.0.0, <1.10.1 -wandb>=0.15.4 -mlflow>=2.0.0 -clearml>=1.5.0 -transformers>=4.0 diff --git a/setup.py b/setup.py index e7ab6e9ac..bdbfdf6a9 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,25 @@ def load_version() -> str: return re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', f.read(), re.M).group(1) +TESTS_REQUIRE = load_requirements("ci/requirements_tests.txt") +OPTIONAL_REQUIRE = load_requirements("ci/requirements_optional.txt") +LOGGERS_REQUIRE = load_requirements("ci/requirements_loggers.txt") +NLP_REQUIRE = load_requirements("ci/requirements_nlp.txt") + + setup( # technical things version=load_version(), packages=find_packages(exclude=["ci", "docs", "pipelines", "tests*"]), python_requires=">=3.8,<4.0", install_requires=load_requirements("ci/requirements.txt"), + extras_require={ + "tests": TESTS_REQUIRE, + "optional": OPTIONAL_REQUIRE, + "loggers": LOGGERS_REQUIRE, + "nlp": NLP_REQUIRE, + "all": OPTIONAL_REQUIRE + TESTS_REQUIRE + LOGGERS_REQUIRE + NLP_REQUIRE, + }, include_package_data=True, long_description=Path("README.md").read_text(), long_description_content_type="text/markdown", From 226cdca564e1be6acd3cd4f3d761aa3221d56955 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Fri, 14 Jun 2024 13:51:38 +0000 Subject: [PATCH 03/14] rename "all" to "full" --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bdbfdf6a9..570b5c574 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def load_version() -> str: "optional": OPTIONAL_REQUIRE, "loggers": LOGGERS_REQUIRE, "nlp": NLP_REQUIRE, - "all": OPTIONAL_REQUIRE + TESTS_REQUIRE + LOGGERS_REQUIRE + NLP_REQUIRE, + "full": OPTIONAL_REQUIRE + TESTS_REQUIRE + LOGGERS_REQUIRE + NLP_REQUIRE, }, include_package_data=True, long_description=Path("README.md").read_text(), From d7495cbcc3579a523f631f8fdf865ae8528f1f8d Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Sat, 15 Jun 2024 15:40:04 +0000 Subject: [PATCH 04/14] reqs - loggers moved to tests --- ci/Dockerfile | 2 -- ci/requirements.txt | 1 - ci/requirements_loggers.txt | 4 ---- ci/requirements_tests.txt | 5 +++++ setup.py | 4 +--- 5 files changed, 6 insertions(+), 10 deletions(-) delete mode 100644 ci/requirements_loggers.txt diff --git a/ci/Dockerfile b/ci/Dockerfile index 6ecf15bba..cfc941e44 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -30,12 +30,10 @@ RUN pip install Cython==0.29.28 COPY ci/requirements.txt requirements.txt COPY ci/requirements_tests.txt requirements_tests.txt COPY ci/requirements_optional.txt requirements_optional.txt -COPY ci/requirements_loggers.txt requirements_loggers.txt COPY ci/requirements_nlp.txt requirements_nlp.txt RUN pip install --no-cache-dir -r requirements.txt ${EXTRA_INDEX} RUN pip install --no-cache-dir -r requirements_optional.txt RUN pip install --no-cache-dir -r requirements_tests.txt -RUN pip install --no-cache-dir -r requirements_loggers.txt RUN pip install --no-cache-dir -r requirements_nlp.txt WORKDIR /oml-lib diff --git a/ci/requirements.txt b/ci/requirements.txt index 095b4995c..945a8ba52 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -9,4 +9,3 @@ matplotlib>3.5.2 hydra-core==1.3.0 validators>=0.18.0 gdown>=4.0 -tensorboard diff --git a/ci/requirements_loggers.txt b/ci/requirements_loggers.txt deleted file mode 100644 index 3d831fea0..000000000 --- a/ci/requirements_loggers.txt +++ /dev/null @@ -1,4 +0,0 @@ -neptune>=1.0.0, <1.10.1 -wandb>=0.15.4 -mlflow>=2.0.0 -clearml>=1.5.0 diff --git a/ci/requirements_tests.txt b/ci/requirements_tests.txt index 1c5f438fc..ff9690716 100644 --- a/ci/requirements_tests.txt +++ b/ci/requirements_tests.txt @@ -1,3 +1,8 @@ pytest>=7.2 pre-commit==2.15.0 python-dotenv>=0.17.0 +neptune>=1.0.0, <1.10.1 +wandb>=0.15.4 +mlflow>=2.0.0 +clearml>=1.5.0 +tensorboard diff --git a/setup.py b/setup.py index 570b5c574..afdb2bc42 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,6 @@ def load_version() -> str: TESTS_REQUIRE = load_requirements("ci/requirements_tests.txt") OPTIONAL_REQUIRE = load_requirements("ci/requirements_optional.txt") -LOGGERS_REQUIRE = load_requirements("ci/requirements_loggers.txt") NLP_REQUIRE = load_requirements("ci/requirements_nlp.txt") @@ -33,9 +32,8 @@ def load_version() -> str: extras_require={ "tests": TESTS_REQUIRE, "optional": OPTIONAL_REQUIRE, - "loggers": LOGGERS_REQUIRE, "nlp": NLP_REQUIRE, - "full": OPTIONAL_REQUIRE + TESTS_REQUIRE + LOGGERS_REQUIRE + NLP_REQUIRE, + "full": OPTIONAL_REQUIRE + NLP_REQUIRE, }, include_package_data=True, long_description=Path("README.md").read_text(), From 9a34d0a09ee3ecf85ebfaef358a83647e57356a1 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Sun, 16 Jun 2024 08:44:17 +0000 Subject: [PATCH 05/14] fix requierements --- ci/Dockerfile | 2 -- ci/requirements_optional.txt | 2 -- ci/requirements_tests.txt | 1 + setup.py | 6 +----- 4 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 ci/requirements_optional.txt diff --git a/ci/Dockerfile b/ci/Dockerfile index cfc941e44..2bfb1dd57 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -29,10 +29,8 @@ RUN pip install Cython==0.29.28 ### Python libs COPY ci/requirements.txt requirements.txt COPY ci/requirements_tests.txt requirements_tests.txt -COPY ci/requirements_optional.txt requirements_optional.txt COPY ci/requirements_nlp.txt requirements_nlp.txt RUN pip install --no-cache-dir -r requirements.txt ${EXTRA_INDEX} -RUN pip install --no-cache-dir -r requirements_optional.txt RUN pip install --no-cache-dir -r requirements_tests.txt RUN pip install --no-cache-dir -r requirements_nlp.txt diff --git a/ci/requirements_optional.txt b/ci/requirements_optional.txt deleted file mode 100644 index 1291952fe..000000000 --- a/ci/requirements_optional.txt +++ /dev/null @@ -1,2 +0,0 @@ -grad-cam==1.4.6 -jupyter>=1.0.0 diff --git a/ci/requirements_tests.txt b/ci/requirements_tests.txt index ff9690716..2582af0e4 100644 --- a/ci/requirements_tests.txt +++ b/ci/requirements_tests.txt @@ -6,3 +6,4 @@ wandb>=0.15.4 mlflow>=2.0.0 clearml>=1.5.0 tensorboard +grad-cam==1.4.6 diff --git a/setup.py b/setup.py index afdb2bc42..1902a648b 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,6 @@ def load_version() -> str: return re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', f.read(), re.M).group(1) -TESTS_REQUIRE = load_requirements("ci/requirements_tests.txt") -OPTIONAL_REQUIRE = load_requirements("ci/requirements_optional.txt") NLP_REQUIRE = load_requirements("ci/requirements_nlp.txt") @@ -30,10 +28,8 @@ def load_version() -> str: python_requires=">=3.8,<4.0", install_requires=load_requirements("ci/requirements.txt"), extras_require={ - "tests": TESTS_REQUIRE, - "optional": OPTIONAL_REQUIRE, "nlp": NLP_REQUIRE, - "full": OPTIONAL_REQUIRE + NLP_REQUIRE, + "full": NLP_REQUIRE, # later will be cv and audio }, include_package_data=True, long_description=Path("README.md").read_text(), From d08dc47a72edcc49f7acb1aaa759a0b2dbb6ed12 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Sun, 16 Jun 2024 09:03:26 +0000 Subject: [PATCH 06/14] docs update for install extra --- README.md | 5 +++++ docs/readme/installation.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 8396559a5..755011cfa 100644 --- a/README.md +++ b/README.md @@ -338,6 +338,11 @@ for our paper pip install -U open-metric-learning ``` +If you need OML for NLP, install the extra requirements with: +```shell +pip install -U open-metric-learning[nlp] +``` +
DockerHub ```shell diff --git a/docs/readme/installation.md b/docs/readme/installation.md index 972141334..098b56ade 100644 --- a/docs/readme/installation.md +++ b/docs/readme/installation.md @@ -2,6 +2,11 @@ pip install -U open-metric-learning ``` +If you need OML for NLP, install the extra requirements with: +```shell +pip install -U open-metric-learning[nlp] +``` +
DockerHub ```shell From 42541791e4c2fb4fcefdd23782143749a2fc301a Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Sun, 16 Jun 2024 11:47:20 +0000 Subject: [PATCH 07/14] fix req's - loggers and gradcam to optional --- ci/Dockerfile | 2 ++ ci/requirements_optional.txt | 6 ++++++ ci/requirements_tests.txt | 6 ------ 3 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 ci/requirements_optional.txt diff --git a/ci/Dockerfile b/ci/Dockerfile index 2bfb1dd57..b2c5bffb4 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -29,9 +29,11 @@ RUN pip install Cython==0.29.28 ### Python libs COPY ci/requirements.txt requirements.txt COPY ci/requirements_tests.txt requirements_tests.txt +COPY ci/requirements_optional.txt requirements_optional.txt COPY ci/requirements_nlp.txt requirements_nlp.txt RUN pip install --no-cache-dir -r requirements.txt ${EXTRA_INDEX} RUN pip install --no-cache-dir -r requirements_tests.txt +RUN pip install --no-cache-dir -r requirements_optional.txt RUN pip install --no-cache-dir -r requirements_nlp.txt WORKDIR /oml-lib diff --git a/ci/requirements_optional.txt b/ci/requirements_optional.txt new file mode 100644 index 000000000..a1960804d --- /dev/null +++ b/ci/requirements_optional.txt @@ -0,0 +1,6 @@ +neptune>=1.0.0, <1.10.1 +wandb>=0.15.4 +mlflow>=2.0.0 +clearml>=1.5.0 +tensorboard +grad-cam==1.4.6 diff --git a/ci/requirements_tests.txt b/ci/requirements_tests.txt index 2582af0e4..1c5f438fc 100644 --- a/ci/requirements_tests.txt +++ b/ci/requirements_tests.txt @@ -1,9 +1,3 @@ pytest>=7.2 pre-commit==2.15.0 python-dotenv>=0.17.0 -neptune>=1.0.0, <1.10.1 -wandb>=0.15.4 -mlflow>=2.0.0 -clearml>=1.5.0 -tensorboard -grad-cam==1.4.6 From dfcaacd9768af3928479555ed0d7c08defa567d2 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 17 Jun 2024 05:32:51 +0000 Subject: [PATCH 08/14] fix for github workflow --- .github/workflows/tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9754a2888..b5c58e74e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -26,6 +26,8 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install -r ci/requirements_tests.txt + python3 -m pip install -r ci/requirements_optional.txt + python3 -m pip install -r ci/requirements_nlp.txt python3 -m pip install -e . - name: Running short tests run: | From 9484153c741539bd808a9920dba217f9b4a9ee60 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 17 Jun 2024 06:01:41 +0000 Subject: [PATCH 09/14] fix for numpy 2.0 --- oml/utils/misc_torch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oml/utils/misc_torch.py b/oml/utils/misc_torch.py index 75009d3c3..defd75d6a 100644 --- a/oml/utils/misc_torch.py +++ b/oml/utils/misc_torch.py @@ -7,7 +7,7 @@ import torch from torch import Tensor, cdist -TSingleValues = Union[int, float, np.float_, np.int_, torch.Tensor] +TSingleValues = Union[int, float, np.float64, np.int_, torch.Tensor] TSequenceValues = Union[List[float], Tuple[float, ...], np.ndarray, torch.Tensor] TOnlineValues = Union[TSingleValues, TSequenceValues] From 00d5d7e5465a20e7a91074eb512d04a5ceb5de1d Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 17 Jun 2024 06:37:28 +0000 Subject: [PATCH 10/14] rename pip extra "full" to "all" --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1902a648b..a711d2073 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def load_version() -> str: install_requires=load_requirements("ci/requirements.txt"), extras_require={ "nlp": NLP_REQUIRE, - "full": NLP_REQUIRE, # later will be cv and audio + "all": NLP_REQUIRE, # later will be cv and audio }, include_package_data=True, long_description=Path("README.md").read_text(), From 6ae8471b06b01f388e1c3bc58d761c825ad0b4ea Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 17 Jun 2024 08:56:28 +0000 Subject: [PATCH 11/14] fix github workflow --- .github/workflows/tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b5c58e74e..8b7d08fbe 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -26,8 +26,6 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install -r ci/requirements_tests.txt - python3 -m pip install -r ci/requirements_optional.txt - python3 -m pip install -r ci/requirements_nlp.txt python3 -m pip install -e . - name: Running short tests run: | @@ -51,6 +49,7 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -r ci/requirements_tests.txt python3 -m pip install -r ci/requirements_optional.txt + python3 -m pip install -r ci/requirements_nlp.txt python3 -m pip install -e . - name: Running all tests run: | From 5573301eaa25e42ba46d97565c591731a9704e2c Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 17 Jun 2024 11:06:04 +0000 Subject: [PATCH 12/14] numpy < 2.0 --- ci/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements.txt b/ci/requirements.txt index 945a8ba52..eed76ed46 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -1,4 +1,5 @@ torch>=2.0,<=2.1.1 +numpy>=1.24.4,<2.0 pytorch-lightning>=2.0,<=2.1.2 torchvision<=0.16.1 albumentations>=1.0,<=1.3.1 From 1f90d4859584932a6cd1463471dd29a8b6897fc7 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 17 Jun 2024 12:06:46 +0000 Subject: [PATCH 13/14] typo --- oml/utils/misc_torch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oml/utils/misc_torch.py b/oml/utils/misc_torch.py index defd75d6a..c93b05e82 100644 --- a/oml/utils/misc_torch.py +++ b/oml/utils/misc_torch.py @@ -101,7 +101,7 @@ def elementwise_dist(x1: Tensor, x2: Tensor, p: int = 2) -> Tensor: assert len(x1.shape) == len(x2.shape) == 2 assert x1.shape == x2.shape - # we need an extra dim here to avoid pairwise behaviour of torch.cdist + # we need an extra dim here to avoid pairwise behavior of torch.cdist if len(x1.shape) == 2: x1 = x1.unsqueeze(1) x2 = x2.unsqueeze(1) @@ -263,7 +263,7 @@ def calc_with_batch(self, val: TSequenceValues) -> None: class OnlineDict(MutableMapping): # type: ignore """ - We don't inherite from built-in 'dict' due to internal C optimization. We mimic to dict with MutableMapping + We don't inherit from built-in 'dict' due to internal C optimization. We mimic to dict with MutableMapping https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/ """ @@ -395,7 +395,7 @@ def __init__(self, embeddings: torch.Tensor): def _fit(self, embeddings: torch.Tensor) -> None: """ - Perform the PCA. Evaluate ``components``, ``expoained_variance``, ``explained_variance_ratio``, + Perform the PCA. Evaluate ``components``, ``explained_variance``, ``explained_variance_ratio``, ``singular_values``, ``mean``. Args: @@ -470,7 +470,7 @@ def calc_principal_axes_number(self, pcf_variance: Tuple[float, ...]) -> torch.T Let :math:`\\lambda_1, \\ldots, \\lambda_d\\in\\mathbb{R}` be a set of eigenvalues of the covariance matrix of :math:`X` sorted in descending order. Then for a given value of desired explained variance :math:`r`, - the number of principal components that explaines :math:`r\\cdot 100\\%%` variance is the largest integer + the number of principal components that explains :math:`r\\cdot 100\\%%` variance is the largest integer :math:`n` such that .. math:: From e42e7408e9d008009465069d7378bd742bf1b805 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 17 Jun 2024 14:03:28 +0000 Subject: [PATCH 14/14] tensorboard moved back to requirements --- ci/requirements.txt | 1 + ci/requirements_optional.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements.txt b/ci/requirements.txt index eed76ed46..ceca04411 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -10,3 +10,4 @@ matplotlib>3.5.2 hydra-core==1.3.0 validators>=0.18.0 gdown>=4.0 +tensorboard diff --git a/ci/requirements_optional.txt b/ci/requirements_optional.txt index a1960804d..303f58d8b 100644 --- a/ci/requirements_optional.txt +++ b/ci/requirements_optional.txt @@ -2,5 +2,4 @@ neptune>=1.0.0, <1.10.1 wandb>=0.15.4 mlflow>=2.0.0 clearml>=1.5.0 -tensorboard grad-cam==1.4.6