From f30ef53a82741b98fe97dc6695dd0cc03057bd09 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:32:17 -0500 Subject: [PATCH 1/4] updates to the current github actions --- .circleci/config.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e710e7..8948480 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: . env/bin/activate pip install -e . - check-version-updated: + check-validator-version-updated: docker: - image: cimg/python:3.10 steps: @@ -87,19 +87,9 @@ jobs: - run: name: Version is updated command: | - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/__init__.py | wc -l) == 1 ]] && echo "prompting/__init__.py has changed" + [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/validators/__init__.py | wc -l) == 1 ]] && echo "prompting/validators/__init__.py has changed" [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep VERSION | wc -l) == 1 ]] && echo "VERSION has changed" - check-changelog-updated: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - run: - name: File CHANGELOG.md is updated - command: | - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep CHANGELOG.md | wc -l) == 1 ]] && echo "CHANGELOG.md has changed" - check-version-not-released: docker: - image: cimg/python:3.10 @@ -123,12 +113,7 @@ workflows: release-branches-requirements: jobs: - - check-version-updated: - filters: - branches: - only: - - /^(release|hotfix)/.*/ - - check-changelog-updated: + - check-validator-version-updated: filters: branches: only: From 6960f82b609c7fbc73af44ad783b8d55275fba04 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:39:52 -0500 Subject: [PATCH 2/4] test commit --- .circleci/config.yml | 1 - prompting/validators/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8948480..2b7d8ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,7 +88,6 @@ jobs: name: Version is updated command: | [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/validators/__init__.py | wc -l) == 1 ]] && echo "prompting/validators/__init__.py has changed" - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep VERSION | wc -l) == 1 ]] && echo "VERSION has changed" check-version-not-released: docker: diff --git a/prompting/validators/__init__.py b/prompting/validators/__init__.py index 38fe3a4..f75e905 100644 --- a/prompting/validators/__init__.py +++ b/prompting/validators/__init__.py @@ -27,7 +27,7 @@ from . import event from . import dataset -__version__ = "2.1.3" +__version__ = "2.1.4" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0])) From f14cbaa04b14c19d4378de0e832c46c22f89df30 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:41:03 -0500 Subject: [PATCH 3/4] black --- prompting/validators/forward.py | 2 +- prompting/validators/reward/reward.py | 8 +++-- tests/validators/reward/test_reward_event.py | 32 ++++++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/prompting/validators/forward.py b/prompting/validators/forward.py index c1971c8..b354962 100644 --- a/prompting/validators/forward.py +++ b/prompting/validators/forward.py @@ -230,7 +230,7 @@ async def forward(self): base_text = ".".join(data.split(".", maxsplit=random_cutoff)[:-1]) # Create a summary task from the context. - summary_task: Task = create_summarization_task(base_text) + summary_task: Task = create_summarization_task(base_text) # Request a summary, given the original context. summarization_event = await run_step( diff --git a/prompting/validators/reward/reward.py b/prompting/validators/reward/reward.py index 54acea4..85f617e 100644 --- a/prompting/validators/reward/reward.py +++ b/prompting/validators/reward/reward.py @@ -167,11 +167,13 @@ def apply( reward_events = {f"{self.name}_{k}": v for k, v in reward_events.items()} reward_events[self.name] = filled_rewards.tolist() reward_events[self.name + "_normalized"] = filled_rewards_normalized.tolist() - + # Warns unexpected behavior for rewards if torch.isnan(filled_rewards_normalized).any(): - bt.logging.warning(f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}") - filled_rewards_normalized.nan_to_num_(nan=0.0) + bt.logging.warning( + f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}" + ) + filled_rewards_normalized.nan_to_num_(nan=0.0) # Return the filled rewards. return filled_rewards_normalized, reward_events diff --git a/tests/validators/reward/test_reward_event.py b/tests/validators/reward/test_reward_event.py index e413dd5..917a447 100644 --- a/tests/validators/reward/test_reward_event.py +++ b/tests/validators/reward/test_reward_event.py @@ -20,6 +20,7 @@ from dataclasses import fields import prompting.validators.reward as reward + class RewardEventTestCase(unittest.TestCase): """ This class contains unit tests for the RewardEvent classes. @@ -30,16 +31,15 @@ class RewardEventTestCase(unittest.TestCase): def setUp(self): self.event_classes = [ - reward.reward.BaseRewardEvent, # Represents a reward model (float) - reward.nsfw.NSFWRewardEvent, # Remaining events are filters + reward.reward.BaseRewardEvent, # Represents a reward model (float) + reward.nsfw.NSFWRewardEvent, # Remaining events are filters reward.blacklist.BlacklistRewardEvent, reward.relevance.RelevanceRewardEvent, - reward.diversity.DiversityRewardEvent + reward.diversity.DiversityRewardEvent, ] self.reward_events = {} for event in self.event_classes: - event_type = event.__name__ self.reward_events[event_type] = [] @@ -53,7 +53,7 @@ def setUp(self): for field in fields(ev): # don't modify the is_filter_model field - if field.name == 'is_filter_model': + if field.name == "is_filter_model": continue # otherwise set the field to a float (including reward) setattr(ev, field.name, 1.234) @@ -61,23 +61,29 @@ def setUp(self): self.reward_events[event_type].append(ev) def test_no_missing_rewards(self): - for name, events in self.reward_events.items(): - parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(r is not None for r in parsed['reward']), f'Events for {name} are missing rewards') - + self.assertTrue( + all(r is not None for r in parsed["reward"]), + f"Events for {name} are missing rewards", + ) def test_imputed_reward_values_are_correct(self): - for name, events in self.reward_events.items(): - expected_value = 1 if events[0].is_filter_model else 0 - indices_missing_reward = [i for i, ev in enumerate(events) if ev.reward is None] + indices_missing_reward = [ + i for i, ev in enumerate(events) if ev.reward is None + ] parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(parsed['reward'][i]==expected_value for i in indices_missing_reward), f'Events for {name} were imputed with incorrect reward value') + self.assertTrue( + all( + parsed["reward"][i] == expected_value + for i in indices_missing_reward + ), + f"Events for {name} were imputed with incorrect reward value", + ) From 62c05be98fc07819c137d6fcba62d1646a1f6630 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:46:55 -0500 Subject: [PATCH 4/4] revert version update --- prompting/validators/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/validators/__init__.py b/prompting/validators/__init__.py index f75e905..38fe3a4 100644 --- a/prompting/validators/__init__.py +++ b/prompting/validators/__init__.py @@ -27,7 +27,7 @@ from . import event from . import dataset -__version__ = "2.1.4" +__version__ = "2.1.3" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0]))