From f30ef53a82741b98fe97dc6695dd0cc03057bd09 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:32:17 -0500 Subject: [PATCH 01/13] updates to the current github actions --- .circleci/config.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e710e7..8948480 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: . env/bin/activate pip install -e . - check-version-updated: + check-validator-version-updated: docker: - image: cimg/python:3.10 steps: @@ -87,19 +87,9 @@ jobs: - run: name: Version is updated command: | - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/__init__.py | wc -l) == 1 ]] && echo "prompting/__init__.py has changed" + [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/validators/__init__.py | wc -l) == 1 ]] && echo "prompting/validators/__init__.py has changed" [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep VERSION | wc -l) == 1 ]] && echo "VERSION has changed" - check-changelog-updated: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - run: - name: File CHANGELOG.md is updated - command: | - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep CHANGELOG.md | wc -l) == 1 ]] && echo "CHANGELOG.md has changed" - check-version-not-released: docker: - image: cimg/python:3.10 @@ -123,12 +113,7 @@ workflows: release-branches-requirements: jobs: - - check-version-updated: - filters: - branches: - only: - - /^(release|hotfix)/.*/ - - check-changelog-updated: + - check-validator-version-updated: filters: branches: only: From 6960f82b609c7fbc73af44ad783b8d55275fba04 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:39:52 -0500 Subject: [PATCH 02/13] test commit --- .circleci/config.yml | 1 - prompting/validators/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8948480..2b7d8ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,7 +88,6 @@ jobs: name: Version is updated command: | [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/validators/__init__.py | wc -l) == 1 ]] && echo "prompting/validators/__init__.py has changed" - [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep VERSION | wc -l) == 1 ]] && echo "VERSION has changed" check-version-not-released: docker: diff --git a/prompting/validators/__init__.py b/prompting/validators/__init__.py index 38fe3a4..f75e905 100644 --- a/prompting/validators/__init__.py +++ b/prompting/validators/__init__.py @@ -27,7 +27,7 @@ from . import event from . import dataset -__version__ = "2.1.3" +__version__ = "2.1.4" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0])) From f14cbaa04b14c19d4378de0e832c46c22f89df30 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:41:03 -0500 Subject: [PATCH 03/13] black --- prompting/validators/forward.py | 2 +- prompting/validators/reward/reward.py | 8 +++-- tests/validators/reward/test_reward_event.py | 32 ++++++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/prompting/validators/forward.py b/prompting/validators/forward.py index c1971c8..b354962 100644 --- a/prompting/validators/forward.py +++ b/prompting/validators/forward.py @@ -230,7 +230,7 @@ async def forward(self): base_text = ".".join(data.split(".", maxsplit=random_cutoff)[:-1]) # Create a summary task from the context. - summary_task: Task = create_summarization_task(base_text) + summary_task: Task = create_summarization_task(base_text) # Request a summary, given the original context. summarization_event = await run_step( diff --git a/prompting/validators/reward/reward.py b/prompting/validators/reward/reward.py index 54acea4..85f617e 100644 --- a/prompting/validators/reward/reward.py +++ b/prompting/validators/reward/reward.py @@ -167,11 +167,13 @@ def apply( reward_events = {f"{self.name}_{k}": v for k, v in reward_events.items()} reward_events[self.name] = filled_rewards.tolist() reward_events[self.name + "_normalized"] = filled_rewards_normalized.tolist() - + # Warns unexpected behavior for rewards if torch.isnan(filled_rewards_normalized).any(): - bt.logging.warning(f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}") - filled_rewards_normalized.nan_to_num_(nan=0.0) + bt.logging.warning( + f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}" + ) + filled_rewards_normalized.nan_to_num_(nan=0.0) # Return the filled rewards. return filled_rewards_normalized, reward_events diff --git a/tests/validators/reward/test_reward_event.py b/tests/validators/reward/test_reward_event.py index e413dd5..917a447 100644 --- a/tests/validators/reward/test_reward_event.py +++ b/tests/validators/reward/test_reward_event.py @@ -20,6 +20,7 @@ from dataclasses import fields import prompting.validators.reward as reward + class RewardEventTestCase(unittest.TestCase): """ This class contains unit tests for the RewardEvent classes. @@ -30,16 +31,15 @@ class RewardEventTestCase(unittest.TestCase): def setUp(self): self.event_classes = [ - reward.reward.BaseRewardEvent, # Represents a reward model (float) - reward.nsfw.NSFWRewardEvent, # Remaining events are filters + reward.reward.BaseRewardEvent, # Represents a reward model (float) + reward.nsfw.NSFWRewardEvent, # Remaining events are filters reward.blacklist.BlacklistRewardEvent, reward.relevance.RelevanceRewardEvent, - reward.diversity.DiversityRewardEvent + reward.diversity.DiversityRewardEvent, ] self.reward_events = {} for event in self.event_classes: - event_type = event.__name__ self.reward_events[event_type] = [] @@ -53,7 +53,7 @@ def setUp(self): for field in fields(ev): # don't modify the is_filter_model field - if field.name == 'is_filter_model': + if field.name == "is_filter_model": continue # otherwise set the field to a float (including reward) setattr(ev, field.name, 1.234) @@ -61,23 +61,29 @@ def setUp(self): self.reward_events[event_type].append(ev) def test_no_missing_rewards(self): - for name, events in self.reward_events.items(): - parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(r is not None for r in parsed['reward']), f'Events for {name} are missing rewards') - + self.assertTrue( + all(r is not None for r in parsed["reward"]), + f"Events for {name} are missing rewards", + ) def test_imputed_reward_values_are_correct(self): - for name, events in self.reward_events.items(): - expected_value = 1 if events[0].is_filter_model else 0 - indices_missing_reward = [i for i, ev in enumerate(events) if ev.reward is None] + indices_missing_reward = [ + i for i, ev in enumerate(events) if ev.reward is None + ] parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(parsed['reward'][i]==expected_value for i in indices_missing_reward), f'Events for {name} were imputed with incorrect reward value') + self.assertTrue( + all( + parsed["reward"][i] == expected_value + for i in indices_missing_reward + ), + f"Events for {name} were imputed with incorrect reward value", + ) From c82e6ae750061b23f9e125affabcc7eb212bb4aa Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:44:14 -0500 Subject: [PATCH 04/13] black --- prompting/validators/forward.py | 2 +- prompting/validators/reward/reward.py | 8 +++-- tests/validators/reward/test_reward_event.py | 32 ++++++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/prompting/validators/forward.py b/prompting/validators/forward.py index c1971c8..b354962 100644 --- a/prompting/validators/forward.py +++ b/prompting/validators/forward.py @@ -230,7 +230,7 @@ async def forward(self): base_text = ".".join(data.split(".", maxsplit=random_cutoff)[:-1]) # Create a summary task from the context. - summary_task: Task = create_summarization_task(base_text) + summary_task: Task = create_summarization_task(base_text) # Request a summary, given the original context. summarization_event = await run_step( diff --git a/prompting/validators/reward/reward.py b/prompting/validators/reward/reward.py index 54acea4..85f617e 100644 --- a/prompting/validators/reward/reward.py +++ b/prompting/validators/reward/reward.py @@ -167,11 +167,13 @@ def apply( reward_events = {f"{self.name}_{k}": v for k, v in reward_events.items()} reward_events[self.name] = filled_rewards.tolist() reward_events[self.name + "_normalized"] = filled_rewards_normalized.tolist() - + # Warns unexpected behavior for rewards if torch.isnan(filled_rewards_normalized).any(): - bt.logging.warning(f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}") - filled_rewards_normalized.nan_to_num_(nan=0.0) + bt.logging.warning( + f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}" + ) + filled_rewards_normalized.nan_to_num_(nan=0.0) # Return the filled rewards. return filled_rewards_normalized, reward_events diff --git a/tests/validators/reward/test_reward_event.py b/tests/validators/reward/test_reward_event.py index e413dd5..917a447 100644 --- a/tests/validators/reward/test_reward_event.py +++ b/tests/validators/reward/test_reward_event.py @@ -20,6 +20,7 @@ from dataclasses import fields import prompting.validators.reward as reward + class RewardEventTestCase(unittest.TestCase): """ This class contains unit tests for the RewardEvent classes. @@ -30,16 +31,15 @@ class RewardEventTestCase(unittest.TestCase): def setUp(self): self.event_classes = [ - reward.reward.BaseRewardEvent, # Represents a reward model (float) - reward.nsfw.NSFWRewardEvent, # Remaining events are filters + reward.reward.BaseRewardEvent, # Represents a reward model (float) + reward.nsfw.NSFWRewardEvent, # Remaining events are filters reward.blacklist.BlacklistRewardEvent, reward.relevance.RelevanceRewardEvent, - reward.diversity.DiversityRewardEvent + reward.diversity.DiversityRewardEvent, ] self.reward_events = {} for event in self.event_classes: - event_type = event.__name__ self.reward_events[event_type] = [] @@ -53,7 +53,7 @@ def setUp(self): for field in fields(ev): # don't modify the is_filter_model field - if field.name == 'is_filter_model': + if field.name == "is_filter_model": continue # otherwise set the field to a float (including reward) setattr(ev, field.name, 1.234) @@ -61,23 +61,29 @@ def setUp(self): self.reward_events[event_type].append(ev) def test_no_missing_rewards(self): - for name, events in self.reward_events.items(): - parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(r is not None for r in parsed['reward']), f'Events for {name} are missing rewards') - + self.assertTrue( + all(r is not None for r in parsed["reward"]), + f"Events for {name} are missing rewards", + ) def test_imputed_reward_values_are_correct(self): - for name, events in self.reward_events.items(): - expected_value = 1 if events[0].is_filter_model else 0 - indices_missing_reward = [i for i, ev in enumerate(events) if ev.reward is None] + indices_missing_reward = [ + i for i, ev in enumerate(events) if ev.reward is None + ] parsed = reward.reward.BaseRewardEvent.parse_reward_events(events) # Ensure that all rewards are not None - self.assertTrue(all(parsed['reward'][i]==expected_value for i in indices_missing_reward), f'Events for {name} were imputed with incorrect reward value') + self.assertTrue( + all( + parsed["reward"][i] == expected_value + for i in indices_missing_reward + ), + f"Events for {name} were imputed with incorrect reward value", + ) From 62c05be98fc07819c137d6fcba62d1646a1f6630 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 15 Nov 2023 11:46:55 -0500 Subject: [PATCH 05/13] revert version update --- prompting/validators/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/validators/__init__.py b/prompting/validators/__init__.py index f75e905..38fe3a4 100644 --- a/prompting/validators/__init__.py +++ b/prompting/validators/__init__.py @@ -27,7 +27,7 @@ from . import event from . import dataset -__version__ = "2.1.4" +__version__ = "2.1.3" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0])) From 4eb41fada04d0e539bb988da7ead8c4ebef33a96 Mon Sep 17 00:00:00 2001 From: ifrit98 Date: Thu, 16 Nov 2023 22:00:15 +0000 Subject: [PATCH 06/13] ensure mock reward model has new blacklist functions or crashes --- prompting/validators/mock.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/prompting/validators/mock.py b/prompting/validators/mock.py index 461aaff..12d2c14 100644 --- a/prompting/validators/mock.py +++ b/prompting/validators/mock.py @@ -60,6 +60,12 @@ def __init__(self, mock_name: str = "MockReward"): self.question_blacklist = [] self.answer_blacklist = [] + def add(self, texts: List[str]): + pass + + def set_counter_to_half(self): + pass + def apply(self, prompt: str, completion: List[str], name: str) -> torch.FloatTensor: mock_reward = torch.tensor([1 for _ in completion], dtype=torch.float32) return mock_reward, {} From de0ed49d34fd2103e8ebd9a601a7a972704fc4bf Mon Sep 17 00:00:00 2001 From: isabella618033 Date: Fri, 17 Nov 2023 15:45:15 +0000 Subject: [PATCH 07/13] parse_reward_events fix --- prompting/validators/reward/reward.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/prompting/validators/reward/reward.py b/prompting/validators/reward/reward.py index 85f617e..096a1e6 100644 --- a/prompting/validators/reward/reward.py +++ b/prompting/validators/reward/reward.py @@ -30,6 +30,11 @@ class BaseRewardEvent: @staticmethod def parse_reward_events(reward_events): + if reward_events == None or len(reward_events) == 0 : + field_names = [field.name for field in fields(BaseRewardEvent())] + empty_reward_event = dict(zip(field_names, [[]] * len(field_names))) + return empty_reward_event + field_names = [field.name for field in fields(reward_events[0])] reward_events = [ asdict(reward_event).values() for reward_event in reward_events From 3ddfdd63da6db0756e8756825ee4ba4f8084eab6 Mon Sep 17 00:00:00 2001 From: isabella618033 Date: Fri, 17 Nov 2023 15:46:56 +0000 Subject: [PATCH 08/13] black formatting --- prompting/validators/reward/reward.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/validators/reward/reward.py b/prompting/validators/reward/reward.py index 096a1e6..abfecb4 100644 --- a/prompting/validators/reward/reward.py +++ b/prompting/validators/reward/reward.py @@ -30,7 +30,7 @@ class BaseRewardEvent: @staticmethod def parse_reward_events(reward_events): - if reward_events == None or len(reward_events) == 0 : + if reward_events == None or len(reward_events) == 0: field_names = [field.name for field in fields(BaseRewardEvent())] empty_reward_event = dict(zip(field_names, [[]] * len(field_names))) return empty_reward_event From 1a0378bfdce242fd8651ec110229e6c7cd4f652f Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Mon, 20 Nov 2023 16:22:56 +0000 Subject: [PATCH 09/13] adds unit tests for parse_reward_events --- tests/validators/reward/test_reward_event.py | 31 +++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/validators/reward/test_reward_event.py b/tests/validators/reward/test_reward_event.py index 917a447..c22edc6 100644 --- a/tests/validators/reward/test_reward_event.py +++ b/tests/validators/reward/test_reward_event.py @@ -72,7 +72,7 @@ def test_no_missing_rewards(self): def test_imputed_reward_values_are_correct(self): for name, events in self.reward_events.items(): - expected_value = 1 if events[0].is_filter_model else 0 + expected_value = 1 indices_missing_reward = [ i for i, ev in enumerate(events) if ev.reward is None ] @@ -87,3 +87,32 @@ def test_imputed_reward_values_are_correct(self): ), f"Events for {name} were imputed with incorrect reward value", ) + + def test_parse_reward_events_with_reward_events(self): + # Create sample reward events + event1 = reward.reward.BaseRewardEvent(reward=1, normalized_reward="event1") + event2 = reward.reward.BaseRewardEvent(reward=2, normalized_reward="event2") + events = [event1, event2] + + # Expected result + expected = { + 'reward': (1, 2), + 'normalized_reward': ('event1', 'event2') + } + + # Call the function and check if the result matches the expected output + result = reward.reward.BaseRewardEvent.parse_reward_events(events) + self.assertEqual(result, expected) + + + def test_parse_reward_events_with_no_reward_events(self): + # Test with None + result_none = reward.reward.BaseRewardEvent.parse_reward_events(None) + self.assertTrue(all(len(lst) == 0 for lst in result_none.values())) + self.assertEqual(result_none, {'reward': [], 'normalized_reward': []}) + + # Test with empty list + result_empty = reward.reward.BaseRewardEvent.parse_reward_events([]) + self.assertTrue(all(len(lst) == 0 for lst in result_empty.values())) + self.assertEqual(result_empty, {'reward': [], 'normalized_reward': []}) + From 23e64a175e5d2b61346f6acf52b9e346489a0146 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Mon, 20 Nov 2023 17:37:51 +0000 Subject: [PATCH 10/13] fix diversity bug --- prompting/validators/reward/diversity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/validators/reward/diversity.py b/prompting/validators/reward/diversity.py index 744254d..3f16977 100644 --- a/prompting/validators/reward/diversity.py +++ b/prompting/validators/reward/diversity.py @@ -164,7 +164,7 @@ def get_rewards( ) -> List[DiversityRewardEvent]: # Check if completions are empty, return 0 if so if len(completions) == 0: - return torch.tensor([]).to(self.device), None + return torch.tensor([]).to(self.device) # Get embeddings for all completions. embeddings = self.get_embeddings(completions) From aabdaddac9f08e5a2a49a176304cb9e560d93d63 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Mon, 20 Nov 2023 17:53:35 +0000 Subject: [PATCH 11/13] runs black over the code --- tests/validators/reward/test_reward_event.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/validators/reward/test_reward_event.py b/tests/validators/reward/test_reward_event.py index c22edc6..d61d41a 100644 --- a/tests/validators/reward/test_reward_event.py +++ b/tests/validators/reward/test_reward_event.py @@ -95,24 +95,19 @@ def test_parse_reward_events_with_reward_events(self): events = [event1, event2] # Expected result - expected = { - 'reward': (1, 2), - 'normalized_reward': ('event1', 'event2') - } + expected = {"reward": (1, 2), "normalized_reward": ("event1", "event2")} - # Call the function and check if the result matches the expected output - result = reward.reward.BaseRewardEvent.parse_reward_events(events) + # Call the function and check if the result matches the expected output + result = reward.reward.BaseRewardEvent.parse_reward_events(events) self.assertEqual(result, expected) - def test_parse_reward_events_with_no_reward_events(self): # Test with None result_none = reward.reward.BaseRewardEvent.parse_reward_events(None) self.assertTrue(all(len(lst) == 0 for lst in result_none.values())) - self.assertEqual(result_none, {'reward': [], 'normalized_reward': []}) + self.assertEqual(result_none, {"reward": [], "normalized_reward": []}) # Test with empty list result_empty = reward.reward.BaseRewardEvent.parse_reward_events([]) self.assertTrue(all(len(lst) == 0 for lst in result_empty.values())) - self.assertEqual(result_empty, {'reward': [], 'normalized_reward': []}) - + self.assertEqual(result_empty, {"reward": [], "normalized_reward": []}) From 26fbf4669f2b0d52ca977a23cb60c92c0bc8292d Mon Sep 17 00:00:00 2001 From: p-ferreira <38992619+p-ferreira@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:28:24 -0500 Subject: [PATCH 12/13] update release version --- prompting/validators/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/validators/__init__.py b/prompting/validators/__init__.py index 38fe3a4..f75e905 100644 --- a/prompting/validators/__init__.py +++ b/prompting/validators/__init__.py @@ -27,7 +27,7 @@ from . import event from . import dataset -__version__ = "2.1.3" +__version__ = "2.1.4" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0])) From fbf37c4d9c223bd2130cb529ef3b8ff781ad917f Mon Sep 17 00:00:00 2001 From: ifrit98 Date: Mon, 20 Nov 2023 18:35:16 +0000 Subject: [PATCH 13/13] updgrade datasets to fix streaming dataset error --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6e3213b..d42f5d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ bittensor==6.1.0 torch transformers==4.30.0 wandb==0.15.10 -datasets==2.14.0 +datasets==2.14.6 plotly==5.14.1 networkx==3.1 scipy==1.10.1