2.1.4 Release

.circleci/config.yml

@@ -78,7 +78,7 @@ jobs:
             . env/bin/activate
             pip install -e .
 
-  check-version-updated:
+  check-validator-version-updated:
     docker:
       - image: cimg/python:3.10
     steps:
@@ -87,18 +87,7 @@ jobs:
       - run:
           name: Version is updated
           command: |
-            [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/__init__.py | wc -l) == 1 ]] && echo "prompting/__init__.py has changed"
-            [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep VERSION | wc -l) == 1 ]] && echo "VERSION has changed"
-
-  check-changelog-updated:
-    docker:
-      - image: cimg/python:3.10
-    steps:
-      - checkout
-      - run:
-          name: File CHANGELOG.md is updated
-          command: |
-            [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep CHANGELOG.md | wc -l) == 1 ]] && echo "CHANGELOG.md has changed"
+            [[ $(git diff-tree --no-commit-id --name-only -r HEAD..main | grep prompting/validators/__init__.py | wc -l) == 1 ]] && echo "prompting/validators/__init__.py has changed"
 
   check-version-not-released:
     docker:
@@ -123,12 +112,7 @@ workflows:
 
   release-branches-requirements:
     jobs:
-      - check-version-updated:
-          filters:
-            branches:
-              only:
-                - /^(release|hotfix)/.*/
-      - check-changelog-updated:
+      - check-validator-version-updated:
           filters:
             branches:
               only:

prompting/validators/__init__.py

@@ -27,7 +27,7 @@
 from . import event
 from . import dataset
 
-__version__ = "2.1.3"
+__version__ = "2.1.4"
 version_split = __version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))

prompting/validators/forward.py

@@ -230,7 +230,7 @@ async def forward(self):
     base_text = ".".join(data.split(".", maxsplit=random_cutoff)[:-1])
 
     # Create a summary task from the context.
-    summary_task: Task = create_summarization_task(base_text)    
+    summary_task: Task = create_summarization_task(base_text)
 
     # Request a summary, given the original context.
     summarization_event = await run_step(

prompting/validators/mock.py

@@ -60,6 +60,12 @@ def __init__(self, mock_name: str = "MockReward"):
         self.question_blacklist = []
         self.answer_blacklist = []
 
+    def add(self, texts: List[str]):
+        pass
+
+    def set_counter_to_half(self):
+        pass
+
     def apply(self, prompt: str, completion: List[str], name: str) -> torch.FloatTensor:
         mock_reward = torch.tensor([1 for _ in completion], dtype=torch.float32)
         return mock_reward, {}

prompting/validators/reward/diversity.py

@@ -164,7 +164,7 @@ def get_rewards(
     ) -> List[DiversityRewardEvent]:
         # Check if completions are empty, return 0 if so
         if len(completions) == 0:
-            return torch.tensor([]).to(self.device), None
+            return torch.tensor([]).to(self.device)
 
         # Get embeddings for all completions.
         embeddings = self.get_embeddings(completions)

prompting/validators/reward/reward.py

@@ -30,6 +30,11 @@ class BaseRewardEvent:
 
     @staticmethod
     def parse_reward_events(reward_events):
+        if reward_events == None or len(reward_events) == 0:
+            field_names = [field.name for field in fields(BaseRewardEvent())]
+            empty_reward_event = dict(zip(field_names, [[]] * len(field_names)))
+            return empty_reward_event
+
         field_names = [field.name for field in fields(reward_events[0])]
         reward_events = [
             asdict(reward_event).values() for reward_event in reward_events
@@ -167,11 +172,13 @@ def apply(
         reward_events = {f"{self.name}_{k}": v for k, v in reward_events.items()}
         reward_events[self.name] = filled_rewards.tolist()
         reward_events[self.name + "_normalized"] = filled_rewards_normalized.tolist()
-        
+
         # Warns unexpected behavior for rewards
         if torch.isnan(filled_rewards_normalized).any():
-            bt.logging.warning(f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}")
-            filled_rewards_normalized.nan_to_num_(nan=0.0)            
+            bt.logging.warning(
+                f"The tensor from {self.name} contains NaN values: {filled_rewards_normalized}"
+            )
+            filled_rewards_normalized.nan_to_num_(nan=0.0)
 
         # Return the filled rewards.
         return filled_rewards_normalized, reward_events

requirements.txt

@@ -2,7 +2,7 @@ bittensor==6.1.0
 torch
 transformers==4.30.0
 wandb==0.15.10
-datasets==2.14.0
+datasets==2.14.6
 plotly==5.14.1
 networkx==3.1
 scipy==1.10.1

tests/validators/reward/test_reward_event.py

@@ -20,6 +20,7 @@
 from dataclasses import fields
 import prompting.validators.reward as reward
 
+
 class RewardEventTestCase(unittest.TestCase):
     """
     This class contains unit tests for the RewardEvent classes.
@@ -30,16 +31,15 @@ class RewardEventTestCase(unittest.TestCase):
 
     def setUp(self):
         self.event_classes = [
-            reward.reward.BaseRewardEvent, # Represents a reward model (float)
-            reward.nsfw.NSFWRewardEvent, # Remaining events are filters
+            reward.reward.BaseRewardEvent,  # Represents a reward model (float)
+            reward.nsfw.NSFWRewardEvent,  # Remaining events are filters
             reward.blacklist.BlacklistRewardEvent,
             reward.relevance.RelevanceRewardEvent,
-            reward.diversity.DiversityRewardEvent
+            reward.diversity.DiversityRewardEvent,
         ]
 
         self.reward_events = {}
         for event in self.event_classes:
-
             event_type = event.__name__
             self.reward_events[event_type] = []
 
@@ -53,31 +53,61 @@ def setUp(self):
 
                 for field in fields(ev):
                     # don't modify the is_filter_model field
-                    if field.name == 'is_filter_model':
+                    if field.name == "is_filter_model":
                         continue
                     # otherwise set the field to a float (including reward)
                     setattr(ev, field.name, 1.234)
 
                 self.reward_events[event_type].append(ev)
 
     def test_no_missing_rewards(self):
-
         for name, events in self.reward_events.items():
-
             parsed = reward.reward.BaseRewardEvent.parse_reward_events(events)
 
             # Ensure that all rewards are not None
-            self.assertTrue(all(r is not None for r in parsed['reward']), f'Events for {name} are missing rewards')
-
+            self.assertTrue(
+                all(r is not None for r in parsed["reward"]),
+                f"Events for {name} are missing rewards",
+            )
 
     def test_imputed_reward_values_are_correct(self):
-
         for name, events in self.reward_events.items():
-
-            expected_value = 1 if events[0].is_filter_model else 0
-            indices_missing_reward = [i for i, ev in enumerate(events) if ev.reward is None]
+            expected_value = 1
+            indices_missing_reward = [
+                i for i, ev in enumerate(events) if ev.reward is None
+            ]
 
             parsed = reward.reward.BaseRewardEvent.parse_reward_events(events)
 
             # Ensure that all rewards are not None
-            self.assertTrue(all(parsed['reward'][i]==expected_value for i in indices_missing_reward), f'Events for {name} were imputed with incorrect reward value')
+            self.assertTrue(
+                all(
+                    parsed["reward"][i] == expected_value
+                    for i in indices_missing_reward
+                ),
+                f"Events for {name} were imputed with incorrect reward value",
+            )
+
+    def test_parse_reward_events_with_reward_events(self):
+        # Create sample reward events
+        event1 = reward.reward.BaseRewardEvent(reward=1, normalized_reward="event1")
+        event2 = reward.reward.BaseRewardEvent(reward=2, normalized_reward="event2")
+        events = [event1, event2]
+
+        # Expected result
+        expected = {"reward": (1, 2), "normalized_reward": ("event1", "event2")}
+
+        # Call the function and check if the result matches the expected output
+        result = reward.reward.BaseRewardEvent.parse_reward_events(events)
+        self.assertEqual(result, expected)
+
+    def test_parse_reward_events_with_no_reward_events(self):
+        # Test with None
+        result_none = reward.reward.BaseRewardEvent.parse_reward_events(None)
+        self.assertTrue(all(len(lst) == 0 for lst in result_none.values()))
+        self.assertEqual(result_none, {"reward": [], "normalized_reward": []})
+
+        # Test with empty list
+        result_empty = reward.reward.BaseRewardEvent.parse_reward_events([])
+        self.assertTrue(all(len(lst) == 0 for lst in result_empty.values()))
+        self.assertEqual(result_empty, {"reward": [], "normalized_reward": []})