fix: make dataset robust to empty samples

Signed-off-by: Mehant Kammakomati <[email protected]>
foundation-model-stack · Sep 5, 2024 · ef3191d · ef3191d
1 parent 6448afb
commit ef3191d
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/tuning/utils/data_loaders.py b/tuning/utils/data_loaders.py
@@ -109,6 +109,11 @@ def __iter__(self):
                             sample[self.tokens_field] = self.tokenizer.encode(
                                 sample[self.text_field]
                             )
+                            if not sample[self.tokens_field]:
+                                logger.warning(
+                                    f"skipping an empty sample : {sample[self.tokens_field]}"
+                                )
+                                continue
                         except Exception as e:  # pylint: disable=broad-exception-caught
                             logger.warning(
                                 "failed to tokenize the data {} of type {}.".format(