updates to message_original error

Watts-Lab · Dec 19, 2024 · 5b2b9f0 · 5b2b9f0
1 parent 57bfe88
commit 5b2b9f0
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 4 deletions.
diff --git a/examples/featurize.py b/examples/featurize.py
@@ -91,7 +91,7 @@
 		"Vectors Not of Same Length": vector_data_mismatch_df,
 		"Vectors Null": vect_null,
 		"Vectors Nan": vect_nan,
-  		"Custom File Equals Default Dir": valid_df,
+  		# "Custom File Equals Default Dir": valid_df,
 		"No 1-1 Mapping": vect_no_one_to_one,
 	}
 

diff --git a/src/team_comm_tools/feature_builder.py b/src/team_comm_tools/feature_builder.py
@@ -290,7 +290,8 @@ def __init__(
         self.user_methods = user_methods
         self.user_columns = user_columns
 
-        if(compute_vectors_from_preprocessed == True):
+        if(compute_vectors_from_preprocessed == True) or custom_vect_path is not None:
+            # if provided custom vector path then will preprocess
             self.vector_colname = self.message_col # because the message col will eventually get preprocessed
         else:
             self.vector_colname = self.message_col + "_original" # because this contains the original message

diff --git a/src/team_comm_tools/utils/check_embeddings.py b/src/team_comm_tools/utils/check_embeddings.py
@@ -72,8 +72,8 @@ def check_embeddings(chat_data: pd.DataFrame, vect_path: str, bert_path: str, or
             # check that message in vector data matches chat data
             preprocessed_chat = chat_data[message_col].astype(str).apply(preprocess_text)
 
-            # removed _original from message_col
-            preprocessed_vector = vector_df[message_col[:-9]].astype(str).apply(preprocess_text)
+            # preprocess vector data
+            preprocessed_vector = vector_df[message_col].astype(str).apply(preprocess_text)
 
             mismatches = chat_data[preprocessed_chat != preprocessed_vector]
             if len(mismatches) != 0: