Skip to content

Commit

Permalink
updated user aggregation methods (max)
Browse files Browse the repository at this point in the history
  • Loading branch information
@amytangzheng
amytangzheng committed Sep 27, 2024
1 parent b2ed12a commit d35aeb1
Show file tree
Hide file tree
Showing 4 changed files with 108 additions and 32 deletions.
8 changes: 4 additions & 4 deletions examples/featurize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@
output_file_path_conv_level = "./jury_TINY_output_conversation_level_custom_agg.csv",
convo_methods = ['mean'], # This will aggregate ONLY the "positive_bert" at the conversation level, using mean; it will aggregate ONLY "negative_bert" at the speaker/user level, using max.
convo_columns = ['positive_bert'],
# user_methods = ['mean', 'max'],
# user_columns = ['positive_bert', 'negative_bert'],
user_methods = ['max'],
user_columns = ['negative_bert'],
user_methods = ['mean', 'max'],
user_columns = ['positive_bert', 'negative_bert'],
# user_methods = ['max'],
# user_columns = ['negative_bert'],
turns = False,
)
tiny_juries_feature_builder_custom_aggregation.featurize(col="message")
Expand Down
20 changes: 11 additions & 9 deletions src/team_comm_tools/utils/calculate_conversation_level_features.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ def __init__(self, chat_data: pd.DataFrame,
self.convo_aggregation = convo_aggregation
self.convo_methods = convo_methods
self.user_aggregation = user_aggregation
self.user_methods = user_methods
self.user_columns = user_columns
# Denotes the columns that can be summarized from the chat level, onto the conversation level.
self.input_columns = list(input_columns)
if 'conversation_num' not in self.input_columns:
Expand Down Expand Up @@ -177,7 +179,7 @@ def get_conversation_level_aggregates(self) -> None:
if 'mean' in self.convo_methods:
self.conv_data = pd.merge(
left=self.conv_data,
right=get_average(self.chat_data.copy(), column, 'average_'+column, self.conversation_id_col),
right=get_mean(self.chat_data.copy(), column, 'mean_'+column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand Down Expand Up @@ -247,7 +249,7 @@ def get_user_level_aggregates(self) -> None:
# Average/Mean of User-Level Feature
self.conv_data = pd.merge(
left=self.conv_data,
right=get_average(self.user_data.copy(), user_method + "_" +user_column, 'average_user_' + user_method + "_" +user_column, self.conversation_id_col),
right=get_mean(self.user_data.copy(), user_method + "_" +user_column, "mean_user_" + user_method + "_" +user_column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand All @@ -269,7 +271,7 @@ def get_user_level_aggregates(self) -> None:
# Minima of User-Level Feature
self.conv_data = pd.merge(
left=self.conv_data,
right=get_min(self.user_data.copy(), user_method + "_" + user_column, 'min_user_sum_' + user_method + "_" + user_column, self.conversation_id_col),
right=get_min(self.user_data.copy(), user_method + "_" + user_column, 'min_user_' + user_method + "_" + user_column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand All @@ -280,7 +282,7 @@ def get_user_level_aggregates(self) -> None:
# Maxima of User-Level Feature
self.conv_data = pd.merge(
left=self.conv_data,
right=get_max(self.user_data.copy(), user_method + "_" + user_column, 'max_user_sum_' + user_method + "_" + user_column, self.conversation_id_col),
right=get_max(self.user_data.copy(), user_method + "_" + user_column, 'max_user_' + user_method + "_" + user_column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand Down Expand Up @@ -335,14 +337,14 @@ def get_user_level_aggregates(self) -> None:
# how="inner"
# )

# Average Columns were created using self.get_user_level_averaged_features()
# Average Columns were created using self.get_user_level_mean_features()
for column in self.columns_to_summarize:

if 'mean' in self.convo_methods:
# Average/Mean of User-Level Feature
self.conv_data = pd.merge(
left=self.conv_data,
right=get_average(self.user_data.copy(), "average_"+column, 'average_user_avg_'+column, self.conversation_id_col),
right=get_mean(self.user_data.copy(), "mean_"+column, 'mean_user_avg_'+column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand All @@ -351,7 +353,7 @@ def get_user_level_aggregates(self) -> None:
# Standard Deviation of User-Level Feature
self.conv_data = pd.merge(
left=self.conv_data,
right=get_stdev(self.user_data.copy(), "average_"+column, 'stdev_user_avg_'+column, self.conversation_id_col),
right=get_stdev(self.user_data.copy(), "mean_"+column, 'stdev_user_avg_'+column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand All @@ -360,7 +362,7 @@ def get_user_level_aggregates(self) -> None:
# Minima of User-Level Feature
self.conv_data = pd.merge(
left=self.conv_data,
right=get_min(self.user_data.copy(), "average_"+column, 'min_user_avg_'+column, self.conversation_id_col),
right=get_min(self.user_data.copy(), "mean_"+column, 'min_user_avg_'+column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand All @@ -369,7 +371,7 @@ def get_user_level_aggregates(self) -> None:
# Maxima of User-Level Feature
self.conv_data = pd.merge(
left=self.conv_data,
right=get_max(self.user_data.copy(), "average_"+column, 'max_user_avg_'+column, self.conversation_id_col),
right=get_max(self.user_data.copy(), "mean_"+column, 'max_user_avg_'+column, self.conversation_id_col),
on=[self.conversation_id_col],
how="inner"
)
Expand Down
60 changes: 52 additions & 8 deletions src/team_comm_tools/utils/calculate_user_level_features.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Importing modules from features
from team_comm_tools.utils.summarize_features import get_user_sum_dataframe, get_user_average_dataframe
from team_comm_tools.utils.summarize_features import get_user_sum_dataframe, get_user_mean_dataframe, get_user_max_dataframe
from team_comm_tools.features.get_user_network import *
from team_comm_tools.features.user_centroids import *
import warnings
Expand Down Expand Up @@ -84,12 +84,15 @@ def calculate_user_level_features(self) -> pd.DataFrame:
:rtype: pd.DataFrame
"""

# Get average features for all features
self.get_user_level_averaged_features()
# Get mean features for all features
self.get_user_level_mean_features()

# Get total counts for all features
self.get_user_level_summed_features()

# Get user summary statistics for all features
self.get_user_level_summary_statistics_features()

# Get 4 discursive features (discursive diversity, variance in DD, incongruent modulation, within-person discursive range)
# self.get_centroids()

Expand All @@ -111,7 +114,47 @@ def get_user_level_summary_statistics_features(self) -> None:
This is an open question, so we are putting a TODO here.
"""
pass

if self.user_aggregation == True:
# For each summarizable feature
for column in self.columns_to_summarize:

# # Average/Mean of feature across the Conversation
# if 'mean' in self.user_methods:
# self.conv_data = pd.merge(
# left=self.conv_data,
# right=get_mean(self.chat_data.copy(), column, 'mean_'+column, self.conversation_id_col),
# on=[self.conversation_id_col],
# how="inner"
# )

# # Standard Deviation of feature across the Conversation
# if 'std' in self.convo_methods:
# self.conv_data = pd.merge(
# left=self.conv_data,
# right=get_stdev(self.chat_data.copy(), column, 'stdev_'+column, self.conversation_id_col),
# on=[self.conversation_id_col],
# how="inner"
# )

# # Minima for the feature across the Conversation
# if 'min' in self.convo_methods:
# self.conv_data = pd.merge(
# left=self.conv_data,
# right=get_min(self.chat_data.copy(), column, 'min_'+column, self.conversation_id_col),
# on=[self.conversation_id_col],
# how="inner"
# )

# Maxima for the feature across the Conversation
if 'max' in self.user_methods:
# print('HELLO')
self.user_data = pd.merge(
left=self.user_data,
right=get_user_max_dataframe(self.chat_data, column, self.conversation_id_col, self.speaker_id_col),
on=[self.conversation_id_col],
how="inner"
)

def get_user_level_summed_features(self) -> None:
"""
Expand Down Expand Up @@ -157,11 +200,11 @@ def get_user_level_summed_features(self) -> None:
how="inner"
)

def get_user_level_averaged_features(self) -> None:
def get_user_level_mean_features(self) -> None:
"""
Aggregate summary statistics by calculating average user-level features from chat-level features.
Aggregate summary statistics by calculating mean user-level features from chat-level features.
This function calculates and merges the average features into the user-level data.
This function calculates and merges the mean features into the user-level data.
:return: None
:rtype: None
Expand All @@ -175,10 +218,11 @@ def get_user_level_averaged_features(self) -> None:
# Average/Mean of feature across the Conversation
self.user_data = pd.merge(
left=self.user_data,
right=get_user_average_dataframe(self.chat_data, column, self.conversation_id_col, self.speaker_id_col),
right=get_user_mean_dataframe(self.chat_data, column, self.conversation_id_col, self.speaker_id_col),
on=[self.conversation_id_col, self.speaker_id_col],
how="inner"
)


def get_centroids(self) -> None:
"""
Expand Down
52 changes: 41 additions & 11 deletions src/team_comm_tools/utils/summarize_features.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,35 +31,65 @@ def get_user_sum_dataframe(chat_level_data, on_column, conversation_id_col, spea
return(grouped_conversation_data)


def get_user_average_dataframe(chat_level_data, on_column, conversation_id_col, speaker_id_col):
def get_user_mean_dataframe(chat_level_data, on_column, conversation_id_col, speaker_id_col):
"""Generate a user-level summary DataFrame by averaging a specified column per individual.
This function groups chat-level data by user and conversation, calculates the average values
This function groups chat-level data by user and conversation, calculates the mean values
of a specified numeric column for each user, and returns the resulting DataFrame.
:param chat_level_data: The DataFrame in which each row represents a single chat.
:type chat_level_data: pandas.DataFrame
:param on_column: The name of the numeric column to average for each user.
:param on_column: The name of the numeric column to mean for each user.
:type on_column: str
:param conversation_id_col: A string representing the column name that should be selected as the conversation ID.
:type conversation_id_col: str
:param speaker_id: The column name representing the user identifier.
:type speaker_id: str
:return: A grouped DataFrame with the average of the specified column per individual.
:return: A grouped DataFrame with the mean of the specified column per individual.
:rtype: pandas.DataFrame
"""
grouped_conversation_data = chat_level_data[[conversation_id_col, speaker_id_col, on_column]].groupby([conversation_id_col, speaker_id_col]).mean().reset_index()
grouped_conversation_data = grouped_conversation_data.rename(columns = {on_column: "average_"+on_column}) # gets this dataframe:
# Batch# Round# Speaker Average Number of Words
grouped_conversation_data = grouped_conversation_data.rename(columns = {on_column: "mean_"+on_column}) # gets this dataframe:
# Batch# Round# Speaker Mean Number of Words
# 0 1 Priya 100
# 0 1 Yuluan 90
return(grouped_conversation_data)

def get_average(input_data, column_to_summarize, new_column_name, conversation_id_col):
"""Generate a summary DataFrame with the average of a specified column per conversation.
def get_user_max_dataframe(chat_level_data, on_column, conversation_id_col, speaker_id_col):
"""Generate a user-level summary DataFrame by maxing a specified column per individual.
This function calculates the average of a specified column for each conversation in the input data,
and returns a DataFrame containing the conversation number and the calculated average.
This function groups chat-level data by user and conversation, calculates the max values
of a specified numeric column for each user, and returns the resulting DataFrame.
:param chat_level_data: The DataFrame in which each row represents a single chat.
:type chat_level_data: pandas.DataFrame
:param on_column: The name of the numeric column to max for each user.
:type on_column: str
:param conversation_id_col: A string representing the column name that should be selected as the conversation ID.
:type conversation_id_col: str
:param speaker_id: The column name representing the user identifier.
:type speaker_id: str
:return: A grouped DataFrame with the max of the specified column per individual.
:rtype: pandas.DataFrame
"""
grouped_conversation_data = chat_level_data[[conversation_id_col, speaker_id_col, on_column]].groupby([conversation_id_col, speaker_id_col]).max().reset_index()
grouped_conversation_data = grouped_conversation_data.rename(columns = {on_column: "max_"+on_column}) # gets this dataframe:
# Batch# Round# Speaker Max Number of Words
# 0 1 Priya 100
# 0 1 Yuluan 90
return(grouped_conversation_data)

def get_user_min_dataframe():
pass

def get_user_stdev_dataframe():
pass

def get_mean(input_data, column_to_summarize, new_column_name, conversation_id_col):
"""Generate a summary DataFrame with the mean of a specified column per conversation.
This function calculates the mean of a specified column for each conversation in the input data,
and returns a DataFrame containing the conversation number and the calculated mean.
:param input_data: The DataFrame containing data at the chat or user level.
:type input_data: pandas.DataFrame
Expand All @@ -69,7 +99,7 @@ def get_average(input_data, column_to_summarize, new_column_name, conversation_i
:type new_column_name: str
:param conversation_id_col: A string representing the column name that should be selected as the conversation ID.
:type conversation_id_col: str
:return: A DataFrame with the conversation number and the average of the specified column.
:return: A DataFrame with the conversation number and the mean of the specified column.
:rtype: pandas.DataFrame
"""
input_data[new_column_name] = input_data.groupby([conversation_id_col], sort=False)[column_to_summarize].transform(lambda x: np.mean(x))
Expand Down

0 comments on commit d35aeb1

Please sign in to comment.