|
| 1 | + |
| 2 | +map_expressions = { |
| 3 | + "KAT1MoralisierendesSegment": "KAT1-Moralisierendes Segment", |
| 4 | + "Moralwerte": "KAT2-Moralwerte", |
| 5 | + "KAT2Subjektive_Ausdrcke": "KAT2-Subjektive Ausdrücke", |
| 6 | + "Protagonistinnen2": "KAT3-Gruppe", |
| 7 | + "Protagonistinnen": "KAT3-Rolle", |
| 8 | + "Protagonistinnen3": "KAT3-own/other", |
| 9 | + "KommunikativeFunktion": "KAT4-Kommunikative Funktion", |
| 10 | + "Forderung": "KAT5-Forderung explizit", |
| 11 | + "KAT5Ausformulierung": "KAT5-Forderung implizit", |
| 12 | + "Kommentar": "KOMMENTAR", |
| 13 | +} |
| 14 | + |
| 15 | +def validate_data_dict(data_dict): |
| 16 | + if not data_dict: |
| 17 | + raise ValueError("data_dict is empty") |
| 18 | + for data_file_name, data_file in data_dict.items(): |
| 19 | + validation_list = ["data", "file_type", "sofa", "paragraph"] |
| 20 | + missing_cats = [] |
| 21 | + for category in validation_list: |
| 22 | + if category not in list(data_file.keys()): |
| 23 | + missing_cats.append(category) |
| 24 | + |
| 25 | + if missing_cats: |
| 26 | + raise ValueError(f"Data dict is missing categories: {missing_cats}") |
| 27 | + |
| 28 | + |
| 29 | +class AnalyseOccurrence: |
| 30 | + """Contains statistical information methods about the data.""" |
| 31 | + |
| 32 | + def __init__( |
| 33 | + self, |
| 34 | + data_dict: dict, |
| 35 | + mode: str = "instances", |
| 36 | + file_names: str = None, |
| 37 | + ) -> None: |
| 38 | + |
| 39 | + validate_data_dict(data_dict) |
| 40 | + |
| 41 | + self.mode = mode |
| 42 | + self.data_dict = data_dict |
| 43 | + self.mode_dict = { |
| 44 | + "instances": self.report_instances, |
| 45 | + "spans": self.report_spans, |
| 46 | + "span_index": self.report_index, |
| 47 | + } |
| 48 | + self.file_names = self._initialize_files(file_names) |
| 49 | + self.instance_dict = self._initialize_dict() |
| 50 | + # call the analysis method |
| 51 | + self.mode_dict[self.mode]() |
| 52 | + # map the df columns to the expressions given |
| 53 | + self.map_categories() |
| 54 | + |
| 55 | + def _initialize_files(self, file_names: str) -> list: |
| 56 | + """Helper method to get file names in list.""" |
| 57 | + # get the file names from the global dict of dicts |
| 58 | + if file_names is None: |
| 59 | + file_names = list(self.data_dict.keys()) |
| 60 | + # or use the file names that were passed explicitly |
| 61 | + elif isinstance(file_names, str): |
| 62 | + file_names = [file_names] |
| 63 | + return file_names |
| 64 | + |
| 65 | + def _initialize_dict(self) -> defaultdict: |
| 66 | + """Helper method to initialize dict.""" |
| 67 | + return defaultdict(lambda: defaultdict(lambda: defaultdict(int))) |
| 68 | + |
| 69 | + def _initialize_df(self): |
| 70 | + """Helper method to initialize data frame.""" |
| 71 | + self.df = pd.DataFrame(self.instance_dict) |
| 72 | + self.df.index = self.df.index.set_names((["Main Category", "Sub Category"])) |
| 73 | + |
| 74 | + def _get_categories(self, span_dict, file_name): |
| 75 | + """Helper method to initialize a dict with the given main and sub categories.""" |
| 76 | + for main_cat_key, main_cat_value in span_dict.items(): |
| 77 | + for sub_cat_key, sub_cat_value in main_cat_value.items(): |
| 78 | + # the tuple index makes it easy to convert the dict into a pandas dataframe |
| 79 | + self.instance_dict[file_name][(main_cat_key, sub_cat_key)] = len( |
| 80 | + sub_cat_value |
| 81 | + ) |
| 82 | + return self.instance_dict |
| 83 | + |
| 84 | + def _add_total(self): |
| 85 | + """Helper method to set additional headers in data frame.""" |
| 86 | + self.df.loc[("total instances", "with invalid"), :] = self.df.sum(axis=0).values |
| 87 | + self.df.loc[("total instances", "without invalid"), :] = ( |
| 88 | + self.df.loc[("total instances", "with invalid"), :].values |
| 89 | + - self.df.loc["KAT1MoralisierendesSegment", "Keine Moralisierung"].values |
| 90 | + ) |
| 91 | + |
| 92 | + def _clean_df(self): |
| 93 | + """Helper method to sort data frame and clean up values.""" |
| 94 | + self.df = self.df.sort_values( |
| 95 | + by=[ |
| 96 | + "Main Category", |
| 97 | + "Sub Category", |
| 98 | + # self.file_names[0], |
| 99 | + ], |
| 100 | + ascending=True, |
| 101 | + ) |
| 102 | + # fill NaN with 0 for instances or None for spans |
| 103 | + if self.mode == "instances": |
| 104 | + self.df = self.df.fillna(0) |
| 105 | + if self.mode == "spans": |
| 106 | + self.df = self.df.replace({np.nan: None}) |
| 107 | + # remove quotes - not sure if this is necessary |
| 108 | + # self.df = self.df.applymap(lambda x: x.replace('"','') if isinstance(x, str) else x) |
| 109 | + |
| 110 | + def report_instances(self): |
| 111 | + """Reports number of occurrences of a category per text source.""" |
| 112 | + # instances reports the number of occurrences |
| 113 | + # filename: main_cat: sub_cat: instances |
| 114 | + for file_name in self.file_names: |
| 115 | + span_dict = self.data_dict[file_name]["data"] |
| 116 | + # initilize total instances rows for easier setting later. |
| 117 | + # only for mode instances |
| 118 | + self.instance_dict[file_name][("total instances", "with invalid")] = 0 |
| 119 | + self.instance_dict[file_name][("total instances", "without invalid")] = 0 |
| 120 | + self.instance_dict = self._get_categories(span_dict, file_name) |
| 121 | + # initialize data frame |
| 122 | + self._initialize_df() |
| 123 | + # add rows for total instances |
| 124 | + # only do this for mode instances |
| 125 | + self._add_total() |
| 126 | + |
| 127 | + def report_spans(self): |
| 128 | + """Reports spans of a category per text source.""" |
| 129 | + # span reports the spans of the annotations separated by separator-token |
| 130 | + self.instance_dict = self._get_categories( |
| 131 | + self.data_dict[self.file_names[0]]["data"], self.file_names[0] |
| 132 | + ) |
| 133 | + self._initialize_df() |
| 134 | + self.df[:] = self.df[:].astype("object") |
| 135 | + for file_name in self.file_names: |
| 136 | + span_dict = self.data_dict[file_name]["data"] |
| 137 | + span_text = self.data_dict[file_name]["sofa"] |
| 138 | + for main_cat_key, main_cat_value in span_dict.items(): |
| 139 | + for sub_cat_key in main_cat_value.keys(): |
| 140 | + # save the span begin and end character index for further analysis |
| 141 | + # span_dict[main_cat_key][sub_cat_key] = |
| 142 | + # find the text for each span |
| 143 | + span_annotated_text = [ |
| 144 | + span_text[span["begin"] : span["end"]] |
| 145 | + for span in span_dict[main_cat_key][sub_cat_key] |
| 146 | + ] |
| 147 | + # clean the spans from # |
| 148 | + span_annotated_text = [ |
| 149 | + span.replace("#", "") for span in span_annotated_text |
| 150 | + ] |
| 151 | + # clean the spans from " |
| 152 | + # span_annotated_text = [ |
| 153 | + # span.replace('"', "") for span in span_annotated_text |
| 154 | + # ] |
| 155 | + # convert list to &-separated spans |
| 156 | + span_annotated_text = " & ".join(span_annotated_text) |
| 157 | + self.df.at[ |
| 158 | + (main_cat_key, sub_cat_key), |
| 159 | + file_name, |
| 160 | + ] = span_annotated_text |
| 161 | + |
| 162 | + def report_index(self): |
| 163 | + self.report_instances() |
| 164 | + self.df[:] = self.df[:].astype("object") |
| 165 | + for file_name in self.file_names: |
| 166 | + span_dict = self.data_dict[file_name]["data"] |
| 167 | + for main_cat_key, main_cat_value in span_dict.items(): |
| 168 | + for sub_cat_key in main_cat_value.keys(): |
| 169 | + # report the beginning and end of each span as a tuple |
| 170 | + span_list = [ |
| 171 | + (span["begin"], span["end"]) |
| 172 | + for span in span_dict[main_cat_key][sub_cat_key] |
| 173 | + ] |
| 174 | + self.df.at[ |
| 175 | + (main_cat_key, sub_cat_key), |
| 176 | + file_name, |
| 177 | + ] = span_list |
| 178 | + |
| 179 | + def map_categories(self): |
| 180 | + self.df = self.df.rename(map_expressions) |
| 181 | + self._clean_df() |
| 182 | + |
| 183 | + |
| 184 | + |
0 commit comments