Yuxuan - Unhashable value checks and some restructure (#343)

* Add a check for unhashable values, refactor some functions to improve readability

* reverse preprocess_text_lowercase_but_retain_punctuation

* skip timestamp col when fill na

* Update error message

* adjust function order

* bug fix

* bug fix

* update liwc docs

* Added tests to confirm featurebuilder behavior on unhashable columns is correct

---------

Co-authored-by: Xinlan Emily Hu <[email protected]>

docs/build/html/feature_builder.html

@@ -57,6 +57,7 @@
 <li class="toctree-l3"><a class="reference internal" href="#feature_builder.FeatureBuilder.conv_level_features"><code class="docutils literal notranslate"><span class="pre">FeatureBuilder.conv_level_features()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#feature_builder.FeatureBuilder.featurize"><code class="docutils literal notranslate"><span class="pre">FeatureBuilder.featurize()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#feature_builder.FeatureBuilder.get_first_pct_of_chat"><code class="docutils literal notranslate"><span class="pre">FeatureBuilder.get_first_pct_of_chat()</span></code></a></li>
+<li class="toctree-l3"><a class="reference internal" href="#feature_builder.FeatureBuilder.load_custem_liwc_dict"><code class="docutils literal notranslate"><span class="pre">FeatureBuilder.load_custem_liwc_dict()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#feature_builder.FeatureBuilder.merge_conv_data_with_original"><code class="docutils literal notranslate"><span class="pre">FeatureBuilder.merge_conv_data_with_original()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#feature_builder.FeatureBuilder.preprocess_chat_data"><code class="docutils literal notranslate"><span class="pre">FeatureBuilder.preprocess_chat_data()</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#feature_builder.FeatureBuilder.save_features"><code class="docutils literal notranslate"><span class="pre">FeatureBuilder.save_features()</span></code></a></li>
@@ -242,6 +243,24 @@
 </dl>
 </dd></dl>
 
+<dl class="py method">
+<dt class="sig sig-object py" id="feature_builder.FeatureBuilder.load_custem_liwc_dict">
+<span class="sig-name descname"><span class="pre">load_custem_liwc_dict</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">custom_liwc_dictionary_path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">dict</span></span></span><a class="headerlink" href="#feature_builder.FeatureBuilder.load_custem_liwc_dict" title="Link to this definition"></a></dt>
+<dd><p>Load the custom LIWC dictionary from the provided path.</p>
+<p>This function reads the custom LIWC dictionary from the provided path and returns the dictionary.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>custom_liwc_dictionary_path</strong> (<em>str</em>) – Path to the custom LIWC dictionary file</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>Custom LIWC dictionary</p>
+</dd>
+<dt class="field-odd">Return type<span class="colon">:</span></dt>
+<dd class="field-odd"><p>dict</p>
+</dd>
+</dl>
+</dd></dl>
+
 <dl class="py method">
 <dt class="sig sig-object py" id="feature_builder.FeatureBuilder.merge_conv_data_with_original">
 <span class="sig-name descname"><span class="pre">merge_conv_data_with_original</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#feature_builder.FeatureBuilder.merge_conv_data_with_original" title="Link to this definition"></a></dt>

docs/build/html/genindex.html

@@ -638,10 +638,12 @@ <h2 id="L">L</h2>
 </li>
       <li><a href="features/lexical_features_v2.html#features.lexical_features_v2.liwc_features">liwc_features() (in module features.lexical_features_v2)</a>
 </li>
-      <li><a href="utils/check_embeddings.html#utils.check_embeddings.load_liwc_dict">load_liwc_dict() (in module utils.check_embeddings)</a>
+      <li><a href="feature_builder.html#feature_builder.FeatureBuilder.load_custem_liwc_dict">load_custem_liwc_dict() (feature_builder.FeatureBuilder method)</a>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="utils/check_embeddings.html#utils.check_embeddings.load_liwc_dict">load_liwc_dict() (in module utils.check_embeddings)</a>
+</li>
       <li><a href="features/politeness_v2_helper.html#features.politeness_v2_helper.load_saved_data">load_saved_data() (in module features.politeness_v2_helper)</a>
 </li>
       <li><a href="features/politeness_v2_helper.html#features.politeness_v2_helper.load_to_dict">load_to_dict() (in module features.politeness_v2_helper)</a>
@@ -801,11 +803,13 @@ <h2 id="R">R</h2>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="utils/check_embeddings.html#utils.check_embeddings.read_in_lexicons">read_in_lexicons() (in module utils.check_embeddings)</a>
 </li>
-  </ul></td>
-  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="utils/assign_chunk_nums.html#utils.assign_chunk_nums.reduce_chunks">reduce_chunks() (in module utils.assign_chunk_nums)</a>
 </li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="features/get_user_network.html#features.get_user_network.remove_active_user">remove_active_user() (in module features.get_user_network)</a>
+</li>
+      <li><a href="utils/preprocess.html#utils.preprocess.remove_unhashable_cols">remove_unhashable_cols() (in module utils.preprocess)</a>
 </li>
   </ul></td>
 </tr></table>
@@ -824,6 +828,8 @@ <h2 id="S">S</h2>
       <li><a href="features/politeness_v2_helper.html#features.politeness_v2_helper.sentenciser">sentenciser() (in module features.politeness_v2_helper)</a>
 </li>
       <li><a href="feature_builder.html#feature_builder.FeatureBuilder.set_self_conv_data">set_self_conv_data() (feature_builder.FeatureBuilder method)</a>
+</li>
+      <li><a href="utils/check_embeddings.html#utils.check_embeddings.sort_words">sort_words() (in module utils.check_embeddings)</a>
 </li>
       <li><a href="utils/check_embeddings.html#utils.check_embeddings.str_to_vec">str_to_vec() (in module utils.check_embeddings)</a>
 </li>

docs/build/html/utils/check_embeddings.html

@@ -71,6 +71,7 @@
 <li class="toctree-l4"><a class="reference internal" href="#utils.check_embeddings.is_valid_term"><code class="docutils literal notranslate"><span class="pre">is_valid_term()</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="#utils.check_embeddings.load_liwc_dict"><code class="docutils literal notranslate"><span class="pre">load_liwc_dict()</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="#utils.check_embeddings.read_in_lexicons"><code class="docutils literal notranslate"><span class="pre">read_in_lexicons()</span></code></a></li>
+<li class="toctree-l4"><a class="reference internal" href="#utils.check_embeddings.sort_words"><code class="docutils literal notranslate"><span class="pre">sort_words()</span></code></a></li>
 <li class="toctree-l4"><a class="reference internal" href="#utils.check_embeddings.str_to_vec"><code class="docutils literal notranslate"><span class="pre">str_to_vec()</span></code></a></li>
 </ul>
 </li>
@@ -275,24 +276,24 @@
 
 <dl class="py function">
 <dt class="sig sig-object py" id="utils.check_embeddings.is_valid_term">
-<span class="sig-prename descclassname"><span class="pre">utils.check_embeddings.</span></span><span class="sig-name descname"><span class="pre">is_valid_term</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dicTerm</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#utils.check_embeddings.is_valid_term" title="Link to this definition"></a></dt>
+<span class="sig-prename descclassname"><span class="pre">utils.check_embeddings.</span></span><span class="sig-name descname"><span class="pre">is_valid_term</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dicTerm</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#utils.check_embeddings.is_valid_term" title="Link to this definition"></a></dt>
 <dd><p>Check if a dictionary term is valid.</p>
-<p>This function returns <cite>True</cite> if the term matches the regex pattern and <cite>False</cite> otherwise.
-The regex pattern matches:</p>
+<p>This function returns True if the term matches the regex pattern and False otherwise.
+The pattern matches the following criteria:</p>
 <ul class="simple">
-<li><p>Alphanumeric characters (a-z, A-Z, 0-9)</p></li>
-<li><p>Valid symbols: <cite>-</cite>, <cite>‘</cite>, <cite>*</cite>, <cite>/</cite></p></li>
-<li><p>The <cite>*</cite> symbol can appear only once at the end of a word</p></li>
-<li><p>Emojis are valid only when they appear alone</p></li>
-<li><p>The <cite>/</cite> symbol can appear only once after alphanumeric characters</p></li>
+<li><p>Alphanumeric characters (a-zA-Z0-9)</p></li>
+<li><p>Valid symbols: -, ‘, *, /</p></li>
+<li><p>The * symbol can only appear once at the end of a word</p></li>
+<li><p>8 emojis are valid only when they appear alone</p></li>
+<li><p>The / symbol can only appear once after alphanumeric characters</p></li>
 <li><p>Spaces are allowed between valid words</p></li>
 </ul>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><p><strong>dicTerm</strong> (<em>str</em>) – The dictionary term to validate.</p>
+<dd class="field-odd"><p><strong>dicTerm</strong> (<em>str</em>) – The dictionary term</p>
 </dd>
 <dt class="field-even">Returns<span class="colon">:</span></dt>
-<dd class="field-even"><p><cite>True</cite> if the term is valid, <cite>False</cite> otherwise.</p>
+<dd class="field-even"><p>True if the term is valid, False otherwise</p>
 </dd>
 <dt class="field-odd">Return type<span class="colon">:</span></dt>
 <dd class="field-odd"><p>bool</p>
@@ -308,7 +309,18 @@
 <p>This functions reads the content of a LIWC dictionary file in the official format,
 and convert it to a dictionary with lexicon: regular expression format.
 We assume the dicText has two parts: the header, which maps numbers to “category names,”
-and the body, which maps words in the lexicon to different category numbers, separated by a ‘%’ sign.</p>
+and the body, which maps words in the lexicon to different category numbers, separated by ‘%’.
+Below is an example:
+‘’’
+%
+1   function
+2   pronoun
+3   ppron
+%
+again       1       2
+against     1       2       3
+‘’’
+Note that the elements in each line are separated by ‘      ‘.</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><p><strong>dicText</strong> (<em>str</em>) – The content of a .dic file</p>
@@ -327,6 +339,25 @@
 <span class="sig-prename descclassname"><span class="pre">utils.check_embeddings.</span></span><span class="sig-name descname"><span class="pre">read_in_lexicons</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">directory</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">lexicons_dict</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#utils.check_embeddings.read_in_lexicons" title="Link to this definition"></a></dt>
 <dd></dd></dl>
 
+<dl class="py function">
+<dt class="sig sig-object py" id="utils.check_embeddings.sort_words">
+<span class="sig-prename descclassname"><span class="pre">utils.check_embeddings.</span></span><span class="sig-name descname"><span class="pre">sort_words</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">lexicons</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#utils.check_embeddings.sort_words" title="Link to this definition"></a></dt>
+<dd><p>Sorts the dictionary terms in a list.</p>
+<p>This function sorts the dictionary terms in a list by their length in descending order.
+The hyphenated words are sorted first, followed by the non-hyphenated words.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>dicTerms</strong> (<em>list</em>) – List of dictionary terms</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>dicTerms</p>
+</dd>
+<dt class="field-odd">Return type<span class="colon">:</span></dt>
+<dd class="field-odd"><p>str</p>
+</dd>
+</dl>
+</dd></dl>
+
 <dl class="py function">
 <dt class="sig sig-object py" id="utils.check_embeddings.str_to_vec">
 <span class="sig-prename descclassname"><span class="pre">utils.check_embeddings.</span></span><span class="sig-name descname"><span class="pre">str_to_vec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str_vec</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#utils.check_embeddings.str_to_vec" title="Link to this definition"></a></dt>