Improve stripping HTML

Previous method was a bit scarce on whitespace, so words on
different lines / paragraph borders could be mashed together.

Author: oneiros

Gemfile

@@ -36,6 +36,12 @@ gem "mission_control-jobs"
 # Reduces boot times through caching; required in config/boot.rb
 gem "bootsnap", require: false
 
+# Decode HTML entities
+gem "htmlentities"
+
+# Scrub HTML (this is a dependency of rails already, putting it here for compeleteness sake
+gem "loofah"
+
 gem "fasp_base", github: "mastodon/fasp_ruby", glob: "fasp_base/*.gemspec"
 gem "fasp_data_sharing", github: "mastodon/fasp_ruby", glob: "fasp_data_sharing/*.gemspec"
 

Gemfile.lock

@@ -137,6 +137,7 @@ GEM
     globalid (1.2.1)
       activesupport (>= 6.1)
     hashdiff (1.1.2)
+    htmlentities (4.3.4)
     http-2 (1.0.2)
     httpx (1.4.1)
       http-2 (>= 1.0.0)
@@ -394,8 +395,10 @@ DEPENDENCIES
   faker
   fasp_base!
   fasp_data_sharing!
+  htmlentities
   importmap-rails
   jbuilder
+  loofah
   mission_control-jobs
   pg (~> 1.1)
   propshaft

app/models/actor.rb

@@ -19,6 +19,7 @@ class Actor < ApplicationRecord
     presence: { if: :discoverable? },
     absence: { unless: :discoverable? }
 
+  before_validation :set_full_text
   after_save :remove_unindexable_content
 
   scope :discoverable, -> { where(discoverable: true) }
@@ -49,7 +50,7 @@ def update_from_json(json_object)
 
   def set_full_text
     self.full_text = if discoverable?
-      [ name, username, stripped_summary ].compact.join(" ")
+      [ name, username, summary ].compact.join(" ")
     else
       nil
     end
@@ -58,8 +59,4 @@ def set_full_text
   def remove_unindexable_content
     content_objects.destroy_all unless indexable?
   end
-
-  def stripped_summary
-    Rails::Html::FullSanitizer.new.sanitize(summary) if summary.present?
-  end
 end

app/models/concerns/full_text_searchable_concern.rb

@@ -2,7 +2,7 @@ module FullTextSearchableConcern
   extend ActiveSupport::Concern
 
   included do
-    before_validation :set_full_text
+    normalizes :full_text, with: StrippedHtmlNormalizer.new
 
     scope :search, ->(term) {
       where("to_tsvector(#{table_name}.pg_text_search_configuration, #{table_name}.full_text) @@ to_tsquery(?)", term)

app/models/content_object.rb

@@ -1,4 +1,5 @@
 class ContentObject < ApplicationRecord
+  include FullTextSearchableConcern
   include LanguageTaggableConcern
   include RankableConcern
 
@@ -39,7 +40,7 @@ def json_to_attributes(json_object)
         last_edited_at: json_object["updated"] || json_object["published"],
         sensitive: json_object["sensitive"],
         language: json_object["contentMap"]&.keys&.first || "en",
-        full_text: Rails::HTML::FullSanitizer.new.sanitize(json_object["content"]),
+        full_text: json_object["content"],
         shares: json_object.dig("shares", "totalItems") || 0,
         likes: json_object.dig("likes", "totalItems") || 0,
         hashtags: hashtags.map { |name| Hashtag.find_or_initialize_by(name:) },

app/normalizers/stripped_html_normalizer.rb

@@ -0,0 +1,10 @@
+class StrippedHtmlNormalizer
+  def call(html)
+    return "" unless html
+
+    coder = HTMLEntities.new
+    html = coder.decode(html)
+
+    Loofah.fragment(html).to_text(encode_special_chars: false)
+  end
+end

test/normalizers/stripped_html_normalizer_test.rb

@@ -0,0 +1,25 @@
+require "test_helper"
+
+class StrippedHtmlNormalizerTest < ActiveSupport::TestCase
+  setup do
+    @normalizer = StrippedHtmlNormalizer.new
+  end
+
+  test "#call returns input without HTML tags" do
+    result = @normalizer.call("<span>test</span>")
+
+    assert_equal "test", result
+  end
+
+  test "#call returns whitespace in place of stripped tags" do
+    result = @normalizer.call("<p>Firstname</p><p>Lastname</p>")
+
+    assert_match /Firstname\s+Lastname/, result
+  end
+
+  test "#call decodes html entities" do
+    result = @normalizer.call("R&amp;D")
+
+    assert_equal "R&D", result
+  end
+end