feat(documentation): Add documentation support for three.js

assets/javascripts/news.json

@@ -1,4 +1,8 @@
 [
+  [
+    "2025-02-23",
+    "New documentation: <a href=\"/threejs/\">Three.js</a>"
+  ],
   [
     "2025-02-16",
     "New documentation: <a href=\"/openlayers/\">OpenLayers</a>"

docs/file-scrapers.md

@@ -300,3 +300,18 @@ it to `docs/sqlite`
 ```sh
 curl https://sqlite.org/2022/sqlite-doc-3400000.zip | bsdtar --extract --file - --directory=docs/sqlite/ --strip-components=1
 ```
+
+## Three.js
+Download the docs from https://github.com/mrdoob/three.js/tree/dev/files or run the following commands in your terminal:
+Make sure to set the version per the release tag (e.g. r160). Note that the r prefix is already included, only the version number is needed.
+
+```sh
+curl https://codeload.github.com/mrdoob/three.js/tar.gz/refs/tags/r${VERSION} > threejs.tar.gz
+tar -xzf threejs.tar.gz
+mkdir -p docs/threejs~${VERSION}
+mv three.js-r${VERSION}/list.json tmp/list.json
+mv three.js-r${VERSION}/docs/* docs/threejs~${VERSION}/
+
+rm -rf three.js-r${VERSION}/
+rm threejs.tar.gz
+```

lib/docs/filters/threejs/clean_html.rb

@@ -0,0 +1,229 @@
+module Docs
+  class Threejs
+    class CleanHtmlFilter < Filter
+      PATTERNS = {
+        method_this: /\[method:this\s+([^\]]+)\]\s*\((.*?)\)/,
+        method_return: /\[method:([^\s\]]+)\s+([^\]]+)\]\s*\((.*?)\)/,
+        method_no_params: /\[method:([^\s\]]+)\s+([^\]]+)\](?!\()/,
+        property: /\[property:([^\]]+?)\s+([^\]]+?)\]/,
+        example_link: /\[example:([^\s\]]+)\s+([^\]]+)\]/,
+        external_link_text: /\[link:([^\s\]]+)\s+([^\]]+)\]/,
+        external_link: /\[link:([^\]]+)\]/,
+        page_link_text: /\[page:([^\]]+?)\s+([^\]]+?)\]/,
+        page_link: /\[page:([^\]]+?)\]/,
+        inline_code: /`([^`]+)`/,
+        name_placeholder: /\[name\]/,
+        constructor_param: /\[param:([^\]]+?)\s+([^\]]+?)\]/
+      }.freeze
+
+      def call
+        remove_unnecessary_elements
+        wrap_code_blocks
+        process_sections
+        format_links
+        add_section_structure
+        format_notes
+        add_heading_attributes
+        doc
+      end
+
+      private
+
+      def remove_unnecessary_elements
+        css('head, script, style').remove
+      end
+
+      def wrap_code_blocks
+        css('code').each do |node|
+          next if node.parent.name == 'pre'
+          node.wrap('<pre>')
+          node.parent['data-language'] = 'javascript'
+        end
+      end
+
+      def process_sections
+        # Handle source links
+        css('h2').each do |node|
+          next unless node.content.strip == 'Source'
+          node.next_element.remove
+          node.remove
+        end
+
+        # Handle method signatures and properties
+        css('h3').each do |node|
+          content = node.inner_html
+          content = handle_method_signatures(content)
+          content = handle_properties(content)
+          node.inner_html = content
+        end
+
+        # Handle name placeholders and constructor params
+        css('h1, h3').each do |node|
+          content = node.inner_html
+          content = handle_name_placeholders(content)
+          content = format_constructor_params(content)
+          node.inner_html = content
+        end
+      end
+
+      def handle_method_signatures(content)
+        content
+          .gsub(PATTERNS[:method_this]) { format_method_signature('this', $1, $2) }
+          .gsub(PATTERNS[:method_return]) do |match|
+            next if $2.start_with?('this')
+            format_method_signature($1, $2, $3, true)
+          end
+          .gsub(PATTERNS[:method_no_params]) { format_method_signature($1, $2, nil, true) }
+      end
+
+      def format_method_signature(type_or_this, name, params_str, with_return = false)
+        params = if params_str
+          params_str.split(',').map { |param| format_parameter(param.strip) }.join("<span class='sig-paren'>, </span>")
+        end
+
+        html = "<dt class='sig sig-object js' id='#{name}'>"
+        if type_or_this == 'this'
+          html << "<span class='property'><span class='pre'>this</span></span>."
+        end
+        html << "<span class='sig-name descname'>#{name}</span>" \
+               "<span class='sig-paren'>(</span>" \
+               "#{params}" \
+               "<span class='sig-paren'>)</span>"
+        if with_return
+          html << "<span class='sig-returns'><span class='sig-colon'>:</span> " \
+                 "<span class='sig-type'>#{type_or_this}</span></span>"
+        end
+        html << "</dt>"
+      end
+
+      def format_parameter(param)
+        if param.include?(' ')
+          type, name = param.split(' ', 2).map(&:strip)
+          "<span class='sig-param'><span class='sig-type'>#{type}</span> <span class='sig-name'>#{name}</span></span>"
+        else
+          "<span class='sig-param'>#{param}</span>"
+        end
+      end
+
+      def handle_properties(content)
+        content.gsub(PATTERNS[:property]) do |match|
+          type, name = $1, $2
+          "<dt class='sig sig-object js'>" \
+          "<span class='sig-name descname'>#{name}</span>" \
+          "<span class='sig-colon'>:</span> " \
+          "<span class='sig-type'>#{type}</span></dt>"
+        end
+      end
+
+      def handle_name_placeholders(content)
+        content.gsub(PATTERNS[:name_placeholder]) do
+          name = slug.split('/').last.gsub('.html', '')
+          "<span class='descname'>#{name}</span>"
+        end
+      end
+
+      def format_constructor_params(content)
+        content.gsub(PATTERNS[:constructor_param]) do |match|
+          type, name = $1, $2
+          "<span class='sig-param'><span class='sig-type'>#{type}</span> <code class='sig-name'>#{name}</code></span>"
+        end
+      end
+
+      def format_links
+        css('*').each do |node|
+          next if node.text?
+
+          content = node.inner_html
+            .gsub(PATTERNS[:example_link]) { create_external_link("https://threejs.org/examples/##{$1}", $2) }
+            .gsub(PATTERNS[:external_link_text]) { create_external_link($1, $2) }
+            .gsub(PATTERNS[:external_link]) { create_external_link($1, $1) }
+            .gsub(PATTERNS[:page_link_text]) { create_internal_link($1, $2) }
+            .gsub(PATTERNS[:page_link]) { create_internal_link($1, $1) }
+
+          node.inner_html = content
+        end
+
+        normalize_href_attributes
+      end
+
+      def create_external_link(url, text)
+        %Q(<a class='reference external' href='#{url}'>#{text}</a>)
+      end
+
+      def create_internal_link(path, text)
+        %Q(<a class='reference internal' href='#{path.downcase}'><code class='xref js js-#{path.downcase}'>#{text}</code></a>)
+      end
+
+      def normalize_href_attributes
+        css('a[href]').each do |link|
+          next if link['href'].start_with?('http')
+          link['href'] = link['href'].remove('../').downcase.sub(/\.html$/, '')
+          link['class'] = 'reference internal'
+        end
+      end
+
+      def add_section_structure
+        css('h2').each do |node|
+          node['class'] = 'section-title'
+          section = node.next_element
+          next unless section
+
+          wrapper = doc.document.create_element('div')
+          wrapper['class'] = 'section'
+          node.after(wrapper)
+          wrapper.add_child(node)
+
+          current = section
+          while current && current.name != 'h2'
+            next_el = current.next
+            wrapper.add_child(current)
+            current = next_el
+          end
+        end
+
+        css('p.desc').each { |node| node['class'] = 'section-desc' }
+      end
+
+      def format_notes
+        css('p').each do |node|
+          next unless node.content.start_with?('Note:')
+
+          wrapper = doc.document.create_element('div')
+          wrapper['class'] = 'admonition note'
+
+          title = doc.document.create_element('p')
+          title['class'] = 'first admonition-title'
+          title.content = 'Note'
+
+          content = doc.document.create_element('p')
+          content['class'] = 'last'
+          content.inner_html = node.inner_html.sub('Note:', '').strip
+
+          wrapper.add_child(title)
+          wrapper.add_child(content)
+          node.replace(wrapper)
+        end
+      end
+
+      def add_heading_attributes
+        css('h1, h2, h3, h4').each do |node|
+          node['id'] ||= node.content.strip.downcase.gsub(/[^\w]+/, '-')
+          existing_class = node['class'].to_s
+          node['class'] = "#{existing_class} section-header"
+        end
+
+        format_inline_code
+      end
+
+      def format_inline_code
+        selectors = ['p', 'li', 'dt', 'dd', '.property-type'].join(', ')
+        css(selectors).each do |node|
+          next if node.at_css('pre')
+          node.inner_html = node.inner_html.gsub(PATTERNS[:inline_code]) do |match|
+            "<code class='docutils literal notranslate'><span class='pre'>#{$1}</span></code>"
+          end
+        end
+      end
+    end
+  end
+end 

lib/docs/filters/threejs/entries.rb

@@ -0,0 +1,54 @@
+module Docs
+  class Threejs
+    class EntriesFilter < Docs::EntriesFilter
+      def get_name
+        # Try to get name from the title first
+        if title = at_css('.lesson-title h1')&.content
+          title
+        else
+          # Fallback to path-based name for API docs
+          slug.split('/').last.gsub('.html', '').titleize
+        end
+      end
+
+      def get_type
+        if slug.start_with?('api/en/')
+          # For API documentation, use the section as type
+          # e.g. "api/en/animation/AnimationAction" -> "Animation"
+          path_parts = slug.split('/')
+          if path_parts.length >= 3
+            path_parts[2].titleize
+          else
+            'API'
+          end
+        elsif slug.start_with?('manual/en/')
+          # For manual pages, get the section from the path
+          # e.g. "manual/en/introduction/Creating-a-scene" -> "Introduction"
+          path_parts = slug.split('/')
+          if path_parts.length >= 3
+            path_parts[2].titleize
+          else
+            'Manual'
+          end
+        else
+          'Other'
+        end
+      end
+
+      def additional_entries
+        entries = []
+
+        # Get all methods and properties from h3 headings
+        css('h3').each do |node|
+          name = node.content.strip
+          # Skip if it's a constructor or doesn't have an ID
+          next if name == get_name || !node['id']
+
+          entries << [name, node['id'], get_type]
+        end
+
+        entries
+      end
+    end
+  end
+end 

lib/docs/scrapers/threejs.rb

@@ -0,0 +1,81 @@
+module Docs
+  class Threejs < UrlScraper
+    self.name = 'Three.js'
+    self.type = 'simple'
+    self.slug = 'threejs'
+    self.links = {
+      home: 'https://threejs.org/',
+      code: 'https://github.com/mrdoob/three.js'
+    }
+
+    html_filters.push 'threejs/entries', 'threejs/clean_html'
+
+    # The content is directly in the body
+    options[:container] = 'body'
+
+    options[:skip] = %w(
+      prettify.js
+      lesson.js
+      lang.css
+      lesson.css
+      editor.html
+      list.js
+      page.js
+    )
+
+    options[:only_patterns] = [
+      /\Aapi\/en\/.+\.html/,   # API documentation
+      /\Amanual\/en\/.+\.html/ # Manual pages
+    ]
+
+    options[:skip_patterns] = [
+      /examples/,
+      /\A_/,
+      /\Aresources\//,
+      /\Ascenes\//
+    ]
+
+    options[:attribution] = <<-HTML
+      &copy; 2010&ndash;#{Time.current.year} Three.js Authors<br>
+      Licensed under the MIT License.
+    HTML
+
+    self.release = '173'
+    self.base_url = "https://threejs.org/docs"
+
+    def get_latest_version(opts)
+      get_latest_github_release('mrdoob', 'three.js', opts)[1..]
+    end
+
+    def initial_paths
+      paths = []
+      url = 'https://threejs.org/docs/list.json'
+      response = Request.run(url)
+      json_data = JSON.parse(response.body)
+
+      # Process both API and manual sections
+      process_documentation(json_data['en'], paths)
+      paths
+    end
+
+    private
+
+    def process_documentation(data, paths, prefix = '')
+      data.each do |category, items|
+        if items.is_a?(Hash)
+          if items.values.first.is_a?(String)
+            # This is a leaf node with actual pages
+            items.each do |name, path|
+              paths << "#{path}.html"
+            end
+          else
+            # This is a category with subcategories
+            items.each do |subcategory, subitems|
+              process_documentation(items, paths, "#{prefix}#{category}/")
+            end
+          end
+        end
+      end
+    end
+  end
+end