diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52d4db54..8b21889e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,7 @@ jobs: strategy: matrix: ruby: - - "2.4" - - "2.5" - - "2.6" - - "2.7" + - "3.3.0" fail-fast: false steps: @@ -28,11 +25,6 @@ jobs: ruby-version: ${{ matrix.ruby }} bundler-cache: true - - uses: actions/setup-python@v2 - with: - # This should match lib/github/markups.rb GitHub::Markups::MARKUP_RST - python-version: '3.x' - - uses: actions/cache@v2 with: path: ~/.cache/pip @@ -49,9 +41,6 @@ jobs: curl -L http://cpanmin.us | perl - --sudo App::cpanminus sudo cpanm --installdeps --notest Pod::Simple - - name: Install Python dependencies - run: python -m pip install docutils - - name: Run rake run: | export PATH=$PATH:/.perl6/bin:/opt/rakudo-pkg/bin diff --git a/.gitignore b/.gitignore index 589d00dc..f1a14831 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ pkg/ Gemfile.lock .project .buildpath +vendor/ *~ diff --git a/Dockerfile b/Dockerfile index aa0f09aa..c23a2346 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,74 @@ -FROM ubuntu:trusty +FROM ubuntu:focal RUN apt-get update -qq -RUN apt-get install -y apt-transport-https +RUN apt-get install -y \ + apt-transport-https \ + locales \ + software-properties-common \ + curl \ + gnupg2 -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 379CE192D401AB61 -RUN echo "deb https://dl.bintray.com/nxadm/rakudo-pkg-debs `lsb_release -cs` main" | tee -a /etc/apt/sources.list.d/rakudo-pkg.list +# add the Raku repository +RUN curl -1sLf 'https://dl.cloudsmith.io/public/nxadm-pkgs/rakudo-pkg/gpg.0DD4CA7EB1C6CC6B.key' | gpg --dearmor >> /usr/share/keyrings/nxadm-pkgs-rakudo-pkg-archive-keyring.gpg +RUN curl -1sLf 'https://dl.cloudsmith.io/public/nxadm-pkgs/rakudo-pkg/config.deb.txt?distro=ubuntu&codename=focal&component=main' > /etc/apt/sources.list.d/nxadm-pkgs-rakudo-pkg.list +# add the Node.js repository +RUN curl -1sLf https://deb.nodesource.com/setup_20.x | bash RUN apt-get update -qq RUN apt-get install -y \ - perl rakudo-pkg curl git build-essential python python-pip \ - libssl-dev libreadline-dev zlib1g-dev \ - libicu-dev cmake pkg-config + perl \ + rakudo-pkg \ + git \ + libssl-dev \ + libreadline-dev \ + zlib1g-dev \ + libicu-dev \ + cmake \ + build-essential \ + g++ \ + pkg-config \ + nodejs \ + libffi-dev \ + libyaml-dev \ + gcc \ + libxslt-dev \ + libxml2-dev \ + zlib1g-dev \ + libidn11-dev \ + pandoc \ + vim-nox ENV PATH $PATH:/opt/rakudo-pkg/bin -RUN install-zef-as-user && zef install Pod::To::HTML +RUN install-zef +ENV PATH $PATH:/root/.raku/bin +RUN zef install Pod::To::HTML2 RUN curl -L http://cpanmin.us | perl - App::cpanminus RUN cpanm --installdeps --notest Pod::Simple -RUN pip install docutils +RUN apt-get purge ruby -y +# Install Rbenv and Ruby +RUN git clone https://github.com/rbenv/rbenv.git ~/.rbenv && echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc && echo 'eval "$(rbenv init -)"' >> ~/.bashrc +RUN git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build ENV PATH $PATH:/root/.rbenv/bin:/root/.rbenv/shims -RUN curl -fsSL https://github.com/rbenv/rbenv-installer/raw/master/bin/rbenv-installer | bash -RUN rbenv install 2.4.1 -RUN rbenv global 2.4.1 -RUN rbenv rehash +RUN cd /root/.rbenv/plugins/ruby-build && git pull && cd - +ENV RUBY_VERSION 3.3.0 +RUN rbenv install -l +RUN rbenv install $RUBY_VERSION && rbenv global $RUBY_VERSION && rbenv rehash +RUN rbenv install --list-all +RUN echo 'gem: --no-rdoc --no-ri' >> /.gemrc +RUN gem install rubygems-update && update_rubygems +RUN gem install bundler:2.4.22 + +RUN bundle config --global build.nokogiri --use-system-libraries -RUN gem install bundler +#RUN dpkg -i https://github.com/jgm/pandoc/releases/download/3.1.12.1/pandoc-3.1.12.1-1-amd64.deb WORKDIR /data/github-markup COPY github-markup.gemspec . COPY Gemfile . -COPY Gemfile.lock . -COPY lib/github-markup.rb lib/github-markup.rb +ADD lib ./lib RUN bundle ENV LC_ALL en_US.UTF-8 diff --git a/Gemfile b/Gemfile index c98072f2..6ae5df35 100644 --- a/Gemfile +++ b/Gemfile @@ -1,17 +1,19 @@ source "http://rubygems.org" gemspec +gem "nokogiri", ">= 1.16.2", force_ruby_platform: true gem "posix-spawn", :platforms => :ruby gem "redcarpet", :platforms => :ruby gem "kramdown", :platforms => :jruby gem "RedCloth" -# using a tag version here because 0.18.3 was not published by the author to encourage users to upgrade. -# however we want to bump up to this version since this has a security patch -gem "commonmarker", git: "https://github.com/gjtorikian/commonmarker.git", tag: "v0.18.3" -gem "rdoc", "~>3.6" -gem "org-ruby", "= 0.9.9" -gem "creole", "~>0.3.6" -gem "wikicloth", "=0.8.3" -gem "twitter-text", "~> 1.14" -gem "asciidoctor", "~> 2.0.5" -gem "rake" +gem "commonmarker", "~> 0.23.10" +gem 'rdoc', '~> 6.6', '>= 6.6.2' +gem 'org-ruby', '~> 0.9.12' +gem 'creole', '~> 0.5.0' +gem 'idn-ruby', '~> 0.1.5' +gem 'twitter-text', '~> 3.1' +gem "wikicloth", "= 0.8.3" +gem 'asciidoctor', '~> 2.0', '>= 2.0.21' +gem 'rake', '~> 13.1' +gem 'pandoc-ruby', '~> 2.1.10' + diff --git a/github-markup.gemspec b/github-markup.gemspec index 594d3507..e4da71d2 100644 --- a/github-markup.gemspec +++ b/github-markup.gemspec @@ -1,30 +1,38 @@ -require File.expand_path("../lib/github-markup", __FILE__) - -Gem::Specification.new do |s| - s.name = "github-markup" - s.version = GitHub::Markup::VERSION - s.summary = "The code GitHub uses to render README.markup" - s.description = <<~DESC - This gem is used by GitHub to render any fancy markup such as Markdown, - Textile, Org-Mode, etc. Fork it and add your own! - DESC - s.authors = ["Chris Wanstrath"] - s.email = "chris@ozmm.org" - s.homepage = "https://github.com/github/markup" - s.license = "MIT" - - s.files = `git ls-files`.split($\) - s.files += Dir['vendor/**/*'] - s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) } - s.test_files = s.files.grep(%r{^(test|spec|features)/}) - s.require_paths = %w[lib] - - s.add_development_dependency 'rake', '~> 12' - s.add_development_dependency 'activesupport', '~> 4.0' - s.add_development_dependency 'minitest', '~> 5.4', '>= 5.4.3' - s.add_development_dependency 'html-pipeline', '~> 1.0' - s.add_development_dependency 'sanitize', '>= 4.6.3' - s.add_development_dependency 'nokogiri', '~> 1.8.1' - s.add_development_dependency 'nokogiri-diff', '~> 0.2.0' - s.add_development_dependency "github-linguist", ">= 7.1.3" -end +# frozen_string_literal: true + +require_relative "lib/github/markup/version" + +Gem::Specification.new do |spec| + spec.name = "github-markup" + spec.version = GitHub::Markup::VERSION + spec.homepage = "https://github.com/github/markup" + spec.summary = "The code GitHub uses to render README.markup" + spec.description = <<~DESC + This gem is used by GitHub to render any fancy markup such as Markdown, + Textile, Org-Mode, etc. Fork it and add your own! +DESC + spec.authors = ["Chris Wanstrath", "Rob Crouch"] + spec.license = "MIT" + + spec.metadata = { + "bug_tracker_uri" => "https://github.com/github/markup/issues", + "source_code_uri" => "https://github.com/github/markup" + } + + spec.required_ruby_version = ">= 2.4" + + spec.files = Dir.glob("lib/**/*", File::FNM_DOTMATCH) + spec.executables = spec.files.grep(%r{^bin/}).map{ |f| File.basename(f) } + spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) + + spec.add_development_dependency 'rake', '~> 13.1' + spec.add_development_dependency 'activesupport', '~> 7.1', '>= 7.1.3.2' + spec.add_development_dependency 'minitest', '~> 5.4', '>= 5.4.3' + spec.add_development_dependency 'html-pipeline', '~> 1.0' + spec.add_development_dependency "sanitize", "~> 4.6", ">= 4.6.3" + spec.add_development_dependency 'nokogiri', '~> 1.8.1' + spec.add_development_dependency 'nokogiri-diff', '~> 0.2.0' + spec.add_development_dependency "github-linguist", "~> 7.1", ">= 7.1.3" + + +end \ No newline at end of file diff --git a/lib/github/commands/rest2html b/lib/github/commands/rest2html deleted file mode 100755 index c7c89dce..00000000 --- a/lib/github/commands/rest2html +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -""" -rest2html - A small wrapper file for parsing ReST files at GitHub. - -Written in 2008 by Jannis Leidel - -Brandon Keepers -Bryan Veloso -Chris Wanstrath -Dave Abrahams -Garen Torikian -Gasper Zejn -Michael Jones -Sam Whited -Tyler Chung -Vicent Marti - -To the extent possible under law, the author(s) have dedicated all copyright -and related and neighboring rights to this software to the public domain -worldwide. This software is distributed without any warranty. - -You should have received a copy of the CC0 Public Domain Dedication along with -this software. If not, see . -""" - -__author__ = "Jannis Leidel" -__license__ = "CC0" -__version__ = "0.1" - -import sys -import os - -# This fixes docutils failing with unicode parameters to CSV-Table. The -S -# switch and the following 3 lines can be removed after upgrading to python 3. -if sys.version_info[0] < 3: - reload(sys) - sys.setdefaultencoding('utf-8') - -import site - -try: - import locale - locale.setlocale(locale.LC_ALL, '') -except: - pass - -import codecs -import io - -from docutils import nodes, utils -from docutils.parsers.rst import directives, roles -from docutils.parsers.rst.directives.body import CodeBlock, Directive -from docutils.core import publish_parts -from docutils.writers.html4css1 import Writer, HTMLTranslator -from docutils.parsers.rst.states import Body -from docutils import nodes - -# By default, docutils provides two choices for unknown directives: -# - Show errors if the reporting level is high enough -# - Silently do not display them otherwise -# Comments are not displayed, either. - -# This code monkey-patches docutils to show preformatted lines -# in both these cases. - -# Recommended practice for repositories with rst files: -# - If github is the preferred viewer for the rst files (e.g. README.rst), -# then design the files to properly display for github. E.g., do not -# include unknown directives or restructuredText comments. -# - If github is NOT the preferred viewer for the rst files (e.g. -# the files are designed to be used with sphinx extensions at readthedocs) -# then include a restructuredText comment at the top of the file -# explaining that the document will display much more nicely with its -# preferred viewer, e.g. at readthedocs. - -original_behavior = False # Documents original docutils behavior -github_display = True - -def extract_extension_options(field_list, option_spec): - """ - Overrides `utils.extract_extension_options` and inlines - `utils.assemble_option_dict` to make it ignore unknown options passed to - directives (i.e. ``:caption:`` for ``.. code-block:``). - """ - - dropped = set() - options = {} - for name, value in utils.extract_options(field_list): - convertor = option_spec.get(name) - if name in options or name in dropped: - raise utils.DuplicateOptionError('duplicate option "%s"' % name) - - # silently drop unknown options as long as they are not duplicates - if convertor is None: - dropped.add(name) - continue - - # continue as before - try: - options[name] = convertor(value) - except (ValueError, TypeError) as detail: - raise detail.__class__('(option: "%s"; value: %r)\n%s' - % (name, value, ' '.join(detail.args))) - return options - -utils.extract_extension_options = extract_extension_options - -def unknown_directive(self, type_name): - lineno = self.state_machine.abs_line_number() - indented, indent, offset, blank_finish = \ - self.state_machine.get_first_known_indented(0, strip_indent=False) - text = '\n'.join(indented) - if original_behavior: - error = self.reporter.error( - 'Unknown directive type "%s".' % type_name, - nodes.literal_block(text, text), line=lineno) - return [error], blank_finish - elif github_display: - cls = ['unknown_directive'] - result = [nodes.literal_block(text, text, classes=cls)] - return result, blank_finish - else: - return [nodes.comment(text, text)], blank_finish - -def comment(self, match): - if not match.string[match.end():].strip() \ - and self.state_machine.is_next_line_blank(): # an empty comment? - return [nodes.comment()], 1 # "A tiny but practical wart." - indented, indent, offset, blank_finish = \ - self.state_machine.get_first_known_indented(match.end()) - while indented and not indented[-1].strip(): - indented.trim_end() - if not original_behavior: - firstline = ''.join(indented[:1]).split() - if ' '.join(firstline[:2]) == 'github display': - if len(firstline) == 3 and firstline[2] in ('on', 'off'): - global github_display - github_display = firstline[2] == 'on' - if len(indented) == 1: - return [nodes.comment()], 1 - text = '\n'.join(indented[1:]) - cls = ['github_comment'] - result = [nodes.literal_block(text, text, classes=cls)] - return result, blank_finish - text = '\n'.join(indented) - return [nodes.comment(text, text)], blank_finish - -Body.comment = comment -Body.unknown_directive = unknown_directive - - -SETTINGS = { - 'cloak_email_addresses': False, - 'file_insertion_enabled': False, - 'raw_enabled': True, - 'strip_comments': True, - 'doctitle_xform': True, - 'initial_header_level': 2, - 'report_level': 5, - 'syntax_highlight': 'none', - 'math_output': 'latex', - 'field_name_limit': 50, -} - -default_highlight_language = None - -class HighlightDirective(Directive): - required_arguments = 1 - optional_arguments = 1 - option_spec = {} - def run(self): - """Track the default syntax highlighting language - """ - global default_highlight_language - default_highlight_language = self.arguments[0] - return [] - -class DoctestDirective(CodeBlock): - """Render Sphinx 'doctest:: [group]' blocks as 'code:: python' - """ - - def run(self): - """Discard any doctest group argument, render contents as python code - """ - self.arguments = ['python'] - return super(DoctestDirective, self).run() - - -class GitHubHTMLTranslator(HTMLTranslator): - - # removes the
tag wrapped around docs - # see also: http://bit.ly/1exfq2h (warning! sourceforge link.) - def depart_document(self, node): - HTMLTranslator.depart_document(self, node) - self.html_body.pop(0) # pop the starting
off - self.html_body.pop() # pop the ending
off - - # technique for visiting sections, without generating additional divs - # see also: http://bit.ly/NHtyRx - # the a is to support ::contents with ::sectnums: http://git.io/N1yC - def visit_section(self, node): - id_attribute = node.attributes['ids'][0] - self.body.append('\n' % id_attribute) - self.section_level += 1 - - def depart_section(self, node): - self.section_level -= 1 - - def visit_literal_block(self, node): - classes = node.attributes['classes'] - if len(classes) >= 2 and classes[0] == 'code': - language = classes[1] - del classes[:] - self.body.append(self.starttag(node, 'pre', lang=language)) - elif default_highlight_language is not None: - self.body.append(self.starttag(node, 'pre', lang=default_highlight_language)) - else: - self.body.append(self.starttag(node, 'pre')) - - def visit_doctest_block(self, node): - self.body.append(self.starttag(node, 'pre', lang='pycon')) - - # always wrap two-backtick rst inline literals in , not - # this also avoids the generation of superfluous tags - def visit_literal(self, node): - self.body.append(self.starttag(node, 'code', suffix='')) - - def depart_literal(self, node): - self.body.append('') - - def visit_table(self, node): - classes = ' '.join(['docutils', self.settings.table_style]).strip() - self.body.append( - self.starttag(node, 'table', CLASS=classes)) - - def depart_table(self, node): - self.body.append('\n') - - def depart_image(self, node): - uri = node['uri'] - ext = os.path.splitext(uri)[1].lower() - # we need to swap RST's use of `object` with `img` tags - # see http://git.io/5me3dA - if ext == ".svg": - # preserve essential attributes - atts = {} - for attribute, value in node.attributes.items(): - # we have no time for empty values - if value: - if attribute == "uri": - atts['src'] = value - else: - atts[attribute] = value - - # toss off `object` tag - self.body.pop() - # add on `img` with attributes - self.body.append(self.starttag(node, 'img', **atts)) - HTMLTranslator.depart_image(self, node) - - -def kbd(name, rawtext, text, lineno, inliner, options={}, content=[]): - return [nodes.raw('', '%s' % text, format='html')], [] - - -def main(): - """ - Parses the given ReST file or the redirected string input and returns the - HTML body. - - Usage: rest2html < README.rst - rest2html README.rst - """ - try: - text = codecs.open(sys.argv[1], 'r', 'utf-8').read() - except IOError: # given filename could not be found - return '' - except IndexError: # no filename given - if sys.version_info[0] < 3: # python 2.x - text = sys.stdin.read() - else: # python 3 - input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') - text = input_stream.read() - - writer = Writer() - writer.translator_class = GitHubHTMLTranslator - - roles.register_canonical_role('kbd', kbd) - - # Render source code in Sphinx doctest blocks - directives.register_directive('doctest', DoctestDirective) - - # Set default highlight language - directives.register_directive('highlight', HighlightDirective) - - parts = publish_parts(text, writer=writer, settings_overrides=SETTINGS) - if 'html_body' in parts: - html = parts['html_body'] - - # publish_parts() in python 2.x return dict values as Unicode type - # in py3k Unicode is unavailable and values are of str type - if isinstance(html, str): - return html - else: - return html.encode('utf-8') - return '' - -if __name__ == '__main__': - if sys.version_info[0] < 3: # python 2.x - sys.stdout.write("%s%s" % (main(), "\n")) - else: # python 3 - output_stream = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') - output_stream.write("%s%s" % (main(), "\n")) - sys.stdout.flush() diff --git a/lib/github-markup.rb b/lib/github/markup/version.rb similarity index 73% rename from lib/github-markup.rb rename to lib/github/markup/version.rb index 58f9b1c5..7262d652 100644 --- a/lib/github-markup.rb +++ b/lib/github/markup/version.rb @@ -1,6 +1,6 @@ module GitHub module Markup - VERSION = '4.0.2' + VERSION = '4.0.3' Version = VERSION end end diff --git a/lib/github/markups.rb b/lib/github/markups.rb index 2c30c99d..3e43fd0c 100644 --- a/lib/github/markups.rb +++ b/lib/github/markups.rb @@ -1,6 +1,7 @@ require "github/markup/markdown" require "github/markup/rdoc" require "shellwords" +require 'pandoc-ruby' markup_impl(::GitHub::Markups::MARKUP_MARKDOWN, ::GitHub::Markup::Markdown.new) @@ -47,13 +48,9 @@ Asciidoctor.convert(content, :safe => :secure, :attributes => attributes) end -command( - ::GitHub::Markups::MARKUP_RST, - "python3 #{Shellwords.escape(File.dirname(__FILE__))}/commands/rest2html", - /re?st(\.txt)?/, - ["reStructuredText"], - "restructuredtext" -) +markup(::GitHub::Markups::MARKUP_RST, "pandoc-ruby", /re?st(\.txt)?/, ["reStructuredText"]) do |filename, content, options: {}| + PandocRuby.new(content, from: "rst").to_html +end command(::GitHub::Markups::MARKUP_POD6, :pod62html, /pod6/, ["Pod 6"], "pod6") command(::GitHub::Markups::MARKUP_POD, :pod2html, /pod/, ["Pod"], "pod") diff --git a/script/bootstrap b/script/bootstrap index f89f3181..6b87f215 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -5,4 +5,4 @@ set -e cd $(dirname "$0")/.. bundle install -pip3 install docutils + diff --git a/script/bootstrap.contrib b/script/bootstrap.contrib index 834b6b61..3c0a38bd 100755 --- a/script/bootstrap.contrib +++ b/script/bootstrap.contrib @@ -5,11 +5,8 @@ set -e cd $(dirname "$0")/.. bundle install --path vendor/bundle -virtualenv vendor/python && source vendor/python/bin/activate -pip install docutils echo "" echo "*** DONE ***" echo "" -echo "activate python environment with 'source vendor/python/bin/activate'" echo "run tests with 'bundle exec rake'" diff --git a/script/cibuild b/script/cibuild index 33251773..6d6fd2b5 100755 --- a/script/cibuild +++ b/script/cibuild @@ -10,7 +10,7 @@ export RUBY_HEAP_SLOTS_INCREMENT=400000 export RUBY_HEAP_SLOTS_GROWTH_FACTOR=1 export PATH="/usr/share/rbenv/shims:$PATH" -export RBENV_VERSION="1.9.3" +export RBENV_VERSION="3.3.0" # bootstrap gem environment changes echo "Bootstrapping gem environment ..." diff --git a/test/markup_test.rb b/test/markup_test.rb index 24e78000..bd296ff7 100644 --- a/test/markup_test.rb +++ b/test/markup_test.rb @@ -5,6 +5,7 @@ require 'github/markup' require 'minitest/autorun' require 'html/pipeline' +require 'sanitize' require 'nokogiri' require 'nokogiri/diff'