Skip to content

Commit 49961a8

Browse files
authored
resolves #187 add support for custom lexers (#225)
pygments.rb no longer stores list of lexers in a file. Instead, Pygments is queried for available lexers. In order to avoid spawning Pygments process when pygments.rb is just loaded, lexers are now stored in a lazily initialized cache.
1 parent d3a9fa6 commit 49961a8

File tree

7 files changed

+97
-108
lines changed

7 files changed

+97
-108
lines changed

.rubocop.yml

-2
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ Layout/LineLength:
1010
Max: 120
1111
Metrics/MethodLength:
1212
Enabled: false
13-
Security/MarshalLoad:
14-
Enabled: false
1513
Style/StructInheritance:
1614
Enabled: false
1715
Style/Documentation:

CHANGELOG.adoc

+4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
This document provides a high-level view of the changes to the {project-name} by release.
66
For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
77

8+
== Unreleased
9+
10+
* Add support for custom lexers ({uri-repo}/pull/187[#187])
11+
812
== 2.1.0 (2021-02-14) - @slonopotamus
913

1014
* Update Pygments to 2.8.0

Rakefile

-12
Original file line numberDiff line numberDiff line change
@@ -28,18 +28,6 @@ task :bench do
2828
sh 'ruby bench.rb'
2929
end
3030

31-
# ==========================================================
32-
# Cache lexers
33-
# ==========================================================
34-
35-
# Write all the lexers to a file for easy lookup
36-
task :lexers do
37-
sh 'ruby cache_lexers.rb'
38-
end
39-
40-
task(:test).enhance([:lexers])
41-
task(:build).enhance([:lexers])
42-
4331
# ==========================================================
4432
# Vendor
4533
# ==========================================================

cache_lexers.rb

-9
This file was deleted.

lib/pygments.rb

+7-4
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,25 @@
11
# frozen_string_literal: true
22

3-
require File.join(File.dirname(__FILE__), 'pygments/popen')
43
require 'forwardable'
54

6-
module Pygments
7-
autoload :Lexer, 'pygments/lexer'
5+
require_relative 'pygments/lexer'
6+
require_relative 'pygments/popen'
87

8+
module Pygments
99
class << self
1010
extend Forwardable
1111

12+
def lexers
13+
LexerCache.instance.raw_lexers
14+
end
15+
1216
def engine
1317
Thread.current.thread_variable_get(:pygments_engine) ||
1418
Thread.current.thread_variable_set(:pygments_engine, Pygments::Popen.new)
1519
end
1620

1721
def_delegators :engine,
1822
:formatters,
19-
:lexers,
2023
:lexers!,
2124
:filters,
2225
:styles,

lib/pygments/lexer.rb

+84-62
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,14 @@
11
# frozen_string_literal: true
22

3+
require 'singleton'
4+
35
module Pygments
46
class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
5-
@lexers = []
6-
@index = {}
7-
@name_index = {}
8-
@alias_index = {}
9-
@extname_index = {}
10-
@mimetypes_index = {}
11-
12-
# Internal: Create a new Lexer object
13-
#
14-
# hash - A hash of attributes
15-
#
16-
# Returns a Lexer object
17-
def self.create(hash)
18-
lexer = new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
19-
20-
@lexers << lexer
21-
22-
@index[lexer.name.downcase] = @name_index[lexer.name] = lexer
23-
24-
lexer.aliases.each do |name|
25-
@alias_index[name] = lexer
26-
@index[name.downcase] ||= lexer
27-
end
28-
29-
lexer.filenames.each do |filename|
30-
extnames = []
31-
32-
extname = File.extname(filename)
33-
if (m = extname.match(/\[(.+)\]/))
34-
m[1].scan(/./).each do |s|
35-
extnames << extname.sub(m[0], s)
36-
end
37-
elsif extname != ''
38-
extnames << extname
39-
end
40-
41-
extnames.each do |the_extname|
42-
@extname_index[the_extname] = lexer
43-
@index[the_extname.downcase.sub(/^\./, '')] ||= lexer
44-
end
45-
end
46-
47-
lexer.mimetypes.each do |type|
48-
@mimetypes_index[type] = lexer
49-
end
50-
51-
lexer
52-
end
53-
547
# Public: Get all Lexers
558
#
56-
# Returns an Array of Lexers
9+
# @return [Array<Lexer>]
5710
def self.all
58-
@lexers
11+
LexerCache.instance.lexers
5912
end
6013

6114
# Public: Look up Lexer by name or alias.
@@ -65,12 +18,15 @@ def self.all
6518
# Lexer.find('Ruby')
6619
# => #<Lexer name="Ruby">
6720
#
68-
# Returns the Lexer or nil if none was found.
21+
# @return [Lexer, nil]
6922
def self.find(name)
70-
@index[name.to_s.downcase]
23+
LexerCache.instance.index[name.to_s.downcase]
7124
end
7225

7326
# Public: Alias for find.
27+
#
28+
# @param name [String]
29+
# @return [Lexer, nil]
7430
def self.[](name)
7531
find(name)
7632
end
@@ -84,9 +40,10 @@ def self.[](name)
8440
# Lexer.find_by_name('Ruby')
8541
# # => #<Lexer name="Ruby">
8642
#
87-
# Returns the Lexer or nil if none was found.
43+
# @param name [String]
44+
# @return [Lexer, nil]
8845
def self.find_by_name(name)
89-
@name_index[name]
46+
LexerCache.instance.name_index[name]
9047
end
9148

9249
# Public: Look up Lexer by one of its aliases.
@@ -98,9 +55,10 @@ def self.find_by_name(name)
9855
# Lexer.find_by_alias('rb')
9956
# # => #<Lexer name="Ruby">
10057
#
101-
# Returns the Lexer or nil if none was found.
58+
# @param name [String]
59+
# @return [Lexer, nil]
10260
def self.find_by_alias(name)
103-
@alias_index[name]
61+
LexerCache.instance.alias_index[name]
10462
end
10563

10664
# Public: Look up Lexer by one of it's file extensions.
@@ -112,9 +70,10 @@ def self.find_by_alias(name)
11270
# Lexer.find_by_extname('.rb')
11371
# # => #<Lexer name="Ruby">
11472
#
115-
# Returns the Lexer or nil if none was found.
73+
# @param extname [String]
74+
# @return [Lexer, nil]
11675
def self.find_by_extname(extname)
117-
@extname_index[extname]
76+
LexerCache.instance.extname_index[extname]
11877
end
11978

12079
# Public: Look up Lexer by one of it's mime types.
@@ -126,9 +85,10 @@ def self.find_by_extname(extname)
12685
# Lexer.find_by_mimetype('application/x-ruby')
12786
# # => #<Lexer name="Ruby">
12887
#
129-
# Returns the Lexer or nil if none was found.
88+
# @param type [String]
89+
# @return [Lexer, nil]
13090
def self.find_by_mimetype(type)
131-
@mimetypes_index[type]
91+
LexerCache.instance.mimetypes_index[type]
13292
end
13393

13494
# Public: Highlight syntax of text
@@ -146,5 +106,67 @@ def highlight(text, options = {})
146106
alias eql? equal?
147107
end
148108

149-
lexers.values.each { |h| Lexer.create(h) }
109+
class LexerCache
110+
include Singleton
111+
112+
# @return [Array<Lexer>]
113+
attr_reader(:lexers)
114+
# @return [Map<String, Lexer>]
115+
attr_reader(:index)
116+
# @return [Map<String, Lexer>]
117+
attr_reader(:name_index)
118+
# @return [Map<String, Lexer]
119+
attr_reader(:alias_index)
120+
# @return [Map<String, Lexer>]
121+
attr_reader(:extname_index)
122+
# @return [Map<String, Lexer>]
123+
attr_reader(:mimetypes_index)
124+
125+
attr_reader(:raw_lexers)
126+
127+
def initialize
128+
@lexers = []
129+
@index = {}
130+
@name_index = {}
131+
@alias_index = {}
132+
@extname_index = {}
133+
@mimetypes_index = {}
134+
@raw_lexers = Pygments.lexers!
135+
136+
@raw_lexers.values.each do |hash|
137+
lexer = Lexer.new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
138+
139+
@lexers << lexer
140+
141+
@index[lexer.name.downcase] = @name_index[lexer.name] = lexer
142+
143+
lexer.aliases.each do |name|
144+
@alias_index[name] = lexer
145+
@index[name.downcase] ||= lexer
146+
end
147+
148+
lexer.filenames.each do |filename|
149+
extnames = []
150+
151+
extname = File.extname(filename)
152+
if (m = extname.match(/\[(.+)\]/))
153+
m[1].scan(/./).each do |s|
154+
extnames << extname.sub(m[0], s)
155+
end
156+
elsif extname != ''
157+
extnames << extname
158+
end
159+
160+
extnames.each do |the_extname|
161+
@extname_index[the_extname] = lexer
162+
@index[the_extname.downcase.sub(/^\./, '')] ||= lexer
163+
end
164+
end
165+
166+
lexer.mimetypes.each do |type|
167+
@mimetypes_index[type] = lexer
168+
end
169+
end
170+
end
171+
end
150172
end

lib/pygments/popen.rb

+2-19
Original file line numberDiff line numberDiff line change
@@ -103,25 +103,8 @@ def formatters
103103
end
104104
end
105105

106-
# Get all lexers from a serialized array.
107-
# This avoids needing to spawn mentos when it's not really needed
108-
# (e.g., one-off jobs, loading the Rails env, etc).
109-
#
110-
# Should be preferred to #lexers!
111-
#
112-
# @return [Array<String>] an array of lexers
113-
def lexers
114-
lexer_file = File.join(__dir__, '..', '..', 'lexers')
115-
begin
116-
File.open(lexer_file, 'rb') do |f|
117-
Marshal.load(f)
118-
end
119-
rescue Errno::ENOENT
120-
raise MentosError, %(Error loading #{lexer_file}. Was it created and vendored?)
121-
end
122-
end
123-
124-
# Get back all available lexers from mentos itself
106+
# Get all available lexers from mentos itself
107+
# Do not use this method directly, instead use Pygments#lexers
125108
#
126109
# @return [Array<String>] an array of lexers
127110
def lexers!

0 commit comments

Comments
 (0)