-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathweb2intel.rb
executable file
·237 lines (196 loc) · 8.39 KB
/
web2intel.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
#!/usr/bin/env ruby
require './lib/w2i_includes'
require './lib/local_creds'
zray = []
result = []
commands = []
ARGV.each {|arg| commands << arg}
### HTML file sources
if ARGV[0] =~ Regex.cysitt
puts "#Title: CYBERsitter Malicious Web Site Filter - Top 100"
Processing.feedformat('html', Urls.cybersitter, '//b', '')
elsif ARGV[0] =~ Regex.qmal
puts "#Title: Quttera Malicious Domains Database"
Processing.feedformat('html', Urls.quttera_mal_url, "//tr[1]/td[2]", '')
elsif ARGV[0] =~ Regex.qsus
puts "#Title: Quttera Suspicious Domains Database"
Processing.feedformat('html', Urls.quttera_sus_url, "//tr[1]/td[2]", '')
elsif ARGV[0] =~ Regex.qpot
puts "#Title: Quttera Potentially Suspicious Domains Database"
Processing.feedformat('html', Urls.quttera_potsus_url, "//tr[1]/td[2]", '')
elsif ARGV[0] =~ Regex.sucuri_iframe
puts "#Title: Sucuri Research Labs Hidden iframes List"
Processing.feedformat('html', Urls.sucuri_url, "//table[1]//tr//td[3]//a", '')
elsif ARGV[0] =~ Regex.sucuri_redirect
puts "#Title: Sucuri Research Labs Conditional Redirect List"
Processing.feedformat('html', Urls.sucuri_url, "//table[2]//tr//td[3]//a", '')
elsif ARGV[0] =~ Regex.sucuri_js
puts "#Title: Sucuri Research Labs Encoded JavaScript List"
Processing.feedformat('html', Urls.sucuri_url, "//table[3]//tr//td[3]//a", '')
elsif ARGV[0] =~ Regex.webins
puts "#Title: Web Inspector Unsafe Site List"
Processing.feedformat('html', Urls.webins_url, "//table[1]//tr//td//span", '')
elsif ARGV[0] =~ Regex.malwr
puts "#Title: Malwr.com Recently Extracted Domains List"
Processing.feedformat('html', Urls.malwr, "//table[1]//tr//td//span[1]", '')
### Flat file sources
elsif ARGV[0] =~ Regex.et_ip
puts "#Title: Emerging Threats List of Compromised IPs"
Processing.feedformat('flat_dom', Urls.et_ip, '', '')
elsif ARGV[0] =~ Regex.vxvault
puts "#Title: VX Vault Last 100 Links"
Processing.feedformat('flat_dom', Urls.vxvault, '', '')
elsif ARGV[0] =~ Regex.malc0de
puts "#Title: Malc0de Blocklist"
Processing.feedformat('flat_dom', Urls.malc0de, '', '')
elsif ARGV[0] =~ Regex.mwdoms
puts "#Title: DNS-BH – Malware Domain Blocklist"
Processing.feedformat('flat_dom', Urls.mwdoms, '', '')
elsif ARGV[0] =~ Regex.isc_low
puts "#Title: SANS Internet Storm Center LOW Confidence Blocklist"
Processing.feedformat('flat_dom', Urls.isc_low_url, '', '')
elsif ARGV[0] =~ Regex.isc_med
puts "#Title: SANS Internet Storm Center MEDIUM Confidence Blocklist"
Processing.feedformat('flat_dom', Urls.isc_med_url, '', '')
elsif ARGV[0] =~ Regex.isc_high
puts "#Title: SANS Internet Storm Center HIGH Confidence Blocklist"
Processing.feedformat('flat_dom', Urls.isc_high_url, '', '')
elsif ARGV[0] =~ Regex.goz
puts "#Title: Gameover Zeus Domain List"
Processing.feedformat('flat_dom', Urls.goz, '', '')
elsif ARGV[0] =~ Regex.dyndns
puts "#Title: Dynamic DNS Provider List"
Processing.feedformat('flat_dom', Urls.dyndns, '', '')
elsif ARGV[0] =~ Regex.nothink
puts "#Title: alware DNS Network Traffic Blacklist From nothink.org"
Processing.feedformat('flat_dom', Urls.nothink, '', '')
elsif ARGV[0] =~ Regex.zeus_dom
puts "#Title: ZeuS Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.zeus_dom, '', '')
elsif ARGV[0] =~ Regex.zeus_ip
puts "#Title: ZeuS IP Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.zeus_ip, '', '')
elsif ARGV[0] =~ Regex.feodo_dom
puts "#Title: puts "#Feodo Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.feodo_dom, '', '')
elsif ARGV[0] =~ Regex.feodo_ip
puts "#Title: Feodo IP Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.feodo_ip, '', '')
elsif ARGV[0] =~ Regex.palevo_dom
puts "#Title: Palevo Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.palevo_dom, '', '')
elsif ARGV[0] =~ Regex.palevo_ip
puts "#Title: Palevo IP Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.palevo_ip, '', '')
elsif ARGV[0] =~ Regex.spyeye_dom
puts "#Title: Spyeye Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.spyeye_dom, '', '')
elsif ARGV[0] =~ Regex.spyeye_ip
puts "#Title: Spyeye IP Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.spyeye_ip, '', '')
## Flat list with line breaks
elsif ARGV[0] =~ Regex.cybercrime ####NOT WORKING####
puts "#Title: Cybercrime Tracker List"
Processing.feedformat('flat_br', Urls.cybercrime, '', '')
### Twitter sources
elsif ARGV[0] =~ Regex.tweet
puts "#Title: Twitter-based Intelligence From Twitter.com/#{ARGV[1]}"
Processing.feedformat('twitter', '', '', ARGV[1])
### RSS file sources
elsif ARGV[0] =~ Regex.mta
<<<<<<< HEAD
Processing.feedformat('rss', Urls.malware_ta, '//ul//li', '')
=======
Processing.feedformat('rss', Urls.malware_ta, '//ul//li', '')
>>>>>>> origin/master
=begin
####NOT WORKING YET####
elsif ARGV[0] =~ Regex.ptank
puts "#Title: OpenDNS PhishTank Domains"
puts Msg.time_head
buffer = open(Urls.phishtank)
buffer.each do |x|
parsed = JSON.parse(x)
if parsed["url"] != nil
ap parsed["url"]
end
end
#puts result.uniq.reject &:empty?
#result = JSON.parse(buffer)
#puts result.inspect
#trends = result['url']
#trends.each do |subject|
# puts subject['url']
#end
####NOT WORKING YET####
=end
<<<<<<< HEAD
=======
=begin
>>>>>>> origin/master
elsif ARGV[0] =~ Regex.all
puts "#Title: CYBERsitter Malicious Web Site Filter - Top 100"
Processing.feedformat('html', Urls.cybersitter, '//b', '')
puts "#Title: Quttera Malicious Domains Database"
Processing.feedformat('html', Urls.quttera_mal_url, "//tr[1]/td[2]", '')
puts "#Title: Quttera Suspicious Domains Database"
Processing.feedformat('html', Urls.quttera_sus_url, "//tr[1]/td[2]", '')
puts "#Title: Quttera Potentially Suspicious Domains Database"
Processing.feedformat('html', Urls.quttera_potsus_url, "//tr[1]/td[2]", '')
puts "#Title: Sucuri Research Labs Hidden iframes List"
Processing.feedformat('html', Urls.sucuri_url, "//table[1]//tr//td[3]//a", '')
puts "#Title: Sucuri Research Labs Conditional Redirect List"
Processing.feedformat('html', Urls.sucuri_url, "//table[2]//tr//td[3]//a", '')
puts "#Title: Sucuri Research Labs Encoded JavaScript List"
Processing.feedformat('html', Urls.sucuri_url, "//table[3]//tr//td[3]//a", '')
puts "#Title: Web Inspector Unsafe Site List"
Processing.feedformat('html', Urls.webins_url, "//table[1]//tr//td//span", '')
puts "#Title: Malwr.com Recently Extracted Domains List"
Processing.feedformat('html', Urls.malwr, "//table[1]//tr//td//span[1]", '')
puts "#Title: VX Vault Last 100 Links"
Processing.feedformat('flat_dom', Urls.vxvault, '', '')
puts "#Title: Malc0de Blocklist"
Processing.feedformat('flat_dom', Urls.malc0de, '', '')
puts "#Title: DNS-BH – Malware Domain Blocklist"
Processing.feedformat('flat_dom', Urls.mwdoms, '', '')
puts "#Title: SANS Internet Storm Center HIGH Confidence Blocklist"
Processing.feedformat('flat_dom', Urls.isc_high_url, '', '')
puts "#Title: Gameover Zeus Domain List"
Processing.feedformat('flat_dom', Urls.goz, '', '')
puts "#Title: Dynamic DNS Provider List"
Processing.feedformat('flat_dom', Urls.dyndns, '', '')
puts "#Title: Malware DNS Network Traffic Blacklist From nothink.org"
Processing.feedformat('flat_dom', Urls.nothink, '', '')
puts "#Title: ZeuS Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.zeus_dom, '', '')
puts "#Title: puts "#Feodo Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.feodo_dom, '', '')
puts "#Title: Feodo IP Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.feodo_ip, '', '')
puts "#Title: Palevo Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.palevo_dom, '', '')
puts "#Title: Spyeye Domain Blocklist From abuse.ch"
Processing.feedformat('flat_dom', Urls.spyeye_dom, '', '')
if ARGV[1] != nil
puts Msg.url_option
else open(Urls.malware_ta) do |rss|
feed = RSS::Parser.parse(rss)
puts "#Title: #{feed.channel.description}"
puts Msg.time_head
feed.items.each do |item|
page = Nokogiri::HTML(open(item.link))
page.xpath('//ul//li').each do |cell|
if PublicSuffix.valid?("#{cell.text.strip}") && "#{cell.text.strip}" !~ /\.zip/ && "#{cell.text.strip}" !~ /\.google\.com/
zray << Regex.reg.match("#{cell.text.strip}").to_s
end
end
end
end
puts zray.uniq.reject &:empty?
end
<<<<<<< HEAD
=======
=end
>>>>>>> origin/master
else puts Msg.invalid
end