Skip to content
This repository was archived by the owner on May 7, 2024. It is now read-only.

Commit d6ae4ea

Browse files
mahtinmahtin
committed
added Content-Type processing on add, removed old api web checking (hence remove bs4), refactoring of all add methods, network now handles data/json cleanly, results returned based on Content-Type via new code, setup/requirements removed BeautifulSoup/bs4
1 parent 91bd97d commit d6ae4ea

File tree

8 files changed

+382
-459
lines changed

8 files changed

+382
-459
lines changed

‎CloudFlare/api_decode_from_web.py

+102-100
Original file line numberDiff line numberDiff line change
@@ -1,103 +1,105 @@
11
""" API extras for Cloudflare API"""
22

3-
import sys
4-
import datetime
3+
# TODO - remove at some point
54

6-
API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
7-
8-
class mybs4():
9-
""" mybs4 """
10-
11-
_BeautifulSoup = None
12-
_Comment = None
13-
14-
def __init__(self):
15-
""" __init__ """
16-
pass
17-
18-
def _available(self):
19-
""" _available() """
20-
if not mybs4._BeautifulSoup:
21-
try:
22-
from bs4 import BeautifulSoup, Comment
23-
mybs4._BeautifulSoup = BeautifulSoup
24-
mybs4._Comment = Comment
25-
self.Comment = mybs4._Comment
26-
except ImportError:
27-
return False
28-
return True
29-
30-
def BeautifulSoup(self, content, parser):
31-
""" BeautifulSoup() """
32-
self._available()
33-
return mybs4._BeautifulSoup(content, parser)
34-
35-
my_bs4 = mybs4()
36-
37-
def do_section(section):
38-
""" API extras for Cloudflare API"""
39-
40-
cmds = []
41-
# look for deprecated first in section
42-
deprecated = False
43-
deprecated_date = ''
44-
deprecated_already = False
45-
for tag2 in section.find_all('h3'):
46-
# <h3 class="text-warning" data-reactid="13490">Deprecation Warning</h3>
47-
if 'Deprecation Warning' in str(tag2):
48-
deprecated = True
49-
break
50-
for tag2 in section.find_all('p'):
51-
# <p class="deprecation-date" data-reactid="13491">End of life Date: November 2, 2020</p>
52-
if 'End of life Date:' in str(tag2):
53-
for child in tag2.children:
54-
deprecated_date = str(child).replace('End of life Date:','').strip()
55-
try:
56-
# clean up date
57-
d = datetime.datetime.strptime(deprecated_date, '%B %d, %Y')
58-
if d <= datetime.datetime.now():
59-
# already done!
60-
deprecated_already = True
61-
deprecated_date = d.strftime('%Y-%m-%d')
62-
except ValueError:
63-
# Lets not worry about all the date formats that could show-up. Leave as a string
64-
pass
65-
break
66-
if deprecated_date != '':
67-
break
68-
# look for all API calls in section
69-
for tag2 in section.find_all('pre'):
70-
cmd = []
71-
for child in tag2.children:
72-
if isinstance(child, my_bs4.Comment):
73-
# remove <!-- react-text ... -> parts
74-
continue
75-
cmd.append(str(child).strip())
76-
if len(cmd) == 0:
77-
continue
78-
action = cmd[0]
79-
if action == '' or action not in API_TYPES:
80-
continue
81-
cmd = ''.join(cmd[1:])
82-
if cmd[0] != '/':
83-
cmd = '/' + cmd
84-
v = {'action': action, 'cmd': cmd, 'deprecated': deprecated, 'deprecated_date': deprecated_date, 'deprecated_already': deprecated_already}
85-
cmds.append(v)
86-
return cmds
87-
88-
def api_decode_from_web(content):
89-
""" API extras for Cloudflare API"""
90-
91-
soup = my_bs4.BeautifulSoup(content, 'html.parser')
92-
93-
for child in soup.find_all('p'):
94-
t = child.get_text()
95-
if 'Last modified' in t:
96-
sys.stderr.write("Retrieved API: %s\n" % (t.strip().replace('\n', ' ')))
97-
break
98-
99-
all_cmds = []
100-
for section in soup.find_all('section'):
101-
all_cmds += do_section(section)
102-
103-
return sorted(all_cmds, key=lambda v: v['cmd'])
5+
#import sys
6+
#import datetime
7+
#
8+
#API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
9+
#
10+
#class mybs4():
11+
# """ mybs4 """
12+
#
13+
# _BeautifulSoup = None
14+
# _Comment = None
15+
#
16+
# def __init__(self):
17+
# """ __init__ """
18+
# pass
19+
#
20+
# def _available(self):
21+
# """ _available() """
22+
# if not mybs4._BeautifulSoup:
23+
# try:
24+
# from bs4 import BeautifulSoup, Comment
25+
# mybs4._BeautifulSoup = BeautifulSoup
26+
# mybs4._Comment = Comment
27+
# self.Comment = mybs4._Comment
28+
# except ImportError:
29+
# return False
30+
# return True
31+
#
32+
# def BeautifulSoup(self, content, parser):
33+
# """ BeautifulSoup() """
34+
# self._available()
35+
# return mybs4._BeautifulSoup(content, parser)
36+
#
37+
#my_bs4 = mybs4()
38+
#
39+
#def do_section(section):
40+
# """ API extras for Cloudflare API"""
41+
#
42+
# cmds = []
43+
# # look for deprecated first in section
44+
# deprecated = False
45+
# deprecated_date = ''
46+
# deprecated_already = False
47+
# for tag2 in section.find_all('h3'):
48+
# # <h3 class="text-warning" data-reactid="13490">Deprecation Warning</h3>
49+
# if 'Deprecation Warning' in str(tag2):
50+
# deprecated = True
51+
# break
52+
# for tag2 in section.find_all('p'):
53+
# # <p class="deprecation-date" data-reactid="13491">End of life Date: November 2, 2020</p>
54+
# if 'End of life Date:' in str(tag2):
55+
# for child in tag2.children:
56+
# deprecated_date = str(child).replace('End of life Date:','').strip()
57+
# try:
58+
# # clean up date
59+
# d = datetime.datetime.strptime(deprecated_date, '%B %d, %Y')
60+
# if d <= datetime.datetime.now():
61+
# # already done!
62+
# deprecated_already = True
63+
# deprecated_date = d.strftime('%Y-%m-%d')
64+
# except ValueError:
65+
# # Lets not worry about all the date formats that could show-up. Leave as a string
66+
# pass
67+
# break
68+
# if deprecated_date != '':
69+
# break
70+
# # look for all API calls in section
71+
# for tag2 in section.find_all('pre'):
72+
# cmd = []
73+
# for child in tag2.children:
74+
# if isinstance(child, my_bs4.Comment):
75+
# # remove <!-- react-text ... -> parts
76+
# continue
77+
# cmd.append(str(child).strip())
78+
# if len(cmd) == 0:
79+
# continue
80+
# action = cmd[0]
81+
# if action == '' or action not in API_TYPES:
82+
# continue
83+
# cmd = ''.join(cmd[1:])
84+
# if cmd[0] != '/':
85+
# cmd = '/' + cmd
86+
# v = {'action': action, 'cmd': cmd, 'deprecated': deprecated, 'deprecated_date': deprecated_date, 'deprecated_already': deprecated_already}
87+
# cmds.append(v)
88+
# return cmds
89+
#
90+
#def api_decode_from_web(content):
91+
# """ API extras for Cloudflare API"""
92+
#
93+
# soup = my_bs4.BeautifulSoup(content, 'html.parser')
94+
#
95+
# for child in soup.find_all('p'):
96+
# t = child.get_text()
97+
# if 'Last modified' in t:
98+
# sys.stderr.write("Retrieved API: %s\n" % (t.strip().replace('\n', ' ')))
99+
# break
100+
#
101+
# all_cmds = []
102+
# for section in soup.find_all('section'):
103+
# all_cmds += do_section(section)
104+
#
105+
# return sorted(all_cmds, key=lambda v: v['cmd'])

‎CloudFlare/api_v4.py

+28-19
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ def zones(self):
110110
self.add('AUTH', 'zones', 'custom_pages')
111111
self.add('AUTH', 'zones', 'dns_records')
112112
self.add('AUTH', 'zones', 'dns_records/export')
113-
self.add('AUTH', 'zones', 'dns_records/import')
113+
self.add('AUTH', 'zones', 'dns_records/import', content_type={'POST':'multipart/form-data'})
114114
self.add('AUTH', 'zones', 'dns_records/scan')
115115
self.add('VOID', 'zones', 'dns_settings')
116116
self.add('AUTH', 'zones', 'dns_settings/use_apex_ns')
@@ -185,6 +185,15 @@ def zones_settings(self):
185185
self.add('AUTH', 'zones', 'settings/waf')
186186
self.add('AUTH', 'zones', 'settings/webp')
187187
self.add('AUTH', 'zones', 'settings/websockets')
188+
self.add('VOID', 'zones', 'settings/zaraz')
189+
self.add('VOID', 'zones', 'settings/zaraz/v2')
190+
self.add('AUTH', 'zones', 'settings/zaraz/v2/config')
191+
self.add('AUTH', 'zones', 'settings/zaraz/v2/default')
192+
self.add('AUTH', 'zones', 'settings/zaraz/v2/export')
193+
self.add('AUTH', 'zones', 'settings/zaraz/v2/history')
194+
self.add('AUTH', 'zones', 'settings/zaraz/v2/history/configs')
195+
self.add('AUTH', 'zones', 'settings/zaraz/v2/publish')
196+
self.add('AUTH', 'zones', 'settings/zaraz/v2/workflow')
188197

189198
def zones_analytics(self):
190199
""" zones analytics """
@@ -399,7 +408,7 @@ def accounts(self):
399408

400409
self.add('VOID', 'accounts', 'dlp')
401410
self.add('AUTH', 'accounts', 'dlp/datasets')
402-
self.add('AUTH', 'accounts', 'dlp/datasets', 'upload')
411+
self.add('AUTH', 'accounts', 'dlp/datasets', 'upload', content_type={'POST':'application/octet-stream'})
403412
self.add('VOID', 'accounts', 'dlp/patterns')
404413
self.add('AUTH', 'accounts', 'dlp/patterns/validate')
405414
self.add('AUTH', 'accounts', 'dlp/payload_log')
@@ -448,7 +457,7 @@ def accounts(self):
448457
self.add('AUTH', 'accounts', 'storage/kv/namespaces')
449458
self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'bulk')
450459
self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'keys')
451-
self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'values')
460+
self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'values', content_type={'PUT':'multipart/form-data'})
452461
self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'metadata')
453462

454463
self.add('AUTH', 'accounts', 'subscriptions')
@@ -458,9 +467,9 @@ def accounts(self):
458467
self.add('VOID', 'accounts', 'vectorize')
459468
self.add('AUTH', 'accounts', 'vectorize/index')
460469
self.add('AUTH', 'accounts', 'vectorize/indexes')
461-
self.add('AUTH', 'accounts', 'vectorize/indexes', 'insert')
470+
self.add('AUTH', 'accounts', 'vectorize/indexes', 'insert', content_type={'POST':'application/x-ndjson'})
462471
self.add('AUTH', 'accounts', 'vectorize/indexes', 'query')
463-
self.add('AUTH', 'accounts', 'vectorize/indexes', 'upsert')
472+
self.add('AUTH', 'accounts', 'vectorize/indexes', 'upsert', content_type={'POST':'application/x-ndjson'})
464473

465474
self.add('AUTH', 'accounts', 'virtual_dns')
466475
self.add('VOID', 'accounts', 'virtual_dns', 'dns_analytics')
@@ -475,7 +484,7 @@ def accounts(self):
475484
self.add('VOID', 'accounts', 'workers/dispatch')
476485
self.add('AUTH', 'accounts', 'workers/dispatch/namespaces')
477486
self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts')
478-
self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'content')
487+
self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'content', content_type={'PUT':'multipart/form-data'})
479488
self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'settings')
480489
self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'tags')
481490
self.add('AUTH', 'accounts', 'workers/domains')
@@ -485,15 +494,15 @@ def accounts(self):
485494
self.add('AUTH', 'accounts', 'workers/queues')
486495
self.add('AUTH', 'accounts', 'workers/queues', 'consumers')
487496
self.add('AUTH', 'accounts', 'workers/scripts')
488-
self.add('AUTH', 'accounts', 'workers/scripts', 'content')
497+
self.add('AUTH', 'accounts', 'workers/scripts', 'content', content_type={'PUT':'multipart/form-data'})
489498
self.add('AUTH', 'accounts', 'workers/scripts', 'content/v2')
490499
self.add('AUTH', 'accounts', 'workers/scripts', 'schedules')
491-
self.add('AUTH', 'accounts', 'workers/scripts', 'settings')
500+
self.add('AUTH', 'accounts', 'workers/scripts', 'settings', content_type={'PATCH':'multipart/form-data'})
492501
self.add('AUTH', 'accounts', 'workers/scripts', 'tails')
493502
self.add('AUTH', 'accounts', 'workers/scripts', 'usage-model')
494503
self.add('VOID', 'accounts', 'workers/services')
495504
self.add('VOID', 'accounts', 'workers/services', 'environments')
496-
self.add('AUTH', 'accounts', 'workers/services', 'environments', 'content')
505+
self.add('AUTH', 'accounts', 'workers/services', 'environments', 'content', content_type={'PUT':'multipart/form-data'})
497506
self.add('AUTH', 'accounts', 'workers/services', 'environments', 'settings')
498507
self.add('AUTH', 'accounts', 'workers/subdomain')
499508

@@ -505,7 +514,7 @@ def accounts_addressing(self):
505514
self.add('AUTH', 'accounts', 'addressing/address_maps', 'accounts')
506515
self.add('AUTH', 'accounts', 'addressing/address_maps', 'ips')
507516
self.add('AUTH', 'accounts', 'addressing/address_maps', 'zones')
508-
self.add('AUTH', 'accounts', 'addressing/loa_documents')
517+
self.add('AUTH', 'accounts', 'addressing/loa_documents', content_type={'POST':'multipart/form-data'})
509518
self.add('AUTH', 'accounts', 'addressing/loa_documents', 'download')
510519
self.add('AUTH', 'accounts', 'addressing/prefixes')
511520
self.add('VOID', 'accounts', 'addressing/prefixes', 'bgp')
@@ -559,7 +568,7 @@ def accounts_stream(self):
559568
self.add('AUTH', 'accounts', 'stream')
560569
self.add('AUTH', 'accounts', 'stream', 'audio')
561570
self.add('AUTH', 'accounts', 'stream', 'audio/copy')
562-
self.add('AUTH', 'accounts', 'stream', 'captions')
571+
self.add('AUTH', 'accounts', 'stream', 'captions', content_type={'PUT':'multipart/form-data'})
563572
self.add('AUTH', 'accounts', 'stream', 'embed')
564573
self.add('AUTH', 'accounts', 'stream', 'downloads')
565574
self.add('AUTH', 'accounts', 'stream', 'token')
@@ -568,7 +577,7 @@ def accounts_stream(self):
568577
self.add('AUTH', 'accounts', 'stream/direct_upload')
569578
self.add('AUTH', 'accounts', 'stream/keys')
570579
# self.add('AUTH', 'accounts', 'stream/preview')
571-
self.add('AUTH', 'accounts', 'stream/watermarks')
580+
self.add('AUTH', 'accounts', 'stream/watermarks', content_type={'POST':'multipart/form-data'})
572581
self.add('AUTH', 'accounts', 'stream/webhook')
573582
self.add('AUTH', 'accounts', 'stream/live_inputs')
574583
self.add('AUTH', 'accounts', 'stream/live_inputs', 'outputs')
@@ -666,7 +675,7 @@ def accounts_extras(self):
666675
""" extras """
667676

668677
self.add('VOID', 'accounts', 'ai')
669-
self.add('AUTH', 'accounts', 'ai/run')
678+
self.add('AUTH', 'accounts', 'ai/run', content_type={'POST':['application/json','application/octet-stream']})
670679
self.add('AUTH', 'accounts', 'ai/run/proxy')
671680

672681
self.add('VOID', 'accounts', 'alerting')
@@ -737,14 +746,14 @@ def accounts_extras(self):
737746
self.add('AUTH', 'accounts', 'gateway/rules')
738747

739748
self.add('VOID', 'accounts', 'images')
740-
self.add('AUTH', 'accounts', 'images/v1')
749+
self.add('AUTH', 'accounts', 'images/v1', content_type={'POST':'multipart/form-data'})
741750
self.add('AUTH', 'accounts', 'images/v1', 'blob')
742751
self.add('AUTH', 'accounts', 'images/v1/config')
743752
self.add('AUTH', 'accounts', 'images/v1/keys')
744753
self.add('AUTH', 'accounts', 'images/v1/stats')
745754
self.add('AUTH', 'accounts', 'images/v1/variants')
746755
self.add('AUTH', 'accounts', 'images/v2')
747-
self.add('AUTH', 'accounts', 'images/v2/direct_upload')
756+
self.add('AUTH', 'accounts', 'images/v2/direct_upload', content_type={'POST':'multipart/form-data'})
748757

749758
self.add('VOID', 'accounts', 'intel')
750759
self.add('VOID', 'accounts', 'intel-phishing')
@@ -757,7 +766,7 @@ def accounts_extras(self):
757766
self.add('AUTH', 'accounts', 'intel/domain/bulk')
758767
self.add('AUTH', 'accounts', 'intel/indicator-feeds')
759768
self.add('AUTH', 'accounts', 'intel/indicator-feeds', 'data')
760-
self.add('AUTH', 'accounts', 'intel/indicator-feeds', 'snapshot')
769+
self.add('AUTH', 'accounts', 'intel/indicator-feeds', 'snapshot', content_type={'PUT':'multipart/form-data'})
761770
self.add('VOID', 'accounts', 'intel/indicator-feeds/permissions')
762771
self.add('AUTH', 'accounts', 'intel/indicator-feeds/permissions/add')
763772
self.add('AUTH', 'accounts', 'intel/indicator-feeds/permissions/remove')
@@ -776,7 +785,7 @@ def accounts_extras(self):
776785

777786
self.add('VOID', 'accounts', 'pages')
778787
self.add('AUTH', 'accounts', 'pages/projects')
779-
self.add('AUTH', 'accounts', 'pages/projects', 'deployments')
788+
self.add('AUTH', 'accounts', 'pages/projects', 'deployments', content_type={'POST':'multipart/form-data'})
780789
self.add('VOID', 'accounts', 'pages/projects', 'deployments', 'history')
781790
self.add('AUTH', 'accounts', 'pages/projects', 'deployments', 'history/logs')
782791
self.add('AUTH', 'accounts', 'pages/projects', 'deployments', 'retry')
@@ -845,7 +854,7 @@ def zones_extras(self):
845854

846855
self.add('VOID', 'zones', 'hostnames')
847856
self.add('AUTH', 'zones', 'hostnames/settings')
848-
self.add('AUTH', 'zones', 'snippets')
857+
self.add('AUTH', 'zones', 'snippets', content_type={'PUT':'multipart/form-data'})
849858
self.add('AUTH', 'zones', 'snippets', 'content')
850859
self.add('AUTH', 'zones', 'snippets/snippet_rules')
851860

@@ -907,7 +916,7 @@ def zones_api_gateway(self):
907916
self.add('AUTH', 'zones', 'api_gateway/schemas')
908917
self.add('VOID', 'zones', 'api_gateway/settings')
909918
self.add('AUTH', 'zones', 'api_gateway/settings/schema_validation')
910-
self.add('AUTH', 'zones', 'api_gateway/user_schemas')
919+
self.add('AUTH', 'zones', 'api_gateway/user_schemas', content_type={'POST':'multipart/form-data'})
911920
self.add('AUTH', 'zones', 'api_gateway/user_schemas', 'operations')
912921

913922

0 commit comments

Comments
 (0)