forked from gautam132002/crashpredict
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfetch.py
117 lines (89 loc) · 3.16 KB
/
fetch.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
# from selenium import webdriver
# from selenium.webdriver.firefox.options import Options
# from bs4 import BeautifulSoup
# import json
# import pandas as pd
# def main():
# url = "https://ws.duelbits.com/games/crash/history"
# html_source = fetch_and_display_html(url)
# json_data = extract_json_from_html(html_source)
# if json_data:
# process_and_save_to_csv(json_data, "data.csv")
# return json_data[0]
# def fetch_and_display_html(url):
# try:
# options = Options()
# options.headless = True
# browser = webdriver.Firefox(options=options)
# browser.get(url)
# browser.implicitly_wait(10)
# html_source = browser.page_source
# return html_source
# except Exception as e:
# print(f"Error: {str(e)}")
# finally:
# browser.quit()
# def extract_json_from_html(html_source):
# soup = BeautifulSoup(html_source, 'html.parser')
# json_div = soup.find('div', {'id': 'json'})
# if json_div:
# json_data = json.loads(json_div.text)
# return json_data.get("history", [])
# else:
# print("Error: Unable to find div with id='json'")
# return []
# def process_and_save_to_csv(json_data, filename):
# if json_data:
# df = pd.DataFrame(json_data)
# df.to_csv(filename, index=False)
# print(f"Data saved to {filename}")
# else:
# print("Error: No JSON data to process and save")
# if __name__ == "__main__":
# x = main()
# print(x)
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service
from webdriver_manager.firefox import GeckoDriverManager
from bs4 import BeautifulSoup
import json
import pandas as pd
def main():
url = "https://ws.duelbits.com/games/crash/history"
html_source = fetch_and_display_html(url)
json_data = extract_json_from_html(html_source)
if json_data:
process_and_save_to_csv(json_data, "data.csv")
return json_data[0]
def fetch_and_display_html(url):
try:
options = Options()
options.headless = True
service = Service(GeckoDriverManager().install()) # Use GeckoDriverManager to get the path dynamically
browser = webdriver.Firefox(service=service, options=options)
browser.get(url)
browser.implicitly_wait(10)
html_source = browser.page_source
return html_source
except Exception as e:
print(f"Error: {str(e)}")
finally:
if browser:
browser.quit()
def extract_json_from_html(html_source):
soup = BeautifulSoup(html_source, 'html.parser')
json_div = soup.find('div', {'id': 'json'})
if json_div:
json_data = json.loads(json_div.text)
return json_data.get("history", [])
else:
print("Error: Unable to find div with id='json'")
return []
def process_and_save_to_csv(json_data, filename):
if json_data:
df = pd.DataFrame(json_data)
df.to_csv(filename, index=False)
print(f"Data saved to {filename}")
else:
print("Error: No JSON data to process and save")