From c311abc78ad180e78a8b28237b70b223a55a1af3 Mon Sep 17 00:00:00 2001 From: realiti4 Date: Fri, 26 Feb 2021 18:44:39 +0300 Subject: [PATCH] bitstamp api is added --- setup.py | 2 +- tradingfeatures/__init__.py | 3 +- tradingfeatures/bitstamp.py | 136 ++++++++++++++++++++++++++++++++++++ tradingfeatures/main.py | 73 +++++++++++++++++++ tradingfeatures/tools.py | 1 + 5 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 tradingfeatures/bitstamp.py diff --git a/setup.py b/setup.py index cd83341..a7f43f1 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="tradingfeatures", - version="0.3.8", + version="0.3.9", author="Onur Cetinkol", author_email="realiti44@gmail.com", description="A small package to get history, easily download all avaliable history to csv or update current csv files", diff --git a/tradingfeatures/__init__.py b/tradingfeatures/__init__.py index c25f064..8b2a641 100644 --- a/tradingfeatures/__init__.py +++ b/tradingfeatures/__init__.py @@ -1,3 +1,4 @@ from .tools import bitfinex from tradingfeatures.bitmex_fundings import bitmex -from tradingfeatures.main import base +from tradingfeatures.bitstamp import bitstamp +from tradingfeatures.main import base, base_v2 diff --git a/tradingfeatures/bitstamp.py b/tradingfeatures/bitstamp.py new file mode 100644 index 0000000..e5d2fe1 --- /dev/null +++ b/tradingfeatures/bitstamp.py @@ -0,0 +1,136 @@ +import time +import requests +import pandas as pd + +from datetime import datetime + +class bitstamp: + + def test(self, start=1364778000, end=time.time()): + currency_pair = 'btcusd' + address = f'https://www.bitstamp.net/api/v2/ohlc/{currency_pair}/' + # query = {'start': , 'end': , 'step': 3600, 'limit': 1000} + query = {'start': start, 'end': end, 'step': 3600, 'limit': 1000} + + return self.get_(address, query) + + def get_(self, address, query): + r = requests.get(address, params=query) + if r.status_code != 200: # Bad response handler + print(r.json()) + r.raise_for_status() + + result = r.json()['data']['ohlc'] + + df = pd.DataFrame(result) # fix index + # df['date'] = pd.to_datetime(df['timestamp'], unit='s', utc=True) + # datetime.utcfromtimestamp(1364778000) + + # print('debug') + + # self.get_hist('1h',) + + # # Bitmex remaining limit + # if 'x-ratelimit-remaining' in r.headers: + # if int(r.headers['x-ratelimit-remaining']) <= 1: + # print('sleeping...') + # time.sleep(61) + return df + + def get_hist(self, timeframe, start=1364778000, end=int(time.time()), symbol='tBTCUSD'): + # if timeframe not in self.times_dict: + # raise Exception('enter a valid timeframe') + + # minutes = self.times_dict[timeframe] + minutes = 60 + interval = 60 * minutes + per_step = 1000 + + total_entries = (end - start) // interval + steps = (total_entries // per_step) + 1 + # if steps == 0: steps = 1 + df = pd.DataFrame(columns=['high', 'timestamp', 'volume', 'low', 'close', 'open']) + + for i in range(steps): + start_batch = start + (interval*i*per_step) + end_batch = start_batch + (interval*per_step) + try: + df_temp = self.test(start=str(start_batch), end=str(end_batch)) + except: + print('hata!', start_batch, end_batch) + if steps <= 1: return None + + # print(len(df_temp)) + + df_temp = pd.concat([df, df_temp]) + df = df_temp + + print(f' {i} of {steps}') + # time.sleep(1.01) + + df.drop_duplicates(subset='timestamp', inplace=True) + df = df.astype(float) + + # df['date'] = conv_time_v2(df['timestamp']) + df['date'] = pd.to_datetime(df['timestamp'], unit='s', utc=True) + + return df + + def get_funding_rates(self, df_path=None, reverse='false', save_csv=True): + + address = 'https://www.bitmex.com/api/v1/funding' + symbol = 'XBT' + query = {'symbol': symbol, 'count': 500, 'reverse': 'false', 'startTime': 0} + + r = self.get_(address, query) + + appended_data = [] + + # For updates + if df_path: + df_fundings = pd.read_csv(df_path) # check if it is proper + appended_data.append(df_fundings) + + last_time = df_fundings['timestamp'][-1:] + query['startTime'] = last_time + + for i in range(10000): + r = self.get_(address, query) + + df_fundings = pd.read_json(r.content) + + appended_data.append(df_fundings) + + if len(df_fundings) < query['count']: + print('completed!') + break + + last_time = df_fundings['timestamp'][-1:] + query['startTime'] = last_time + + df_fundings = pd.concat(appended_data, ignore_index=True).drop_duplicates() + if save_csv: + df_fundings.to_csv('bitmex_fundings.csv', index=False) + else: + return df_fundings + + def price_funding_merger(self, df, df_fundings): + # TODO clean it, check it for things that might be missed + + df_fundings['date'] = pd.to_datetime(df_fundings['timestamp']) + df['date'] = pd.to_datetime(df['date']) + + # a much better merger + df.set_index('date', inplace=True) + df_fundings.set_index('date', inplace=True) + df_fundings.pop('timestamp') + + merged = df.join(df_fundings) + merged.fillna(method='ffill', inplace=True) + + return merged + + def shorts_longs(self): + # TODO + pass + diff --git a/tradingfeatures/main.py b/tradingfeatures/main.py index 88ec4a7..4aada3d 100644 --- a/tradingfeatures/main.py +++ b/tradingfeatures/main.py @@ -3,6 +3,7 @@ import pandas as pd from tradingfeatures import bitfinex +from tradingfeatures import bitstamp from tradingfeatures import bitmex @@ -23,6 +24,78 @@ def get(self, limit=10000): merged = merged[self.columns] return merged.to_numpy() + +class base_v2: + + def __init__(self): + self.bitfinex = bitfinex() + self.bitstamp = bitstamp() + self.bitmex = bitmex() + + self.columns = ['open', 'low', 'high', 'close', 'volume'] + self.columns_final = ['close', 'low', 'high', 'volume', 'fundingRate'] + + def get(self, limit=10000): + df_bitfinex = self.bitfinex.get(10000) + df_bitmex = self.bitmex.get_funding_rates(save_csv=False) + + merged = self.bitmex.price_funding_merger(df_bitfinex, df_bitmex) + + merged = merged[self.columns] + + return merged.to_numpy() + + def get_all(self, path, fundings=False, date=True, save=True, update=False): + if update: + df1 = self.df1_updated + else: + df1 = self.bitfinex.get_hist('1h').set_index('timestamp') + df2 = self.bitstamp.get_hist('1h').set_index('timestamp') + + df1.index = df1.index.astype(int) + df2.index = df2.index.astype(int) + + df1 = df1[self.columns] + df2 = df2[self.columns] + + if save: + df1.to_csv(path + '/bitfinex_1h.csv') + df2.to_csv(path + '/bitstamp_1h.csv') + + df_temp = df1.join(df2, lsuffix='_bitfinex', rsuffix='_bitstamp') + + df_final = pd.DataFrame(columns=self.columns) + + for item in self.columns: + if item == 'volume': + df_final[item] = df_temp.loc[:, df_temp.columns.str.contains(item)].sum(axis=1) + else: + df_final[item] = df_temp.loc[:, df_temp.columns.str.contains(item)].mean(axis=1) + + if date: + df_final['date'] = pd.to_datetime(df_final.index, unit='s', utc=True) + + if fundings: + final_columns = self.columns + final_columns.append('fundingRate') + + df_bitmex = self.bitmex.get_funding_rates(save_csv=False) + + merged = self.bitmex.price_funding_merger(df_final, df_bitmex) + df_final = merged[final_columns] + + return df_final + + def update_all(self, path, fundings): + bitfinex_path = path + '/bitfinex_1h.csv' + + self.bitfinex.update_csv(bitfinex_path, alternative_mode=True) + + self.df1_updated = pd.read_csv(bitfinex_path, index_col='timestamp') + + updated = self.get_all(path, fundings, update=True) + + updated.to_csv(path + '/merged_1h.csv') # base = base() diff --git a/tradingfeatures/tools.py b/tradingfeatures/tools.py index 6bfbeca..bfae02f 100644 --- a/tradingfeatures/tools.py +++ b/tradingfeatures/tools.py @@ -64,6 +64,7 @@ def get_hist(self, timeframe, start=1364778000, end=int(time.time()), symbol='tB interval = 60 * minutes steps = ((end - start) // interval) // 120 + # steps = steps + 1 if steps == 0: steps = 1 df = pd.DataFrame(columns=self.columns)