bitstamp api is added

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="tradingfeatures",
-    version="0.3.8",
+    version="0.3.9",
     author="Onur Cetinkol",
     author_email="[email protected]",
     description="A small package to get history, easily download all avaliable history to csv or update current csv files",

tradingfeatures/__init__.py

@@ -1,3 +1,4 @@
 from .tools import bitfinex
 from tradingfeatures.bitmex_fundings import bitmex
-from tradingfeatures.main import base
+from tradingfeatures.bitstamp import bitstamp
+from tradingfeatures.main import base, base_v2

tradingfeatures/bitstamp.py

@@ -0,0 +1,136 @@
+import time
+import requests
+import pandas as pd
+
+from datetime import datetime
+
+class bitstamp:
+
+    def test(self, start=1364778000, end=time.time()):
+        currency_pair = 'btcusd'
+        address = f'https://www.bitstamp.net/api/v2/ohlc/{currency_pair}/'
+        # query = {'start': , 'end': , 'step': 3600, 'limit': 1000}
+        query = {'start': start, 'end': end, 'step': 3600, 'limit': 1000}
+
+        return self.get_(address, query)
+
+    def get_(self, address, query):
+        r = requests.get(address, params=query)
+        if r.status_code != 200:    # Bad response handler
+            print(r.json())
+            r.raise_for_status()
+
+        result = r.json()['data']['ohlc']
+
+        df = pd.DataFrame(result)   # fix index
+        # df['date'] = pd.to_datetime(df['timestamp'], unit='s', utc=True)
+        # datetime.utcfromtimestamp(1364778000)
+
+        # print('debug')
+
+        # self.get_hist('1h',)
+
+        # # Bitmex remaining limit
+        # if 'x-ratelimit-remaining' in r.headers:
+        #     if int(r.headers['x-ratelimit-remaining']) <= 1:
+        #         print('sleeping...')
+        #         time.sleep(61)
+        return df
+
+    def get_hist(self, timeframe, start=1364778000, end=int(time.time()), symbol='tBTCUSD'):
+        # if timeframe not in self.times_dict:
+        #     raise Exception('enter a valid timeframe')
+
+        # minutes = self.times_dict[timeframe]
+        minutes = 60
+        interval = 60 * minutes
+        per_step = 1000
+
+        total_entries = (end - start) // interval
+        steps = (total_entries // per_step) + 1
+        # if steps == 0: steps = 1
+        df = pd.DataFrame(columns=['high', 'timestamp', 'volume', 'low', 'close', 'open'])
+
+        for i in range(steps):
+            start_batch = start + (interval*i*per_step)
+            end_batch = start_batch + (interval*per_step)
+            try:
+                df_temp = self.test(start=str(start_batch), end=str(end_batch))
+            except:
+                print('hata!', start_batch, end_batch)
+                if steps <= 1: return None
+
+            # print(len(df_temp))
+
+            df_temp = pd.concat([df, df_temp])            
+            df = df_temp
+
+            print(f'  {i} of {steps}')
+            # time.sleep(1.01)
+
+        df.drop_duplicates(subset='timestamp', inplace=True)
+        df = df.astype(float)
+
+        # df['date'] = conv_time_v2(df['timestamp'])
+        df['date'] = pd.to_datetime(df['timestamp'], unit='s', utc=True)
+
+        return df
+
+    def get_funding_rates(self, df_path=None, reverse='false', save_csv=True):
+
+        address = 'https://www.bitmex.com/api/v1/funding'
+        symbol = 'XBT'
+        query = {'symbol': symbol, 'count': 500, 'reverse': 'false', 'startTime': 0}        
+
+        r = self.get_(address, query)
+
+        appended_data = []
+
+        # For updates
+        if df_path:
+            df_fundings = pd.read_csv(df_path)  # check if it is proper
+            appended_data.append(df_fundings)
+
+            last_time = df_fundings['timestamp'][-1:]
+            query['startTime'] = last_time             
+
+        for i in range(10000):
+            r = self.get_(address, query)
+
+            df_fundings = pd.read_json(r.content)
+
+            appended_data.append(df_fundings)
+
+            if len(df_fundings) < query['count']:
+                print('completed!')
+                break
+
+            last_time = df_fundings['timestamp'][-1:]
+            query['startTime'] = last_time 
+
+        df_fundings = pd.concat(appended_data, ignore_index=True).drop_duplicates()
+        if save_csv:
+            df_fundings.to_csv('bitmex_fundings.csv', index=False)
+        else:
+            return df_fundings        
+
+    def price_funding_merger(self, df, df_fundings):
+        # TODO clean it, check it for things that might be missed
+
+        df_fundings['date'] = pd.to_datetime(df_fundings['timestamp'])
+        df['date'] = pd.to_datetime(df['date'])
+
+        # a much better merger
+        df.set_index('date', inplace=True)
+        df_fundings.set_index('date', inplace=True)
+        df_fundings.pop('timestamp')
+
+        merged = df.join(df_fundings)
+        merged.fillna(method='ffill', inplace=True)
+
+        return merged
+
+    def shorts_longs(self):
+        # TODO
+        pass
+

tradingfeatures/main.py

@@ -3,6 +3,7 @@
 import pandas as pd
 
 from tradingfeatures import bitfinex
+from tradingfeatures import bitstamp
 from tradingfeatures import bitmex
 
 
@@ -23,6 +24,78 @@ def get(self, limit=10000):
         merged = merged[self.columns]
 
         return merged.to_numpy()
+
+class base_v2:
+
+    def __init__(self):
+        self.bitfinex = bitfinex()
+        self.bitstamp = bitstamp()
+        self.bitmex = bitmex()
+
+        self.columns = ['open', 'low', 'high', 'close', 'volume']
+        self.columns_final = ['close', 'low', 'high', 'volume', 'fundingRate']
+
+    def get(self, limit=10000):
+        df_bitfinex = self.bitfinex.get(10000)
+        df_bitmex = self.bitmex.get_funding_rates(save_csv=False)
+
+        merged = self.bitmex.price_funding_merger(df_bitfinex, df_bitmex)
+
+        merged = merged[self.columns]
+
+        return merged.to_numpy()
+
+    def get_all(self, path, fundings=False, date=True, save=True, update=False):
+        if update:
+            df1 = self.df1_updated
+        else:
+            df1 = self.bitfinex.get_hist('1h').set_index('timestamp')
+        df2 = self.bitstamp.get_hist('1h').set_index('timestamp')
+
+        df1.index = df1.index.astype(int)
+        df2.index = df2.index.astype(int)
+
+        df1 = df1[self.columns]
+        df2 = df2[self.columns]
+
+        if save:
+            df1.to_csv(path + '/bitfinex_1h.csv')
+            df2.to_csv(path + '/bitstamp_1h.csv')
+
+        df_temp = df1.join(df2, lsuffix='_bitfinex', rsuffix='_bitstamp')
+
+        df_final = pd.DataFrame(columns=self.columns)
+
+        for item in self.columns:
+            if item == 'volume':
+                df_final[item] = df_temp.loc[:, df_temp.columns.str.contains(item)].sum(axis=1)
+            else:
+                df_final[item] = df_temp.loc[:, df_temp.columns.str.contains(item)].mean(axis=1)
+
+        if date:
+            df_final['date'] = pd.to_datetime(df_final.index, unit='s', utc=True)
+
+        if fundings:
+            final_columns = self.columns
+            final_columns.append('fundingRate')
+
+            df_bitmex = self.bitmex.get_funding_rates(save_csv=False)
+
+            merged = self.bitmex.price_funding_merger(df_final, df_bitmex)
+            df_final = merged[final_columns]
+
+        return df_final
+
+    def update_all(self, path, fundings):
+        bitfinex_path = path + '/bitfinex_1h.csv'        
+
+        self.bitfinex.update_csv(bitfinex_path, alternative_mode=True)
+
+        self.df1_updated = pd.read_csv(bitfinex_path, index_col='timestamp')
+
+        updated = self.get_all(path, fundings, update=True)
+
+        updated.to_csv(path + '/merged_1h.csv')
 
 # base = base()
 

tradingfeatures/tools.py

@@ -64,6 +64,7 @@ def get_hist(self, timeframe, start=1364778000, end=int(time.time()), symbol='tB
 
         interval = 60 * minutes
         steps = ((end - start) // interval) // 120
+        # steps = steps + 1
         if steps == 0: steps = 1
         df = pd.DataFrame(columns=self.columns)