[DB] Remove Database Dependency

Signed-off-by: Bill Chan <[email protected]>

Author: billpwchan

.gitignore

@@ -164,3 +164,4 @@ cython_debug/
 !/filters/Triple_Cross.py
 
 /backtesting_report/
+/stock_filter_report/

README.md

@@ -73,9 +73,6 @@ Password_md5 = <Futu Login Password Md5 Value>
 HistoryDataFormat = ["code","time_key","open","close","high","low","pe_ratio","turnover_rate","volume","turnover","change_rate","last_close"]
 SubscribedDataFormat = None
 
-[Database]
-Database_path = <Your SQLite Database File Path>
-
 [TradePreference]
 LotSizeMultiplier = <# of Stocks to Buy per Signal>
 MaxPercPerAsset = <Maximum % of Capital Allocated per Asset>
@@ -127,20 +124,6 @@ to https://openapi.futunn.com/futu-api-doc/qa/quote.html
 
 **MAKE SURE YOU LOGIN TO FUTU OPEND FIRST BEFORE STARTING FUTU_ALGO!**
 
-### 3. Initialize SQLite Database
-
-Go to [SQLite official website](https://www.sqlite.org/quickstart.html) and follow the QuickStart instruction to install
-SQLite tools in the device.
-
-Create a folder named 'database' in the root folder, and execute the SQLite DDL file stored in *./util/database_ddl.sql*
-.
-
-```
-./
-  ├── database
-  │       └── stock_data.sqlite
-```
-
 ### 4. Download Data (e.g. 1M Data for max. 2 Years)
 
 For **Windows**:
@@ -166,10 +149,6 @@ If you want to refresh all data, use the following command instead (WITH CAUTION
 
     python main_backend.py -fu  /   python main_backend.py --force_update
 
-Store all data from CSV to SQLite Database *(Currently the database isn't used for any feature)*
-
-    python main_backend.py -d   /   python main_backend.py --database
-
 Execute High-Frequency Trading (HFT) with a Pre-defined Strategy
 
     python main_backend.py -s MACD_Cross    /   python main_backend.py --strategy MACD_Cross

SECURITY.md

@@ -13,9 +13,6 @@ Password_md5 = 2134342ABC2D03780772038A7816
 HistoryDataFormat = ["code","time_key","open","close","high","low","pe_ratio","turnover_rate","volume","turnover","change_rate","last_close"]
 SubscribedDataFormat = None
 
-[Database]
-Database_path = ./database/stock_data.sqlite
-
 [TradePreference]
 LotSizeMultiplier = 2
 MaxPercPerAsset = 10

config_template.ini

@@ -33,8 +33,10 @@
 
 from util import logger
 from util.global_vars import *
+from deprecated import deprecated
 
 
+@deprecated(version='1.0', reason="Database dependency is removed.")
 class DatabaseInterface:
     def __init__(self, database_path):
         Path("./database/").mkdir(parents=True, exist_ok=True)

engines/data_engine.py

@@ -20,7 +20,9 @@
 from datetime import date
 from multiprocessing import Pool, cpu_count
 
-from engines.data_engine import DatabaseInterface, HKEXInterface, YahooFinanceInterface
+import pandas as pd
+
+from engines.data_engine import HKEXInterface, YahooFinanceInterface
 from util import logger
 from util.global_vars import *
 
@@ -87,20 +89,16 @@ def update_filtered_equity_pools(self):
         filtered_stock_list = pool.map(self.validate_stock_individual, self.full_equity_list)
         pool.close()
         pool.join()
-        # filtered_stock_list = []
-        # for stock_code in self.full_equity_list:
-        #     filtered_stock_list.append(self.validate_stock_individual(stock_code))
-
-        # Remove Redundant Records (If Exists)
-        database = DatabaseInterface(database_path=self.config['Database'].get('Database_path'))
-        database.delete_stock_pool_from_date(date.today().strftime("%Y-%m-%d"))
-        database.commit()
+
+        filtered_stock_df = pd.DataFrame([], columns=['filter', 'code'])
+
         # Flatten Nested List
         for sublist in filtered_stock_list:
             for record in sublist:
-                database.add_stock_pool(date.today().strftime("%Y-%m-%d"), record[0], record[1])
+                filtered_stock_df.append({'filter': record[0], 'code': record[1]})
                 self.default_logger.info(f"Added Filtered Stock {record[1]} based on Filter {record[0]}")
-            database.commit()
+        filtered_stock_df.to_csv(PATH_FILTER_REPORT / f'{date.today().strftime("%Y-%m-%d")}_stock_list', index=False,
+                                 encoding='utf-8-sig')
 
     def parse_stock_info(self, stock_code):
         return stock_code, YahooFinanceInterface.get_stock_info(stock_code)

engines/stock_filter_engine.py

@@ -22,6 +22,7 @@
 import subprocess
 from datetime import date
 
+from deprecated import deprecated
 from futu import *
 
 import engines
@@ -45,8 +46,6 @@ def __init__(self):
         self.username = self.config['FutuOpenD.Credential'].get('Username')
         # self.password = self.config['FutuOpenD.Credential'].get('Password')
         self.password_md5 = self.config['FutuOpenD.Credential'].get('Password_md5')
-        self.futu_data = engines.data_engine.DatabaseInterface(
-            database_path=self.config['Database'].get('Database_path'))
         self.trd_env = TrdEnv.REAL if self.config.get('FutuOpenD.Config', 'TrdEnv') == 'REAL' else TrdEnv.SIMULATE
         self.trading_util = engines.TradingUtil(self.quote_ctx, self.trade_ctx, self.trd_env)
         # Futu-Specific Variables
@@ -135,65 +134,10 @@ def __save_historical_data(self, stock_code: str, start_date: date, end_date: da
                 time.sleep(1)
                 self.default_logger.error(f'Historical Data Store Error: {data}')
 
-    def __store_data_database(self, data, k_type):
-        for index, row in data.iterrows():
-            self.futu_data.add_stock_data(row['code'], row['time_key'], row['open'], row['close'], row['high'],
-                                          row['low'], row['pe_ratio'], row['turnover_rate'], row['volume'],
-                                          row['turnover'], row['change_rate'], row['last_close'], k_type)
-        self.futu_data.commit()
-
     def get_market_state(self):
-        """
-        获取全局状态
-
-        :return: (ret, data)
-
-                ret == RET_OK data为包含全局状态的字典，含义如下
-
-                ret != RET_OK data为错误描述字符串
-
-                =====================   ===========   ==============================================================
-                key                      value类型                        说明
-                =====================   ===========   ==============================================================
-                market_sz               str            深圳市场状态，参见MarketState
-                market_us               str            美国市场状态，参见MarketState
-                market_sh               str            上海市场状态，参见MarketState
-                market_hk               str            香港市场状态，参见MarketState
-                market_hkfuture         str            香港期货市场状态，参见MarketState
-                market_usfuture         str            美国期货市场状态，参见MarketState
-                server_ver              str            FutuOpenD版本号
-                trd_logined             str            '1'：已登录交易服务器，'0': 未登录交易服务器
-                qot_logined             str            '1'：已登录行情服务器，'0': 未登录行情服务器
-                timestamp               str            Futu后台服务器当前时间戳(秒)
-                local_timestamp         double         FutuOpenD运行机器当前时间戳(
-                =====================   ===========   ==============================================================
-        """
         return self.quote_ctx.get_global_state()
 
-    def get_referencestock_list(self, stock_code: str) -> pd.DataFrame:
-        """
-        获取证券的关联数据
-        :param code: 证券id，str，例如HK.00700
-        :return: (ret, data)
-        ret == RET_OK 返回pd dataframe数据，数据列格式如下
-
-        ret != RET_OK 返回错误字符串
-        =======================   ===========   ==============================================================================
-        参数                        类型                        说明
-        =======================   ===========   ==============================================================================
-        code                        str           证券代码
-        lot_size                    int           每手数量
-        stock_type                  str           证券类型，参见SecurityType
-        stock_name                  str           证券名字
-        list_time                   str           上市时间（美股默认是美东时间，港股A股默认是北京时间）
-        wrt_valid                   bool          是否是窝轮，如果为True，下面wrt开头的字段有效
-        wrt_type                    str           窝轮类型，参见WrtType
-        wrt_code                    str           所属正股
-        future_valid                bool          是否是期货，如果为True，下面future开头的字段有效
-        future_main_contract        bool          是否主连合约（期货特有字段）
-        future_last_trade_time      string        最后交易时间（期货特有字段，非主连期货合约才有值）
-        =======================   ===========   ==============================================================================
-        """
+    def get_reference_stock_list(self, stock_code: str) -> pd.DataFrame:
         output_df = pd.DataFrame()
         for security_reference_type in self.security_type_list:
             ret, data = self.quote_ctx.get_referencestock_list(stock_code, security_reference_type)
@@ -398,35 +342,11 @@ def update_stock_basicinfo(self):
         output_df.to_csv(output_path, index=False, encoding='utf-8-sig')
         self.default_logger.info(f'Stock Static Basic Info Updated: {output_path}')
 
-    def store_all_data_database(self):
-        """
-        Store all files in ./data/{stock_code}/*.csv to the database in pre-defined format.
-        """
-        file_list = glob.glob(f"./data/*/*_1M.csv", recursive=True)
-        for input_file in file_list:
-            input_csv = pd.read_csv(input_file, index_col=None)
-            self.default_logger.info(f'Saving to Database: {input_file}')
-            self.__store_data_database(input_csv, k_type=KLType.K_1M)
-
-        file_list = glob.glob(f"./data/*/*_1D.csv", recursive=True)
-        for input_file in file_list:
-            input_csv = pd.read_csv(input_file, index_col=None)
-            self.default_logger.info(f'Saving to Database: {input_file}')
-            self.__store_data_database(input_csv, k_type=KLType.K_DAY)
-
-        file_list = glob.glob(f"./data/*/*_1W.csv", recursive=True)
-        for input_file in file_list:
-            input_csv = pd.read_csv(input_file, index_col=None)
-            self.default_logger.info(f'Saving to Database: {input_file}')
-            self.__store_data_database(input_csv, k_type=KLType.K_WEEK)
-
     def cur_kline_evalaute(self, stock_list: list, strategy_map: dict, sub_type: SubType = SubType.K_1M):
         """
             Real-Time K-Line!
-        :param input_data: Dictionary in Format {'HK.00001': pd.Dataframe, 'HK.00002': pd.Dataframe}
         :param stock_list: A List of Stock Code with Format (e.g., [HK.00001, HK.00002])
         :param strategy_map: Strategies defined in ./strategies class. Should be inherited from based class Strategies
-        :param timeout: Subscription Timeout in secs.
         :param sub_type: Subscription SubType for FuTu (i.e., Trading Frequency)
 
         """

engines/trading_engine.py

@@ -612,6 +612,6 @@ def resize_grips(self):
 
 if __name__ == "__main__":
     app = QApplication(sys.argv)
-    app.setWindowIcon(QtGui.QIcon("icon.ico"))
+    app.setWindowIcon(QtGui.QIcon("images/icon.ico"))
     window = SplashScreen()
     sys.exit(app.exec_())

icon.ico renamed to images/icon.ico

@@ -35,10 +35,10 @@ def __daily_update_filters():
 
 def daily_update_data(futu_trade, stock_list: list, force_update: bool = False):
     # Daily Update Filtered Security
-    procs = []
-    proc = Process(target=__daily_update_filters)  # instantiating without any argument
-    procs.append(proc)
-    proc.start()
+    # procs = []
+    # proc = Process(target=__daily_update_filters)  # instantiating without any argument
+    # procs.append(proc)
+    # proc.start()
 
     # Daily Update Stock Info (Need to Rethink!!!)
     # stock_filter.update_stock_info()
@@ -57,16 +57,13 @@ def daily_update_data(futu_trade, stock_list: list, force_update: bool = False):
     for stock_code in stock_list:
         futu_trade.update_DW_data(stock_code, force_update=force_update, k_type=KLType.K_DAY)
         futu_trade.update_DW_data(stock_code, force_update=force_update, k_type=KLType.K_WEEK)
-        futu_trade.update_1M_data(stock_code, force_update=force_update, default_days=80)
-
-    # Daily Update FuTu Historical Data
-    # futu_trade.store_all_data_database()
+        futu_trade.update_1M_data(stock_code, force_update=force_update, default_days=365)
 
     # Clean non-trading days data
     DataProcessingInterface.clear_empty_data()
 
-    for proc in procs:
-        proc.join()
+    # for proc in procs:
+    #     proc.join()
 
 
 def __dynamic_instantiation(prefix: str, module_name: str, optional_parameter=None):
@@ -82,7 +79,7 @@ def __dynamic_instantiation(prefix: str, module_name: str, optional_parameter=No
 def __init_strategy(strategy_name: str, input_data: dict) -> Strategies:
     """
     Return a trading strategy instance using a strategy name in string.
-    :param strategy_name: an available strategy module name in the strategies folder
+    :param strategy_name: an available strategy module name in the strategies' folder
     :param input_data: Initialized input data for the strategy to calculate the technical indicator
     :return: a strategy instance
     """
@@ -147,7 +144,6 @@ def main():
                         action="store_true")
     parser.add_argument("-fu", "--force_update",
                         help="Force Update All Data Up to Max. Allowed Years (USE WITH CAUTION)", action="store_true")
-    parser.add_argument("-d", "--database", help="Store All CSV Data to Database", action="store_true")
 
     # Trading Related Arguments
     strategy_list = [Path(file_name).name[:-3] for file_name in glob.glob("./strategies/*.py") if
@@ -176,8 +172,6 @@ def main():
     # Initialize Stock List
     stock_list = json.loads(config.get('TradePreference', 'StockList'))
     if not stock_list:
-        # stock_list = data_engine.DatabaseInterface(
-        #     database_path=config.get('Database', 'Database_path')).get_stock_list()
         # Directly get list of stock codes from the data folder. Easier.
         stock_list = [str(f.path).replace('./data/', '') for f in os.scandir("./data/") if f.is_dir()]
         stock_list = stock_list[:-1]
@@ -192,14 +186,11 @@ def main():
     if args.update or args.force_update:
         # Daily Update Data
         daily_update_data(futu_trade=futu_trade, stock_list=stock_list, force_update=args.force_update)
-    if args.database:
-        # Update ALl Data to Database
-        futu_trade.store_all_data_database()
     if args.strategy:
         # Stock Basket => 4 Parts
         # 1. Currently Holding Stocks (i.e., in the trading account with existing position)
         # 2. Filtered Stocks (i.e., based on 1D data if -f option is adopted
-        # 3. StockList in config.ini (i.e., if empty, default use all stocks in the database)
+        # 3. StockList in config.ini (i.e., if empty, default use all stocks in the data folder)
         # 4. Top 30 HSI Constituents
         if args.filter:
             stock_list.extend(filtered_stock_list)

main.py

@@ -22,7 +22,7 @@
 import pkg_resources
 
 # ADD FILES
-files = ['icon.ico', 'themes/', 'stock_strategy_map_template.yml', 'config_template.ini']
+files = ['images/icon.ico', 'themes/', 'stock_strategy_map_template.yml', 'config_template.ini']
 
 # TARGET
 target = Executable(

main_backend.py

@@ -21,6 +21,10 @@
 
 import yaml
 
+PATH = Path.cwd()
+PATH_DATA = PATH / 'data'
+PATH_FILTER_REPORT = PATH / 'stock_filter_report'
+
 if not Path("config.ini").exists():
     raise SystemExit("Missing config.ini. Please use the config_template.ini to create your configuration.")