working on robcarver17#754, removed csv warning, removed backup issue

Author: rob

sysbrokers/broker_contract_position_data.py

@@ -2,7 +2,7 @@
 
 from syscore.constants import arg_not_supplied
 
-from sysdata.production.historic_positions_TO_DEPRECATE import contractPositionData
+from sysdata.production.historic_contract_positions import contractPositionData
 from sysdata.data_blob import dataBlob
 from sysobjects.production.positions import listOfContractPositions
 

sysdata/arctic/arctic_historic_contract_positions.py

@@ -26,50 +26,6 @@ def __repr__(self):
     def arctic(self):
         return self._arctic
 
-    def _update_position_for_contract_object_with_date_supplied(
-        self,
-        contract_object: futuresContract,
-        position: int,
-        date_index: datetime.datetime,
-    ):
-        new_position_series = pd.Series([position], index=[date_index])
-
-        try:
-            current_series = self.get_position_as_series_for_contract_object(
-                contract_object
-            )
-            self._update_position_for_contract_object_with_date_and_existing_data(
-                contract_object=contract_object,
-                current_series=current_series,
-                new_position_series=new_position_series,
-            )
-        except missingData:
-            ## no existing data
-            ## no need to update, just write the new series
-            self._write_updated_position_series_for_contract_object(
-                contract_object=contract_object,
-                updated_series=new_position_series,
-            )
-
-    def _update_position_for_contract_object_with_date_and_existing_data(
-        self,
-        contract_object: futuresContract,
-        current_series: pd.Series,
-        new_position_series: pd.Series,
-    ):
-
-        try:
-            assert new_position_series.index[0] > current_series.index[-1]
-        except:
-            error_msg = "Adding a position which is older than the last position!"
-            self.log.critical(error_msg)
-            raise Exception(error_msg)
-
-        updated_series = current_series.append(new_position_series)
-        self._write_updated_position_series_for_contract_object(
-            contract_object=contract_object, updated_series=updated_series
-        )
-
     def _write_updated_position_series_for_contract_object(
         self, contract_object: futuresContract, updated_series: pd.Series
     ):
@@ -80,36 +36,6 @@ def _write_updated_position_series_for_contract_object(
 
         self.arctic.write(ident=ident, data=updated_data_as_df)
 
-    def _delete_last_position_for_contract_object_without_checking(
-        self, contract_object: futuresContract
-    ):
-        try:
-            current_series = self.get_position_as_series_for_contract_object(
-                contract_object
-            )
-            self._delete_last_position_for_contract_object_without_checking_with_current_data(
-                contract_object=contract_object, current_series=current_series
-            )
-        except missingData:
-            ## no existing data can't delete
-            self.log.warn(
-                "Can't delete last position for %s, as none present"
-                % str(contract_object)
-            )
-
-    def _delete_last_position_for_contract_object_without_checking_with_current_data(
-        self, contract_object: futuresContract, current_series: pd.Series
-    ):
-        updated_series = current_series.drop(current_series.index[-1])
-        if len(updated_series) == 0:
-            self._delete_position_series_for_contract_object_without_checking(
-                contract_object
-            )
-        else:
-            self._write_updated_position_series_for_contract_object(
-                contract_object=contract_object, updated_series=updated_series
-            )
-
     def _delete_position_series_for_contract_object_without_checking(
         self, contract_object: futuresContract
     ):

sysdata/arctic/arctic_historic_strategy_positions.py

@@ -36,82 +36,6 @@ def get_list_of_instrument_strategies(self) -> listOfInstrumentStrategies:
 
         return listOfInstrumentStrategies(list_of_instrument_strategies)
 
-    def _delete_last_position_for_instrument_strategy_object_without_checking(
-        self, instrument_strategy: instrumentStrategy
-    ):
-        try:
-            current_series = self.get_position_as_series_for_instrument_strategy_object(
-                instrument_strategy
-            )
-            self._delete_last_position_for_instrument_strategy_object_without_checking_with_current_data(
-                instrument_strategy=instrument_strategy, current_series=current_series
-            )
-        except missingData:
-            ## no existing data can't delete
-            self.log.warn(
-                "Can't delete last position for %s, as none present"
-                % str(instrument_strategy)
-            )
-
-    def _delete_last_position_for_instrument_strategy_object_without_checking_with_current_data(
-        self, instrument_strategy: instrumentStrategy, current_series: pd.Series
-    ):
-        updated_series = current_series.drop(current_series.index[-1])
-        if len(updated_series) == 0:
-            self._delete_position_series_for_instrument_strategy_object_without_checking(
-                instrument_strategy
-            )
-        else:
-            self._write_updated_position_series_for_instrument_strategy_object(
-                instrument_strategy=instrument_strategy,
-                updated_series=updated_series,
-            )
-
-    def _update_position_for_instrument_strategy_object_with_date(
-        self,
-        instrument_strategy: instrumentStrategy,
-        position: int,
-        date_index: datetime.datetime,
-    ):
-
-        new_position_series = pd.Series([position], index=[date_index])
-
-        try:
-            current_series = self.get_position_as_series_for_instrument_strategy_object(
-                instrument_strategy
-            )
-            self._update_position_for_instrument_strategy_object_with_date_and_existing_data(
-                instrument_strategy=instrument_strategy,
-                current_series=current_series,
-                new_position_series=new_position_series,
-            )
-        except missingData:
-            ## no existing data
-            ## no need to update, just write the new series
-            self._write_updated_position_series_for_instrument_strategy_object(
-                instrument_strategy=instrument_strategy,
-                updated_series=new_position_series,
-            )
-
-    def _update_position_for_instrument_strategy_object_with_date_and_existing_data(
-        self,
-        instrument_strategy: instrumentStrategy,
-        current_series: pd.Series,
-        new_position_series: pd.Series,
-    ):
-
-        try:
-            assert new_position_series.index[0] > current_series.index[-1]
-        except:
-            error_msg = "Adding a position which is older than the last position!"
-            self.log.critical(error_msg)
-            raise Exception(error_msg)
-
-        updated_series = current_series.append(new_position_series)
-        self._write_updated_position_series_for_instrument_strategy_object(
-            instrument_strategy=instrument_strategy, updated_series=updated_series
-        )
-
     def _write_updated_position_series_for_instrument_strategy_object(
         self, instrument_strategy: instrumentStrategy, updated_series: pd.Series
     ):

sysdata/arctic/arctic_optimal_positions.py

@@ -0,0 +1,68 @@
+"""
+Read and write data from mongodb for individual futures contracts
+
+"""
+from typing import Union
+
+from syscore.exceptions import missingData
+from sysdata.arctic.arctic_connection import arcticData
+from sysdata.production.optimal_positions import optimalPositionData
+from syslogdiag.log_to_screen import logtoscreen
+
+from sysobjects.production.tradeable_object import (
+    instrumentStrategy,
+    listOfInstrumentStrategies,
+)
+
+import pandas as pd
+
+OPTIMAL_POSITION_COLLECTION = "optimal_positions"
+
+
+class arcticOptimalPositionData(optimalPositionData):
+    def __init__(self, mongo_db=None, log=logtoscreen("arcticOptimalPositionData")):
+
+        super().__init__(log=log)
+
+        self._arctic_connection = arcticData(
+            OPTIMAL_POSITION_COLLECTION, mongo_db=mongo_db
+        )
+
+    def __repr__(self):
+        return repr(self._arctic_connection)
+
+    @property
+    def arctic_connection(self):
+        return self._arctic_connection
+
+    def get_list_of_instrument_strategies_with_optimal_position(
+        self,
+    ) -> listOfInstrumentStrategies:
+
+        raw_list_of_instrument_strategies = self.arctic_connection.get_keynames()
+        list_of_instrument_strategies = [
+            instrumentStrategy.from_key(key)
+            for key in raw_list_of_instrument_strategies
+        ]
+
+        return listOfInstrumentStrategies(list_of_instrument_strategies)
+
+    def get_optimal_position_as_df_for_instrument_strategy(
+        self, instrument_strategy: instrumentStrategy
+    ) -> pd.DataFrame:
+
+        try:
+            ident = instrument_strategy.key
+            df_result = self.arctic_connection.read(ident)
+        except:
+            raise missingData
+
+        return df_result
+
+    def write_optimal_position_as_df_for_instrument_strategy_without_checking(
+        self,
+        instrument_strategy: instrumentStrategy,
+        optimal_positions_as_df: pd.DataFrame,
+    ):
+        ident = instrument_strategy.key
+        self.arctic_connection.write(ident, optimal_positions_as_df)

sysdata/csv/csv_contract_position_data.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from sysdata.production.historic_positions_TO_DEPRECATE import contractPositionData
+from sysdata.production.historic_contract_positions import contractPositionData
 from sysobjects.contracts import futuresContract
 from syscore.fileutils import resolve_path_and_filename_for_package
 from syscore.constants import arg_not_supplied
@@ -28,9 +28,10 @@ def __init__(
     def __repr__(self):
         return "csvContractPositionData accessing %s" % self._datapath
 
-    def write_position_df_for_contract(
-        self, contract: futuresContract, position_df: pd.DataFrame
+    def _write_updated_position_series_for_contract_object(
+        self, contract: futuresContract, update_series: pd.Series
     ):
+        position_df = pd.DataFrame(update_series)
         filename = self._filename_given_contract(contract)
         position_df.to_csv(filename, index_label=DATE_INDEX_NAME)
 

sysdata/csv/csv_optimal_position.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from sysdata.production.optimal_positions_TO_DEPRECATE import optimalPositionData
+from sysdata.production.optimal_positions import optimalPositionData
 from syscore.fileutils import resolve_path_and_filename_for_package
 from syscore.constants import arg_not_supplied
 from syslogdiag.log_to_screen import logtoscreen
@@ -28,11 +28,13 @@ def __init__(
     def __repr__(self):
         return "csvOptimalPositionData accessing %s" % self._datapath
 
-    def write_position_df_for_instrument_strategy(
-        self, instrument_strategy: instrumentStrategy, position_df: pd.DataFrame
+    def write_optimal_position_as_df_for_instrument_strategy_without_checking(
+        self,
+        instrument_strategy: instrumentStrategy,
+        optimal_positions_as_df: pd.DataFrame,
     ):
         filename = self._filename_given_instrument_strategy(instrument_strategy)
-        position_df.to_csv(filename, index_label=DATE_INDEX_NAME)
+        optimal_positions_as_df.to_csv(filename, index_label=DATE_INDEX_NAME)
 
     def _filename_given_instrument_strategy(
         self, instrument_strategy: instrumentStrategy

sysdata/csv/csv_strategy_position_data.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from sysdata.production.historic_positions_TO_DEPRECATE import strategyPositionData
+from sysdata.production.historic_strategy_positions import strategyPositionData
 from syscore.fileutils import resolve_path_and_filename_for_package
 from syscore.constants import arg_not_supplied
 from syslogdiag.log_to_screen import logtoscreen
@@ -28,9 +28,10 @@ def __init__(
     def __repr__(self):
         return "csvStrategyPositionData accessing %s" % self._datapath
 
-    def write_position_df_for_instrument_strategy(
-        self, instrument_strategy: instrumentStrategy, position_df: pd.DataFrame
+    def _write_updated_position_series_for_instrument_strategy_object(
+        self, instrument_strategy: instrumentStrategy, updated_series: pd.Series
     ):
+        position_df = pd.DataFrame(updated_series)
         filename = self._filename_given_instrument_strategy(instrument_strategy)
         position_df.to_csv(filename, index_label=DATE_INDEX_NAME)
 

sysdata/data_blob.py

@@ -190,9 +190,6 @@ def _get_csv_paths_for_class(self, class_object) -> str:
         class_name = get_class_name(class_object)
         csv_data_paths = self.csv_data_paths
         if csv_data_paths is arg_not_supplied:
-            self.log.warn(
-                "No datapaths provided for .csv, will use defaults  (may break in production, should be fine in sim)"
-            )
             return arg_not_supplied
 
         datapath = csv_data_paths.get(class_name, "")

sysdata/futures/contracts.py

@@ -100,22 +100,25 @@ def add_contract_data(
     def get_list_of_contract_dates_for_instrument_code(
         self, instrument_code: str, allow_expired: bool = False
     ) -> listOfContractDateStr:
-        raise NotImplementedError(USE_CHILD_CLASS_ERROR)
+        raise NotImplementedError
 
     def get_all_contract_objects_for_instrument_code(
         self, instrument_code: str
     ) -> listOfFuturesContracts:
-        raise NotImplementedError(USE_CHILD_CLASS_ERROR)
+        raise NotImplementedError
 
     def _get_contract_data_without_checking(
         self, instrument_code: str, contract_date: str
     ) -> futuresContract:
-        raise NotImplementedError(USE_CHILD_CLASS_ERROR)
+        raise NotImplementedError
 
     def _delete_contract_data_without_any_warning_be_careful(
         self, instrument_code: str, contract_date: str
     ):
-        raise NotImplementedError(USE_CHILD_CLASS_ERROR)
+        raise NotImplementedError
+
+    def get_list_of_all_instruments_with_contracts(self) -> list:
+        raise NotImplementedError
 
     def is_contract_in_data(self, instrument_code: str, contract_date_str: str) -> bool:
         raise NotImplementedError

sysdata/mongodb/mongo_futures_contracts.py

@@ -43,9 +43,21 @@ def is_contract_in_data(self, instrument_code: str, contract_date_str: str) -> b
         key = contract_key_from_code_and_id(instrument_code, contract_date_str)
         return self.mongo_data.key_is_in_data(key)
 
-    def get_list_of_all_contract_keys(self) -> list:
+    def _get_list_of_all_contract_keys(self) -> list:
         return self.mongo_data.get_list_of_keys()
 
+    def get_list_of_all_instruments_with_contracts(self) -> list:
+        list_of_keys = self._get_list_of_all_contract_keys()
+        list_of_split_keys = [
+            get_code_and_id_from_contract_key(key) for key in list_of_keys
+        ]
+        list_of_instruments = [
+            instrument_code for instrument_code, _ in list_of_split_keys
+        ]
+        unique_list_of_instruments = list(set(list_of_instruments))
+
+        return unique_list_of_instruments
+
     def get_all_contract_objects_for_instrument_code(
         self, instrument_code: str
     ) -> listOfFuturesContracts:
@@ -60,7 +72,7 @@ def get_all_contract_objects_for_instrument_code(
         return list_of_futures_contracts
 
     def _get_all_contract_keys_for_instrument_code(self, instrument_code: str) -> list:
-        list_of_all_contract_keys = self.get_list_of_all_contract_keys()
+        list_of_all_contract_keys = self._get_list_of_all_contract_keys()
         list_of_relevant_keys = [
             contract_key
             for contract_key in list_of_all_contract_keys