Merge remote-tracking branch 'robcarver17/master' into missing-data

Author: todd

CHANGELOG.md

@@ -260,7 +260,7 @@ Note that the large contract prices will continue to be updated (best to keep do
 
 ## Version 0.23.0
 
-- 'get_filename_for_package' can now take absolute as well as relative paths, and can cope with separate file names
+- 'resolve_path_and_filename_for_package' can now take absolute as well as relative paths, and can cope with separate file names
 - Updated legacy .csv files
 - Fixed a few bugs
 - Can now get unexpired contracts for a given instrument using 'contractDateWithRollParameters.get_unexpired_contracts_from_now_to_contract_date()'

CONTRIBUTING.md

@@ -56,9 +56,9 @@ In general, we try and follow the original texts: [PEP 8](https://peps.python.or
 
 ### General
 
-- Unless there is a single parameter, parameters should be explicit.
-- Type hints should be used, with Unions if required `from typing import Union`
-
+- Unless there is a single parameter, passed parameters should be explicit.
+- Type hints should be used, with Unions if required `from typing import Union` and Lists / Dicts ...`from typing import List, Dict`
+- Verbose doc strings specifying all the parameters are no longer required (superseded by type hints)
 
 ### Naming conventions
 

docs/backtesting.md

@@ -1436,17 +1436,17 @@ You can also save a config object into a yaml file:
 ```python
 from systems.provided.futures_chapter15.basesystem import futures_system
 import yaml
-from syscore.fileutils import get_filename_for_package
+from syscore.fileutils import resolve_path_and_filename_for_package
 
-system=futures_system()
-my_config=system.config
+system = futures_system()
+my_config = system.config
 
 ## make some changes to my_config here
 
-filename=get_filename_for_package("private.this_system_name.config.yaml")
+filename = resolve_path_and_filename_for_package("private.this_system_name.config.yaml")
 
 with open(filename, 'w') as outfile:
-    outfile.write( yaml.dump(my_config, default_flow_style=True) )
+   outfile.write(yaml.dump(my_config, default_flow_style=True))
 ```
 
 This is useful if you've been playing with a backtest configuration, and want
@@ -3794,24 +3794,24 @@ There are a number of different ways one might want to specify path and file nam
 For convenience I have written some functions that translate betweeen these different formats, and the underlying OS representation.
 
 ```python
-from syscore.fileutils import get_resolved_pathname, get_filename_for_package
+from syscore.fileutils import get_resolved_pathname, resolve_path_and_filename_for_package
 
 # Resolve both filename and pathname jointly. Useful when writing the name of eg a configuration file
 ## Absolute format
 ### Windows (note use of double backslash in str) Make sure you include the initial backslash, or will be treated as relative format
-get_filename_for_package("\\home\\rob\\file.csv")
+resolve_path_and_filename_for_package("\\home\\rob\\file.csv")
 
 ### Unix. Make sure you include the initial forward slash,
-get_filename_for_package("/home/rob/file.csv")
+resolve_path_and_filename_for_package("/home/rob/file.csv")
 
 ## Relative format to find a file in the installed pysystemtrade
 ### Dot format. Notice there is no initial 'dot' and we don't need to include 'pysystemtrade'
-get_filename_for_package("syscore.tests.pricedata.csv")
+resolve_path_and_filename_for_package("syscore.tests.pricedata.csv")
 
 # Specify the path and filename separately
-get_filename_for_package("\\home\\rob","file.csv")
-get_filename_for_package("/home/rob","file.csv")
-get_filename_for_package("syscore.tests","pricedata.csv")
+resolve_path_and_filename_for_package("\\home\\rob", "file.csv")
+resolve_path_and_filename_for_package("/home/rob", "file.csv")
+resolve_path_and_filename_for_package("syscore.tests", "pricedata.csv")
 
 # Resolve just the pathname
 get_resolved_pathname("/home/rob")
@@ -3821,13 +3821,13 @@ get_resolved_pathname("syscore.tests")
 ## DON'T USE THESE:-
 ### It's possible to use Unix or Windows for relative filenames, but I prefer not to, so there is a clearer disctinction between absolute and relative.
 ### However this works:
-get_filename_for_package("syscore/tests/pricedata.csv")
+resolve_path_and_filename_for_package("syscore/tests/pricedata.csv")
 
 ### Similarly, I prefer not to use dot format for absolute filenames but it will work
-get_filename_for_package(".home.rob.file.csv")
+resolve_path_and_filename_for_package(".home.rob.file.csv")
 
 ### Finally, You can mix and match the above formats in a single string, but it won't make the code very readable!
-get_filename_for_package("\\home/rob.file.csv")
+resolve_path_and_filename_for_package("\\home/rob.file.csv")
 
 ```
 

docs/doc_generation/list_of_futures_product_pages_generator.py

@@ -1,11 +1,11 @@
 from pathlib import Path
-from syscore.fileutils import get_filename_for_package
+from syscore.fileutils import resolve_path_and_filename_for_package
 from docs.doc_generation.symbol_product_page_map import symbol_product_page_map
 
 # GET ABSOLUTE PATH TO THE DOCS MODULE
 file_in_module_where_md_file_is_to_be_stored = "docs.__init__.py"
-full_path = get_filename_for_package(
-    pathname=file_in_module_where_md_file_is_to_be_stored
+full_path = resolve_path_and_filename_for_package(
+    path_and_filename=file_in_module_where_md_file_is_to_be_stored
 )
 write_path = Path(full_path).parent
 

docs/instruments.md

@@ -311,6 +311,11 @@ system.get_list_of_bad_markets()
 ```
 
 
+### Automatically excluded
+
+It's also possible that there are some instruments that have zero positions. The most likely explanation for this is that you have set a speed limit on trading costs, and there are no trading rules that are cheap enough to trade the given instrument. These are automatically added to the list of markets given a zero weight for optimisation. 
+
+
 ## Customising the list of 'all instruments' and 'excluded for optimisation'
 
 If you make two calls to system *before you do anything else with a system* you can decide exactly what is, or is not, included in the instrument lists. The following calls will reproduce the default system behaviour, but you can modify them if desired. IMPORTANT: they must be called in this order if you want to change the instrument_list() call.
@@ -337,7 +342,7 @@ system.get_list_of_markets_not_trading_but_with_data(
 ```
 
 
-# Operating in production 
+# Operating in production environment
 
 Operating in the production environment is a bit more complex, due to the interaction of configuration files, the way that constraints operate, and the possibility of pulling in additional constraints from a database.
 

sysbrokers/IB/client/ib_price_client.py

@@ -12,7 +12,7 @@
 from syscore.exceptions import missingContract, missingData
 
 from syscore.dateutils import (
-    adjust_timestamp_to_include_notional_close_and_time_offset,
+    replace_midnight_with_notional_closing_time,
     strip_timezone_fromdatetime,
     Frequency,
     DAILY_PRICE_FREQ,
@@ -220,15 +220,12 @@ def _ib_timestamp_to_datetime(self, timestamp_ib) -> datetime.datetime:
         and adjusts yyyymm to closing vector
 
         :param timestamp_str: datetime.datetime
-        :return: pd.datetime
+        :return: datetime.datetime
         """
 
-        local_timestamp_ib = self._adjust_ib_time_to_local(timestamp_ib)
-        timestamp = pd.to_datetime(local_timestamp_ib)
+        timestamp = self._adjust_ib_time_to_local(timestamp_ib)
 
-        adjusted_ts = adjust_timestamp_to_include_notional_close_and_time_offset(
-            timestamp
-        )
+        adjusted_ts = replace_midnight_with_notional_closing_time(timestamp)
 
         return adjusted_ts
 

sysbrokers/IB/ib_Fx_prices_data.py

@@ -7,10 +7,12 @@
 
 from sysobjects.spot_fx_prices import fxPrices
 from syslogdiag.log_to_screen import logtoscreen
-from syscore.fileutils import get_filename_for_package
+from syscore.fileutils import resolve_path_and_filename_for_package
 from syscore.objects import missing_instrument, missing_file
 
-IB_CCY_CONFIG_FILE = get_filename_for_package("sysbrokers.IB.ib_config_spot_FX.csv")
+IB_CCY_CONFIG_FILE = resolve_path_and_filename_for_package(
+    "sysbrokers.IB.ib_config_spot_FX.csv"
+)
 
 ibFXConfig = namedtuple("ibFXConfig", ["ccy1", "ccy2", "invert"])
 

sysbrokers/IB/ib_instruments_data.py

@@ -8,15 +8,17 @@
 from sysbrokers.IB.ib_connection import connectionIB
 from sysbrokers.broker_instrument_data import brokerFuturesInstrumentData
 
-from syscore.fileutils import get_filename_for_package
-from syscore.genutils import value_or_npnan
+from syscore.fileutils import resolve_path_and_filename_for_package
+from syscore.genutils import return_another_value_if_nan
 from syscore.objects import missing_instrument, missing_file
 
 from sysobjects.instruments import futuresInstrument
 
 from syslogdiag.log_to_screen import logtoscreen
 
-IB_FUTURES_CONFIG_FILE = get_filename_for_package("sysbrokers.IB.ib_config_futures.csv")
+IB_FUTURES_CONFIG_FILE = resolve_path_and_filename_for_package(
+    "sysbrokers.IB.ib_config_futures.csv"
+)
 
 
 class IBconfig(pd.DataFrame):
@@ -171,11 +173,11 @@ def get_instrument_object_from_config(
     config_row = config[config.Instrument == instrument_code]
     symbol = config_row.IBSymbol.values[0]
     exchange = config_row.IBExchange.values[0]
-    currency = value_or_npnan(config_row.IBCurrency.values[0], NOT_REQUIRED_FOR_IB)
-    ib_multiplier = value_or_npnan(
+    currency = return_another_value_if_nan(config_row.IBCurrency.values[0], NOT_REQUIRED_FOR_IB)
+    ib_multiplier = return_another_value_if_nan(
         config_row.IBMultiplier.values[0], NOT_REQUIRED_FOR_IB
     )
-    price_magnifier = value_or_npnan(config_row.priceMagnifier.values[0], 1.0)
+    price_magnifier = return_another_value_if_nan(config_row.priceMagnifier.values[0], 1.0)
     ignore_weekly = config_row.IgnoreWeekly.values[0]
 
     # We use the flexibility of futuresInstrument to add additional arguments

sysbrokers/IB/ib_trading_hours.py

@@ -6,7 +6,7 @@
     tradingHours,
     listOfTradingHours,
 )
-from syscore.fileutils import does_file_exist
+from syscore.fileutils import does_filename_exist
 from sysdata.config.production_config import get_production_config
 from sysdata.production.trading_hours import read_trading_hours
 
@@ -17,7 +17,7 @@
 
 
 def get_saved_trading_hours():
-    if does_file_exist(PRIVATE_CONFIG_TRADING_HOURS_FILE):
+    if does_filename_exist(PRIVATE_CONFIG_TRADING_HOURS_FILE):
         return read_trading_hours(PRIVATE_CONFIG_TRADING_HOURS_FILE)
     else:
         return read_trading_hours(IB_CONFIG_TRADING_HOURS_FILE)

syscontrol/monitor.py

@@ -3,10 +3,10 @@
 import datetime
 from sysdata.data_blob import dataBlob
 
-from syscore.fileutils import get_filename_for_package
+from syscore.fileutils import resolve_path_and_filename_for_package
 from sysproduction.data.control_process import dataControlProcess
 from sysproduction.data.control_process import diagControlProcess
-from syscore.dateutils import last_run_or_heartbeat_from_date_or_none
+from syscore.dateutils import date_as_short_pattern_or_question_if_missing
 from syscontrol.list_running_pids import describe_trading_server_login_data
 
 
@@ -83,7 +83,7 @@ def send_update_message(self, process_name, current_status, new_status):
             process_name,
             current_status,
             new_status,
-            last_run_or_heartbeat_from_date_or_none(datetime.datetime.now()),
+            date_as_short_pattern_or_question_if_missing(datetime.datetime.now()),
         )
         self.log_messages.append_msg(msg)
 
@@ -123,7 +123,7 @@ def check_if_pid_running_and_if_not_finish(process_observatory: processMonitor):
 
 
 def generate_html(process_observatory: processMonitor):
-    resolved_filename = get_filename_for_package(filename)
+    resolved_filename = resolve_path_and_filename_for_package(filename)
     trading_server_description = describe_trading_server_login_data()
     dbase_description = str(process_observatory.data.mongo_db)
     with open(resolved_filename, "w") as file: