Merge pull request #49 from ISISComputingGroup/Ticket6511_failing_to_get_blockserver_data

Ticket 6511: Try to get config information from channel access first

Author: JamesKingWork

block_utils.py

@@ -5,6 +5,7 @@
 
 logger = logging.getLogger('JSON_bourne')
 
+
 def shorten_title(title):
     """
     Gets a PV title by shortening its address to the last segment.

external_webpage/data_source_reader.py

@@ -21,6 +21,8 @@
 
 import logging
 import requests
+from external_webpage.utils import dehex_and_decompress
+from CaChannel.util import caget
 
 logger = logging.getLogger('JSON_bourne')
 
@@ -31,6 +33,8 @@
 # Port for configuration
 PORT_CONFIG = 8008
 
+CONFIG_PV = "CS:BLOCKSERVER:GET_CURR_CONFIG_DETAILS"
+
 # Timeout for url get
 URL_GET_TIMEOUT = 60
 
@@ -40,13 +44,14 @@ class DataSourceReader(object):
     Access of external data sources from urls.
     """
 
-    def __init__(self, host):
+    def __init__(self, host, pv_prefix):
         """
         Initialize.
         Args:
             host: The host name for the instrument.
         """
         self._host = host
+        self._pv_prefix = pv_prefix
 
     def get_json_from_blocks_archive(self):
         """
@@ -97,13 +102,20 @@ def _get_json_from_info_page(self, port, group_name):
 
     def read_config(self):
         """
-        Read the configuration from the instrument block server.
+        Read the configuration from the instrument block server. First using channel access then falling back to the
+        blockserver webserver.
 
         Returns: The configuration as a dictionary.
-
         """
+        try:
+            pv = self._pv_prefix + CONFIG_PV
+            raw = caget(pv, as_string=True)
+            config_details = dehex_and_decompress(raw)
+            config_details = json.loads(config_details)
+            return config_details
+        except Exception as ex:
+            logger.error(f"Error getting instrument config details from {pv}, using webserver instead. {ex}")
 
-        # read config
         page = requests.get('http://{}:{}/'.format(self._host, PORT_CONFIG), timeout=URL_GET_TIMEOUT)
         content = page.content.decode("utf-8")
         corrected_page = content.replace("'", '"')\

external_webpage/instrument_information_collator.py

@@ -107,15 +107,16 @@ class InstrumentInformationCollator(object):
     # name of the channel fo the run duration for the current period
     RUN_DURATION_PD_CHANNEL_NAME = "RUNDURATION_PD"
 
-    def __init__(self, host="localhost", reader=None):
+    def __init__(self, host, pv_prefix, reader=None):
         """
         Initialize.
         Args:
             host: The host of the instrument from which to read the information.
+            pv_prefix: The pv_prefix of the instrument from which to read the information.
             reader: A reader object to get external information.
         """
         if reader is None:
-            self.reader = DataSourceReader(host)
+            self.reader = DataSourceReader(host, pv_prefix)
         else:
             self.reader = reader
 

external_webpage/instrument_scapper.py

@@ -37,15 +37,17 @@ def wait(self, seconds):
                 return
             sleep(1)
 
-    def __init__(self, name, host):
+    def __init__(self, name, host, pv_prefix):
         """
         Initialize.
         Args:
             name: Name of instrument.
             host: Host for the instrument.
+            pv_prefix: The pv_prefix of the instrument.
         """
         super(InstrumentScrapper, self).__init__()
         self._host = host
+        self._pv_prefix = pv_prefix
         self._name = name
         self._stop_event = Event()
 
@@ -68,7 +70,7 @@ def run(self):
 
         """
         global scraped_data
-        web_page_scraper = InstrumentInformationCollator(self._host)
+        web_page_scraper = InstrumentInformationCollator(self._host, self._pv_prefix)
         logger.info("Scrapper started for {}".format(self._name))
         while not self._stop_event.is_set():
             try:

external_webpage/utils.py

@@ -0,0 +1,19 @@
+import zlib
+
+
+def dehex_and_decompress(value):
+    """
+    Decompress and dehex pv value
+    Args:
+        value: value to translate
+
+    Returns: dehexed value
+
+    """
+    try:
+        # If it comes as bytes then cast to string
+        value = value.decode("utf-8")
+    except AttributeError:
+        pass
+
+    return zlib.decompress(bytes.fromhex(value)).decode("utf-8")

external_webpage/web_scrapper_manager.py

@@ -6,18 +6,17 @@
 from builtins import object
 import json
 import logging
-import zlib
 from threading import Thread, Event
 from time import sleep
 
-import six
 from CaChannel import CaChannelException
 from CaChannel.util import caget
 
 from external_webpage.instrument_scapper import InstrumentScrapper
+from external_webpage.utils import dehex_and_decompress
 
 # logger for the class
-logger = logging.getLogger('JSON_bourne')
+logger = logging.getLogger("JSON_bourne")
 
 # PV that contains the instrument list
 INST_LIST_PV = "CS:INSTLIST"
@@ -30,6 +29,7 @@ class InstList(object):
     """
     Object that allows the instrument list to be requested from a PV
     """
+
     INSTRUMENT_LIST_CAN_NOT_BE_READ = "Instrument list can not be read"
     INSTRUMENT_LIST_NOT_DECOMPRESSED = "Instrument list can not decompressed"
     INSTRUMENT_LIST_NOT_JSON = "Instrument list is not json"
@@ -68,7 +68,7 @@ def retrieve(self):
             return self._cached_list
 
         try:
-            full_inst_list_string = self._dehex_and_decompress(raw)
+            full_inst_list_string = dehex_and_decompress(raw)
         except Exception as ex:
 
             self.error_on_retrieve = InstList.INSTRUMENT_LIST_NOT_DECOMPRESSED
@@ -85,7 +85,7 @@ def retrieve(self):
 
         try:
             for full_inst in full_inst_list:
-                inst_list[full_inst["name"]] = full_inst["hostName"]
+                inst_list[full_inst["name"]] = (full_inst["hostName"], full_inst["pvPrefix"])
         except (KeyError, TypeError) as ex:
 
             self.error_on_retrieve = InstList.INSTRUMENT_LIST_NOT_CORRECT_FORMAT
@@ -98,34 +98,16 @@ def retrieve(self):
 
         return self._cached_list
 
-    def _dehex_and_decompress(self, value):
-        """
-        Decompress and dehex pv value
-        Args:
-            value: value to translate
-
-        Returns: dehexed value
-
-        """
-        if six.PY2:
-            return zlib.decompress(value.decode('hex'))
-
-        try:
-            # If it comes as bytes then cast to string
-            value = value.decode('utf-8')
-        except AttributeError:
-            pass
-
-        return zlib.decompress(bytes.fromhex(value)).decode("utf-8")
-
 
 class WebScrapperManager(Thread):
     """
     Manager for the web scrappers that are creating the data for the data web
     It is responsible for starting then and making sure they are running
     """
 
-    def __init__(self, scrapper_class=InstrumentScrapper, inst_list=None, local_inst_list=None):
+    def __init__(
+        self, scrapper_class=InstrumentScrapper, inst_list=None, local_inst_list=None
+    ):
         """
         Initialiser.
         Args:
@@ -173,27 +155,30 @@ def maintain_scrapper_list(self):
         inst_list = self._inst_list.retrieve()
         new_scrappers_list = []
         for scrapper in self.scrappers:
-            if scrapper.is_alive() and self._is_scrapper_in_inst_list(inst_list, scrapper):
+            if scrapper.is_alive() and self._is_scrapper_in_inst_list(
+                inst_list, scrapper
+            ):
                 new_scrappers_list.append(scrapper)
             else:
                 scrapper.stop()
         self.scrappers = new_scrappers_list
-        for name, host in self._scrapper_to_start(inst_list):
-            scrapper = self._scrapper_class(name, host)
+        for name, host, pv_prefix in self._scrapper_to_start(inst_list):
+            scrapper = self._scrapper_class(name, host, pv_prefix)
             scrapper.start()
             self.scrappers.append(scrapper)
 
     def _is_scrapper_in_inst_list(self, inst_list, scrapper):
         """
-        Check if scapper is in instrument list
+        Check if scrapper is in instrument list
         Args:
             inst_list: the instrument list
             scrapper: scrapper to checker
 
         Returns: True if in; False otherwise
 
         """
-        for name, host in list(inst_list.items()):
+        for name, data in inst_list.items():
+            host, pv_prefix = data
             if scrapper.is_instrument(name, host):
                 return True
         return False
@@ -207,12 +192,13 @@ def _scrapper_to_start(self, instruments):
         Returns:
 
         """
-        for name, host in list(instruments.items()):
+        for name, data in instruments.items():
+            host, pv_prefix = data
             for scrapper in self.scrappers:
                 if scrapper.is_instrument(name, host):
                     break
             else:
-                yield name, host
+                yield name, host, pv_prefix
 
     def stop_all(self):
         """

tests/test_data_source_reader.py

@@ -2,6 +2,8 @@
 from mock import MagicMock, patch
 from external_webpage.data_source_reader import DataSourceReader
 from hamcrest import *
+import zlib
+import binascii
 
 
 def patch_page_contents(request_response, json):
@@ -10,38 +12,67 @@ def patch_page_contents(request_response, json):
     request_response.return_value = page
 
 
+def compress_and_hex(value):
+    """Compresses the inputted string and encodes it as hex.
+
+    Args:
+        value (str): The string to be compressed
+    Returns:
+        bytes : A compressed and hexed version of the inputted string
+    """
+    assert type(value) == str, \
+        "Non-str argument passed to compress_and_hex, maybe Python 2/3 compatibility issue\n" \
+        "Argument was type {} with value {}".format(value.__class__.__name__, value)
+    compr = zlib.compress(bytes(value, "utf-8"))
+    return binascii.hexlify(compr)
+
+
 class TestDataSourceReader(unittest.TestCase):
     def setUp(self):
-        self.reader = DataSourceReader("HOST")
+        self.reader = DataSourceReader("HOST", "PREFIX")
 
     @patch("requests.get")
-    def test_GIVEN_JSON_with_single_quotes_WHEN_read_THEN_conversion_successful(self, request_response):
+    @patch("external_webpage.data_source_reader.caget")
+    def test_GIVEN_JSON_with_single_quotes_WHEN_read_THEN_conversion_successful(self, caget, request_response):
         patch_page_contents(request_response, b"{'data': 'some_data'}")
 
         json_object = self.reader.read_config()
 
         assert_that(json_object, is_({"data": "some_data"}))
 
     @patch("requests.get")
-    def test_GIVEN_JSON_with_None_WHEN_read_THEN_conversion_successful(self, request_response):
+    @patch("external_webpage.data_source_reader.caget")
+    def test_GIVEN_JSON_with_None_WHEN_read_THEN_conversion_successful(self, caget, request_response):
         patch_page_contents(request_response, b'{"data": None}')
 
         json_object = self.reader.read_config()
 
         assert_that(json_object, is_({"data": None}))
 
     @patch("requests.get")
-    def test_GIVEN_JSON_with_True_WHEN_read_THEN_conversion_successful(self, request_response):
+    @patch("external_webpage.data_source_reader.caget")
+    def test_GIVEN_JSON_with_True_WHEN_read_THEN_conversion_successful(self, caget, request_response):
         patch_page_contents(request_response, b'{"data": True}')
 
         json_object = self.reader.read_config()
 
         assert_that(json_object, is_({"data": True}))
 
     @patch("requests.get")
-    def test_GIVEN_JSON_with_False_WHEN_read_THEN_conversion_successful(self, request_response):
+    @patch("external_webpage.data_source_reader.caget")
+    def test_GIVEN_JSON_with_False_WHEN_read_THEN_conversion_successful(self, caget, request_response):
         patch_page_contents(request_response, b'{"data": False}')
 
         json_object = self.reader.read_config()
 
         assert_that(json_object, is_({"data": False}))
+
+    @patch("requests.get")
+    @patch("external_webpage.data_source_reader.caget")
+    def test_GIVEN_valid_config_from_caget_WHEN_read_THEN_webserver_is_not_tried(self, caget, request_response):
+        caget.return_value = compress_and_hex('{"data": false}')
+
+        json_object = self.reader.read_config()
+
+        assert_that(json_object, is_({"data": False}))
+        request_response.assert_not_called()

tests/test_get_info_from_config_and_web.py

@@ -22,7 +22,7 @@ def setUp(self):
         config = ConfigMother.create_config()
         self.reader.read_config = Mock(return_value=config)
 
-        self.scraper = InstrumentInformationCollator(reader=self.reader)
+        self.scraper = InstrumentInformationCollator("host", "prefix", reader=self.reader)
 
     def test_GIVEN_no_blocks_WHEN_parse_THEN_normal_value_returned(self):
         expected_config_name = "test_config"