Custom scan batching (#15)

Expose modbus `scan_batching` setting as a configurable field in the YAML. This devices how many contiguous registers are scanned in one read operation. The default of 100 remains the same.

Author: tjhowse

README.md

@@ -37,6 +37,7 @@ port: 502
 update_rate: 5
 address_offset: 0
 variant: sungrow
+scan_batching: 100
 ```
 
 `ip` (Required) The IP address of the modbus device to be polled. Presently only modbus TCP/IP is supported.
@@ -45,10 +46,12 @@ variant: sungrow
 
 `update_rate` (Optional: default 5) The number of seconds between polls of the modbus device.
 
-`address_offset` (Optional: default 0) This offset is applied to every register address to accomodate different Modbus addressing systems. In many Modbus devices the first register is enumerated as 1, other times 0. See section 4.4 of the Modbus spec.
+`address_offset` (Optional: default 0) This offset is applied to every register address to accommodate different Modbus addressing systems. In many Modbus devices the first register is enumerated as 1, other times 0. See section 4.4 of the Modbus spec.
 
 `variant` (Optional) Allows variants of the ModbusTcpClient library to be used. Setting this to 'sungrow' enables support of SungrowModbusTcpClient. This library transparently decrypts the modbus comms with sungrow SH inverters running newer firmware versions.
 
+`scan_batching` (Optional: default 100) Modbus read operations are more efficient in bigger batches of contiguous registers, but different devices have different limits on the size of the batched reads. This setting can also be helpful when building a modbus register map for an uncharted device. In some modbus devices a single invalid register in a read range will fail the entire read operation. By setting `scan_batching` to `1` each register will be scanned individually. This will be very inefficient and should not be used in production as it will saturate the link with many read operations.
+
 ```yaml
 registers:
   - pub_topic: "forced_charge/mode"

modbus4mqtt/Sungrow_SH5k_20.yaml

@@ -3,6 +3,7 @@ port: 502
 update_rate: 5
 address_offset: 0
 variant: sungrow
+scan_batching: 100
 registers:
   - pub_topic: "no_export/partial/limit"
     set_topic: "no_export/partial/limit/set"

modbus4mqtt/modbus4mqtt.py

@@ -33,7 +33,8 @@ def connect_modbus(self):
         self._mb = modbus_interface.modbus_interface(self.config['ip'],
                                                      self.config.get('port', 502),
                                                      self.config.get('update_rate', 5),
-                                                     variant=self.config.get('variant', None))
+                                                     variant=self.config.get('variant', None),
+                                                     scan_batching=self.config.get('scan_batching', None))
         failed_attempts = 1
         while self._mb.connect():
             logging.warning("Modbus connection attempt {} failed. Retrying...".format(failed_attempts))

modbus4mqtt/modbus_interface.py

@@ -9,10 +9,11 @@
 DEFAULT_SCAN_BATCHING = 100
 DEFAULT_WRITE_BLOCK_INTERVAL_S = 0.2
 DEFAULT_WRITE_SLEEP_S = 0.05
+DEFAULT_READ_SLEEP_S = 0.05
 
 class modbus_interface():
 
-    def __init__(self, ip, port=502, update_rate_s=DEFAULT_SCAN_RATE_S, variant=None):
+    def __init__(self, ip, port=502, update_rate_s=DEFAULT_SCAN_RATE_S, variant=None, scan_batching=None):
         self._ip = ip
         self._port = port
         # This is a dict of sets. Each key represents one table of modbus registers.
@@ -25,6 +26,10 @@ def __init__(self, ip, port=502, update_rate_s=DEFAULT_SCAN_RATE_S, variant=None
         self._planned_writes = Queue()
         self._writing = False
         self._variant = variant
+        if scan_batching is None:
+            self._scan_batching = DEFAULT_SCAN_BATCHING
+        elif 1 <= scan_batching <= 100:
+            self._scan_batching = scan_batching
 
     def connect(self):
         # Connects to the modbus device
@@ -49,19 +54,21 @@ def add_monitor_register(self, table, addr):
     def poll(self):
         # Polls for the values marked as interesting in self._tables.
         for table in self._tables:
-            # This batches up modbus reads in chunks of DEFAULT_SCAN_BATCHING
+            # This batches up modbus reads in chunks of self._scan_batching
             start = -1
             for k in sorted(self._tables[table]):
-                group = int(k) - int(k) % DEFAULT_SCAN_BATCHING
+                group = int(k) - int(k) % self._scan_batching
                 if (start < group):
                     try:
-                        values = self._scan_value_range(table, group, DEFAULT_SCAN_BATCHING)
-                        for x in range(0, DEFAULT_SCAN_BATCHING):
+                        values = self._scan_value_range(table, group, self._scan_batching)
+                        for x in range(0, self._scan_batching):
                             key = group + x
                             self._values[table][key] = values[x]
+                        # Avoid back-to-back read operations that could overwhelm some modbus devices.
+                        sleep(DEFAULT_READ_SLEEP_S)
                     except ValueError as e:
                         logging.exception("{}".format(e))
-                    start = group + DEFAULT_SCAN_BATCHING-1
+                    start = group + self._scan_batching-1
         self._process_writes()
 
     def get_value(self, table, addr):

requirements.txt

@@ -3,4 +3,4 @@ pyyaml
 click
 paho-mqtt
 pymodbus
-SungrowModbusTcpClient>=0.1.2
+SungrowModbusTcpClient>=0.1.5

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="modbus4mqtt",
-    version="0.3.1",
+    version="0.3.2",
     author="Travis Howse",
     author_email="tjhowse@gmail.com",
     description="A YAML-defined bidirectional Modbus to MQTT interface",
@@ -18,7 +18,7 @@
         'paho-mqtt>=1.5.0',
         'pymodbus>=2.3.0',
         'click>=6.7',
-        'SungrowModbusTcpClient>=0.1.2',
+        'SungrowModbusTcpClient>=0.1.5',
     ],
     tests_require=[
         'nose2>=0.9.2',

tests/test_modbus.py

@@ -166,3 +166,32 @@ def test_masked_writes(self):
 
             m.set_value('holding', 1, 0x0000, 0x0F00)
             self.assertEqual(self.holding_registers.registers[1], 0xF0FF)
+
+    def test_scan_batching_of_one(self):
+        with patch('modbus4mqtt.modbus_interface.ModbusTcpClient') as mock_modbus:
+            mock_modbus().connect.side_effect = self.connect_success
+            mock_modbus().read_input_registers.side_effect = self.read_input_registers
+            mock_modbus().read_holding_registers.side_effect = self.read_holding_registers
+
+            m = modbus_interface.modbus_interface('1.1.1.1', 111, 2, scan_batching=1)
+            m.connect()
+            mock_modbus.assert_called_with('1.1.1.1', 111, RetryOnEmpty=True, framer=modbus_interface.ModbusSocketFramer, retries=1, timeout=1)
+
+            # Confirm registers are added to the correct tables.
+            m.add_monitor_register('holding', 5)
+            m.add_monitor_register('holding', 6)
+            m.add_monitor_register('input', 6)
+            m.add_monitor_register('input', 7)
+
+            m.poll()
+
+            self.assertEqual(m.get_value('holding', 5), 5)
+            self.assertEqual(m.get_value('holding', 6), 6)
+            self.assertEqual(m.get_value('input', 6), 6)
+            self.assertEqual(m.get_value('input', 7), 7)
+
+            # Ensure each register is scanned with a separate read call.
+            mock_modbus().read_holding_registers.assert_any_call(5, 1, unit=1)
+            mock_modbus().read_holding_registers.assert_any_call(6, 1, unit=1)
+            mock_modbus().read_input_registers.assert_any_call(6, 1, unit=1)
+            mock_modbus().read_input_registers.assert_any_call(7, 1, unit=1)