updated for latest

Author: tmori

bindings/python/hako_binary/binary_io.py

@@ -40,9 +40,20 @@ def binTofloat32(binary):
 def binTofloat64(binary):
     return struct.unpack('d', binary)[0]
 
-def binTostring(binary):
-    return binary.decode().replace('\0', '')
-    #return "Not Supported.."
+def binTostring(binary, max_len=128):
+    try:
+        sub = binary[:max_len]
+        end = sub.find(b'\0')
+        if end == -1:
+            end = max_len
+        return sub[:end].decode('utf-8')
+    except UnicodeDecodeError as e:
+        print("UnicodeDecodeError:")
+        print(f"  ERROR: {e}")
+        print(f"  RAW DATA: {binary}")
+        print(f"  HEXDUMP: {' '.join(f'{b:02x}' for b in binary)}")
+        raise e
+
 
 
 def int8Tobin(arg):
@@ -163,6 +174,46 @@ def binToArrayValues(type, arg):
     else:
         return None
 
+def typeTobin_array(type, values, elm_size=None):
+    count = len(values)
+    if type == "int8":
+        return struct.pack(f'<{count}b', *values)
+    elif type == "uint8":
+        return struct.pack(f'<{count}B', *values)
+    elif type == "int16":
+        return struct.pack(f'<{count}h', *values)
+    elif type == "uint16":
+        return struct.pack(f'<{count}H', *values)
+    elif type == "int32":
+        return struct.pack(f'<{count}i', *values)
+    elif type == "bool":
+        return struct.pack(f'<{count}i', *values)
+    elif type == "uint32":
+        return struct.pack(f'<{count}I', *values)
+    elif type == "int64":
+        return struct.pack(f'<{count}q', *values)
+    elif type == "uint64":
+        return struct.pack(f'<{count}Q', *values)
+    elif type == "float32":
+        return struct.pack(f'<{count}f', *values)
+    elif type == "float64":
+        return struct.pack(f'<{count}d', *values)
+    elif type == "string":
+        if elm_size is None:
+            raise ValueError("elm_size required for string array")
+        binaries = []
+        for val in values:
+            raw = val.encode('utf-8') + b'\x00'
+            if len(raw) > elm_size:
+                raw = raw[:elm_size]
+            buffer = bytearray(elm_size)
+            buffer[:len(raw)] = raw
+            binaries.append(buffer)
+        return b''.join(binaries)
+    else:
+        raise ValueError(f"typeTobin_array: Unsupported type {type}")
+
+
 def writeBinary(binary_data, off, bin):
     i = 0
     for data in bin:

bindings/python/hako_binary/binary_reader.py

@@ -4,8 +4,8 @@
 import sys
 import base64
 
-from hako_binary import binary_io
-from hako_binary import offset_parser
+from . import binary_io
+from . import offset_parser
 
 def decode_base64(data):
     return base64.b64decode(data)
@@ -31,9 +31,11 @@ def binary_read_recursive(meta: binary_io.PduMetaData, offmap, binary_data, json
         type = offset_parser.member_type(line)
         name = offset_parser.member_name(line)
         size = offset_parser.member_size(line)
+        #print(f"binary_read_recursive: {name} off={off} size={size} type={type}")
         if (offset_parser.is_primitive(line)):
             if (offset_parser.is_single(line)):
                 bin = binary_io.readBinary(binary_data, off, size)
+                #print(f"binary_read_recursive: {name} off={off} size={size} type={type} bin={bin}")
                 value = binary_io.binTovalue(type, bin)
                 json_data[name] = value
             elif (offset_parser.is_array(line)):
@@ -44,6 +46,7 @@ def binary_read_recursive(meta: binary_io.PduMetaData, offmap, binary_data, json
                 array_size = binary_io.binTovalue("int32", binary_io.readBinary(binary_data, off, 4))
                 offset_from_heap = binary_io.binTovalue("int32", binary_io.readBinary(binary_data, off + 4, 4))
                 one_elm_size = size
+                #print(f"binary_read_recursive: {name} off={off} size={size} type={type} array_size={array_size} offset_from_heap={offset_from_heap}")
                 array_value = binary_io.readBinary(binary_data, meta.heap_off + offset_from_heap, one_elm_size * array_size)
                 json_data[name + '__raw' ] = array_value
                 json_data[name] = binary_io.binToArrayValues(type, array_value)

bindings/python/hako_binary/binary_writer.py

@@ -3,8 +3,8 @@
 import json
 import sys
 
-from hako_binary import binary_io
-from hako_binary import offset_parser
+from . import binary_io
+from . import offset_parser
 
 class DynamicAllocator:
     def __init__(self, is_heap: bool):
@@ -13,16 +13,14 @@ def __init__(self, is_heap: bool):
         self.is_heap = is_heap
 
     def add(self, bytes_data, expected_offset=None, key=None):
-        if self.is_heap == False:
-            if expected_offset is not None:
-                current_size = len(self.data)
-                if current_size < expected_offset:
-                    padding = bytearray(expected_offset - current_size)
-                    self.data.extend(padding)
-        
+        current_size = len(self.data)
+        #print(f"is_heap: {self.is_heap} current_size: {current_size} expected_offset: {expected_offset} len(bytes_data): {len(bytes_data)}")
+        if (current_size < expected_offset):
+            padding = bytearray(expected_offset - current_size)
+            self.data.extend(padding)
         offset = len(self.data)
         self.data.extend(bytes_data)
-        
+        #print(f"add: {bytes_data} offset: {offset} len(self.data): {len(self.data)}")
         if key:
             self.offset_map[key] = offset
 
@@ -84,6 +82,7 @@ def binary_write_recursive(parent_off: int, bw_container: BinaryWriterContainer,
             continue
         type = offset_parser.member_type(line)
         off = offset_parser.member_off(line)
+        #print(f"key: {key} type: {type} off: {off} line: {line}")
         if offset_parser.is_primitive(line):
             if offset_parser.is_single(line):
                 bin = binary_io.typeTobin(type, json_data[key])
@@ -94,15 +93,25 @@ def binary_write_recursive(parent_off: int, bw_container: BinaryWriterContainer,
                 elm_size = offset_parser.member_size(line)
                 array_size = offset_parser.array_size(line)
                 one_elm_size = int(elm_size / array_size)
-                for i, elm in enumerate(json_data[key]):
-                    bin = binary_io.typeTobin(type, elm)
-                    bin = get_binary(type, bin, one_elm_size)
-                    allocator.add(bin, expected_offset=(parent_off + off + i * one_elm_size))
+                #for i, elm in enumerate(json_data[key]):
+                #    bin = binary_io.typeTobin(type, elm)
+                #    bin = get_binary(type, bin, one_elm_size)
+                #    allocator.add(bin, expected_offset=(parent_off + off + i * one_elm_size))
+                binary = binary_io.typeTobin_array(type, json_data[key], one_elm_size)
+                allocator.add(binary, expected_offset=(parent_off + off))
             else:  # varray
-                for i, elm in enumerate(json_data[key]):
-                    bin = binary_io.typeTobin(type, elm)
-                    bin = get_binary(type, bin, offset_parser.member_size(line))
-                    bw_container.heap_allocator.add(bin, expected_offset=(off + i * offset_parser.member_size(line)))
+                offset_from_heap = bw_container.heap_allocator.size()
+                array_size = len(json_data[key])
+                #print(f"varray: {key} array_size: {array_size} offset_from_heap: {offset_from_heap}")
+                #for i, elm in enumerate(json_data[key]):
+                #    bin = binary_io.typeTobin(type, elm)
+                #    bin = get_binary(type, bin, offset_parser.member_size(line))
+                #    bw_container.heap_allocator.add(bin, expected_offset=0)
+                binary = binary_io.typeTobin_array(type, json_data[key], offset_parser.member_size(line))
+                bw_container.heap_allocator.add(binary, expected_offset=0)
+                a_b = array_size.to_bytes(4, byteorder='little')
+                o_b = offset_from_heap.to_bytes(4, byteorder='little')
+                allocator.add(a_b + o_b, expected_offset=parent_off + off)
         else:
             if offset_parser.is_single(line):
                 binary_write_recursive(parent_off + off, bw_container, offmap, allocator, json_data[key], type)
@@ -113,5 +122,11 @@ def binary_write_recursive(parent_off: int, bw_container: BinaryWriterContainer,
                     one_elm_size = int(elm_size / array_size)
                     binary_write_recursive((parent_off + off + i * one_elm_size), bw_container, offmap, allocator, elm, type)
             else:  # varray
+                offset_from_heap = bw_container.heap_allocator.size()
+                array_size = len(json_data[key])
                 for i, elm in enumerate(json_data[key]):
                     binary_write_recursive(0, bw_container, offmap, bw_container.heap_allocator, elm, type)
+                a_b = array_size.to_bytes(4, byteorder='little')
+                o_b = offset_from_heap.to_bytes(4, byteorder='little')
+                allocator.add(a_b + o_b, expected_offset=parent_off + off)
+

bindings/python/hako_binary/offset_map.py

@@ -3,7 +3,7 @@
 import glob
 import json
 import sys
-from hako_binary import offset_parser
+from . import offset_parser
 
 class OffsetMap:
     def __init__(self, offset_path):
@@ -17,6 +17,22 @@ def get(self, typename):
             self.map[typename] =  lines
         return self.map[typename]
 
+    def align8(self, value):
+        return ((value + 7) // 8) * 8
+
+    def get_pdu_size(self, typename):
+        lines = self.get(typename)
+        size = 0
+        last_line = lines[-1]
+        last_offset = offset_parser.member_off(last_line)
+        if (offset_parser.is_varray(last_line)):
+            last_size = 8  # len + off
+        else:
+            last_size = offset_parser.member_size(last_line)
+
+        size = self.align8(last_offset + last_size + 8)
+        return size
+
     def find_filepath(self, path, filename):
         f_array = filename.split('/')
         if (len(f_array) > 1):