[profile] support cdma display && mix mode

Support cdma display on web and doc by port && add engine align

Change-Id: I6eaee0ee11cf00316aa273a77d27e4b68f2184e6

python/PerfAI/PerfAI.doc/include/dma.py

@@ -36,7 +36,7 @@ def __init__(self, core_id, writer):
         :param core_id: the id of current core
         :param writer: the writer of Excel to write
         """
-        self.columns = ['Engine Id', 'Core Id', 'Cmd Id', 'Layer Id', 'Layer Name',
+        self.columns = ['Engine Id', 'Core Id', 'Global Idx', 'Cmd Id', 'Layer Id', 'Layer Name',
                         'Function Type', 'Function Name', 'DMA data size(B)', 'Start Cycle', 'End Cycle',
                         'Asic Cycle', 'Stall Cycle', 'DDR Bandwidth(GB/s)', 'L2M Bandwidth(GB/s)', 'Direction', 'AvgBurstLength', 'Data Type', 'Non32ByteRatio',
                         'MaskWriteRatio', 'cmd_id_dep', 'cmd_special_function', 'src_start_addr', 'dst_start_addr',
@@ -144,30 +144,39 @@ def load(self, reg_info_file, dma_layer_map):
                 self.reg_list.append(reg_dict)
         self.height = len(self.reg_list)
 
-    def add_kpi_field(self):
+    def add_kpi_field(self, is_cdma=False):
         """
         Add some indicators which are convenient for performance analysis artificially.
         :return: None
         """
         for i in range(len(self.reg_list)):
             reg_dict = self.reg_list[i]
             name_key = (int(reg_dict['cmd_type']))
-            if reg_dict['cmd_type'] == 6:
+            sys_cmd_id = 7 if is_cdma else 6
+            sys_wait_id = [4, 6] if is_cdma else [4]
+            transfer_bytes = 0
+            if reg_dict['cmd_type'] == sys_cmd_id:
                 reg_dict['Data Type'] = 'None'
                 # dma_sys do not transfer data
                 reg_dict['Direction'] = '-'
+                if reg_dict['cmd_special_function'] in sys_wait_id:
+                    self.wait_msg_total_time += reg_dict['Asic Cycle']
+            if isinstance(reg_dict['DMA data size(B)'], int) and reg_dict['DMA data size(B)'] > 0:
+                transfer_bytes = reg_dict['DMA data size(B)']
             self.dma_cycle += int(reg_dict['Asic Cycle'])
             self.stall_cycle += int(reg_dict['Stall Cycle'])
             if 'DDR' in reg_dict['Direction'] and isinstance(reg_dict['DMA data size(B)'], int):
-                self.ddr_total_datasize += reg_dict['DMA data size(B)']
-                self.ddr_total_cycle += reg_dict['Asic Cycle']
-                self.ddr_burst_length_sum += reg_dict['gmem_bl_sum']
-                self.ddr_xact_cnt += reg_dict['gmem_xact_cnt']
+                if not is_cdma or transfer_bytes:
+                    self.ddr_total_datasize += reg_dict['DMA data size(B)']
+                    self.ddr_total_cycle += reg_dict['Asic Cycle']
+                    self.ddr_burst_length_sum += reg_dict['gmem_bl_sum']
+                    self.ddr_xact_cnt += reg_dict['gmem_xact_cnt']
             elif 'L2' in reg_dict['Direction'] and isinstance(reg_dict['DMA data size(B)'], int):
-                self.l2_total_datasize += reg_dict['DMA data size(B)']
-                self.l2_total_cycle += reg_dict['Asic Cycle']
-            if reg_dict['cmd_type'] == 6 and reg_dict['cmd_special_function'] == 4:
-                self.wait_msg_total_time += reg_dict['Asic Cycle']
+                if not is_cdma or transfer_bytes:
+                    self.l2_total_datasize += reg_dict['DMA data size(B)']
+                    self.l2_total_cycle += reg_dict['Asic Cycle']
+            # if reg_dict['cmd_type'] == 6 and reg_dict['cmd_special_function'] == 4:
+            #     self.wait_msg_total_time += reg_dict['Asic Cycle']
             if reg_dict['gmem_xact_cnt'] > 0:
                 reg_dict['AvgBurstLength'] = get_ratio_float_2f(reg_dict['gmem_bl_sum'], reg_dict['gmem_xact_cnt'])
                 reg_dict['Non32ByteRatio'] = get_ratio_float_2f(reg_dict['gmem_n32Ba_sa_cnt'], reg_dict['gmem_xact_cnt'])
@@ -516,8 +525,9 @@ def load(self, reg_info_file):
                     elif reg_count == 0:
                         fields = row.split(': ')
                         attr = fields[0][1:]
-                        val = fields[1][:-1]
-                        self.chip_arch_dict[attr] = val
+                        if len(fields) > 1:
+                            val = fields[1][:-1]
+                            self.chip_arch_dict[attr] = val
                         idx = 0
                     else:
                         fields = row.split(': ')

python/PerfAI/PerfAI.doc/include/tiu.py

@@ -42,7 +42,7 @@ def __init__(self, core_id, writer):
         :param writer: the writer of Excel to write
         """
         self.writer = writer
-        self.columns = ['Engine Id', 'Core Id', 'Cmd Id', 'Layer Id', 'Layer Name', 'Function Type', 'Function Name',
+        self.columns = ['Engine Id', 'Core Id', 'Global Idx', 'Cmd Id', 'Layer Id', 'Layer Name', 'Function Type', 'Function Name',
                         'Alg Cycle', 'Asic Cycle', 'Start Cycle', 'End Cycle', 'Avg Cycle Last 200', 'Alg Ops',
                         'uArch Ops', 'uArch Rate', 'Bank Conflict Ratio',
                         'Initial Cycle Ratio', 'Data Type', 'Sim Power(W)', 'des_cmd_id_dep',

python/PerfAI/PerfAI.doc/src/generator/details.py

@@ -11,7 +11,7 @@
 # @Time    : 2023/8/7 11:26
 # @Author  : [email protected]
 # @Project: PerfAI
-import os
+import os, glob, re
 import pandas as pd
 from tqdm import tqdm
 
@@ -83,6 +83,7 @@ def generate_details(input_fold, out_file, g_info, writer, core_num=8, split_ins
     tiu_instance_map, gdma_instance_map = dict(), dict()
     tiu_layer_map, gdma_layer_map = get_engine_layer(g_info)
     chip_arch_act = None
+    file_names = sorted(glob.glob(cdma_reg_file + '_*.txt'))
     for core_id in tqdm(range(act_core_num)):
         # tiu
         cur_tiu_reg_file = tiu_reg_file + '_' + str(core_id) + '.txt'
@@ -111,22 +112,40 @@ def generate_details(input_fold, out_file, g_info, writer, core_num=8, split_ins
         sdma_instance.add_kpi_field()
         sdma_instance.write()
         # cdma
-        cdma_instance = Cdma(core_id, writer, 'CDMA')
-        if act_core_num:
-            cur_cdma_reg_file = cdma_reg_file + '_' + str(core_id) + '.txt'
-            if os.path.exists(cur_cdma_reg_file) and os.path.getsize(cur_cdma_reg_file):
-                tmp_chip_arch = cdma_instance.load(cur_cdma_reg_file)
-                chip_arch_act = chip_arch_act if chip_arch_act else tmp_chip_arch
-                cdma_instance.add_kpi_field()
-                cdma_instance.write()
-            reg_list += tiu_instance.reg_list + gdma_instance.reg_list + sdma_instance.reg_list + cdma_instance.reg_list
-        else:
+        if chip_arch['Chip Arch'] == 'sg2260' :
             reg_list += tiu_instance.reg_list + gdma_instance.reg_list + sdma_instance.reg_list
+            cdma_instance = None
+            if  act_core_num and core_id == 7 and file_names:
+                for f in file_names:
+                    port = eval(re.search(rf"{cdma_reg_file}_(\d+)\.txt", f).group(1))
+                    cdma_instance = Cdma(port, writer, 'CDMA')
+                    if os.path.exists(f) and os.path.getsize(f):
+                        tmp_chip_arch = cdma_instance.load(f)
+                        chip_arch_act = chip_arch_act if chip_arch_act else tmp_chip_arch
+                        cdma_instance.add_kpi_field(True)
+                        cdma_instance.write()
+                    reg_list += cdma_instance.reg_list
+                    cdma_instances.append(cdma_instance)
+            else:
+                cdma_instance = Cdma(core_id, writer, 'CDMA')
+                cdma_instances.append(cdma_instance)
+        else:
+            cdma_instance = Cdma(core_id, writer, 'CDMA')
+            if act_core_num:
+                cur_cdma_reg_file = cdma_reg_file + '_' + str(core_id) + '.txt'
+                if os.path.exists(cur_cdma_reg_file) and os.path.getsize(cur_cdma_reg_file):
+                    tmp_chip_arch = cdma_instance.load(cur_cdma_reg_file)
+                    chip_arch_act = chip_arch_act if chip_arch_act else tmp_chip_arch
+                    cdma_instance.add_kpi_field(True)
+                    cdma_instance.write()
+                reg_list += tiu_instance.reg_list + gdma_instance.reg_list + sdma_instance.reg_list + cdma_instance.reg_list
+            else:
+                reg_list += tiu_instance.reg_list + gdma_instance.reg_list + sdma_instance.reg_list
+            cdma_instances.append(cdma_instance)
         instr_cols = get_instr_cols(tiu_instance.columns, gdma_instance.columns)
         tiu_instances.append(tiu_instance)
         gdma_instances.append(gdma_instance)
         sdma_instances.append(sdma_instance)
-        cdma_instances.append(cdma_instance)
 
     if act_core_num:
         # instr world
@@ -218,7 +237,7 @@ def generate_divided_details(input_fold, g_info, core_num=8):
             if os.path.exists(cur_cdma_reg_file) and os.path.getsize(cur_cdma_reg_file):
                 tmp_chip_arch = cdma_instance.load(cur_cdma_reg_file)
                 chip_arch_act = chip_arch_act if chip_arch_act else tmp_chip_arch
-                cdma_instance.add_kpi_field()
+                cdma_instance.add_kpi_field(True)
                 cdma_instance.write()
             reg_list += tiu_instance.reg_list + gdma_instance.reg_list + sdma_instance.reg_list + cdma_instance.reg_list
         else:

python/PerfAI/PerfAI.web/src/dma.py

@@ -11,6 +11,7 @@
 from decimal import Decimal
 import os
 import sys
+import glob, re
 
 from utils.utils import *
 
@@ -34,7 +35,7 @@ def __init__(self, dirpath, dmaType):
         self.total_burst_length = 0
         self.total_xact_cnt = 0
         self.frequency = 0
-        self.columns = ['Engine Id', 'Core Id', 'Cmd Id', 'Layer Id', 'Layer Name', 'Subnet Id', 'Subnet Type', 'File Line',
+        self.columns = ['Engine Id', 'Core Id', 'Global Idx', 'Cmd Id', 'Layer Id', 'Layer Name', 'Subnet Id', 'Subnet Type', 'File Line',
                         'Function Type', 'Function Name', 'DMA data size(B)', 'Start Cycle', 'End Cycle',
                         'Asic Cycle', 'Stall Cycle', 'DDR Bandwidth(GB/s)','L2M Bandwidth(GB/s)', 'Direction', 'AvgBurstLength', 'Data Type', 'Non32ByteRatio',
                         'MaskWriteRatio', 'cmd_id_dep', 'cmd_special_function', 'src_start_addr', 'dst_start_addr',
@@ -48,6 +49,12 @@ def __init__(self, dirpath, dmaType):
                         'mask_start_addr_h8', 'mask_start_addr_l32', 'mask_data_format', 'localmem_mask_h32',
                         'localmem_mask_l32',
                         'fill_constant_en', 'constant_value', 'index', 'cmd_short', 'intr_en', 'Msg Id', 'Sd\Wt Count']
+        self.sys_cmd_id = '6';
+        self.sys_wait_id = ['4']
+        if self.dmaType == 'CDMA':
+            self.sys_cmd_id = '7'
+            self.sys_wait_id = ['4', '6']
+
     def dma_engine_type(self):
         if self.dmaType == 'CDMA':
             return '4'
@@ -58,12 +65,42 @@ def dma_engine_type(self):
             self.dmaType = 'TDMA'
             return '3'
 
+    # def process_file(self, layer_map):
+    #     engineId = self.dma_engine_type()
+    #     # file_name = f"{self.dirpath}/{self.dmaType.lower()}RegInfo_0.txt"
+    #     file_name = os.path.join(self.dirpath,f'{self.dmaType.lower()}RegInfo_0.txt')
+    #     if os.path.exists(file_name) and os.path.getsize(file_name) > 0:
+    #         with open(file_name, "r") as f:
+    #             lines = f.readlines()
+    #             for line in lines:
+    #                 self.linecount += 1
+    #                 if "\t" in line:
+    #                     fields = line.split(': ')
+    #                     attr = fields[0][1:]
+    #                     val = fields[1][:-1]
+    #                     self.chipArgs[attr] = val
+    #                 if f'__{self.dmaType}_REG_INFO__' in line:
+    #                     break
+    #         self.frequency = int(self.chipArgs['DMA Frequency(MHz)'])
+    #         coreNum = int(self.chipArgs['Core Num'])
+    #         for coreId in range(int(coreNum)):
+    #             curDmaRegFile = f"{self.dirpath}/{self.dmaType.lower()}RegInfo" + '_' + str(coreId) + '.txt'
+    #             if os.path.exists(curDmaRegFile) and os.path.getsize(curDmaRegFile) != 0:
+    #                 self.actual_corenum += 1
+    #         dmaDf_list = [] #list of tiu dataframes
+    #         for coreId in range(self.actual_corenum):
+    #             dmaDf_list.append(self.process_data(coreId,engineId,layer_map))
+    #         return dmaDf_list
+    #     else:
+    #         self.dma_cycle_list.append(0)
+    #         return []
     def process_file(self, layer_map):
         engineId = self.dma_engine_type()
         # file_name = f"{self.dirpath}/{self.dmaType.lower()}RegInfo_0.txt"
-        file_name = os.path.join(self.dirpath,f'{self.dmaType.lower()}RegInfo_0.txt')
-        if os.path.exists(file_name) and os.path.getsize(file_name) > 0:
-            with open(file_name, "r") as f:
+        # file_name = os.path.join(self.dirpath,f'{self.dmaType.lower()}RegInfo_0.txt')
+        file_names = sorted(glob.glob(self.dirpath + f"{self.dmaType.lower()}RegInfo_*.txt"))
+        if file_names and os.path.exists(file_names[0]) and os.path.getsize(file_names[0]) > 0:
+            with open(file_names[0], "r") as f:
                 lines = f.readlines()
                 for line in lines:
                     self.linecount += 1
@@ -76,14 +113,25 @@ def process_file(self, layer_map):
                         break
             self.frequency = int(self.chipArgs['DMA Frequency(MHz)'])
             coreNum = int(self.chipArgs['Core Num'])
-            for coreId in range(int(coreNum)):
-                curDmaRegFile = f"{self.dirpath}/{self.dmaType.lower()}RegInfo" + '_' + str(coreId) + '.txt'
-                if os.path.exists(curDmaRegFile) and os.path.getsize(curDmaRegFile) != 0:
-                    self.actual_corenum += 1
-            dmaDf_list = [] #list of tiu dataframes
-            for coreId in range(self.actual_corenum):
-                dmaDf_list.append(self.process_data(coreId,engineId,layer_map))
-            return dmaDf_list
+            if engineId != '4':
+                for coreId in range(int(coreNum)):
+                    curDmaRegFile = f"{self.dirpath}/{self.dmaType.lower()}RegInfo" + '_' + str(coreId) + '.txt'
+                    if os.path.exists(curDmaRegFile) and os.path.getsize(curDmaRegFile) != 0:
+                        self.actual_corenum += 1
+                dmaDf_list = [] #list of tiu dataframes
+                for coreId in range(self.actual_corenum):
+                    dmaDf_list.append(self.process_data(coreId,engineId,layer_map))
+                return dmaDf_list
+            else:
+                self.actual_corenum = 1
+                dmaDf_list = []
+                for f in file_names:
+                    port = eval(re.search(rf"{self.dmaType.lower()}RegInfo_(\d+)\.txt", f).group(1))
+                    data = self.process_data(port,engineId,layer_map)
+                    data.port = port
+                    dmaDf_list.append(data)
+                return dmaDf_list
+
         else:
             self.dma_cycle_list.append(0)
             return []
@@ -151,21 +199,26 @@ def process_data(self, coreId, engineId, layer_map):
         totalInstRegList = []
         for i in range(len(new_reglist)):
             regDict = new_reglist[i]
-            if regDict['cmd_type'] == '6':
+            # dma_sys do not transfer data
+            if regDict['cmd_type'] == self.sys_cmd_id:
                 regDict['Data Type'] = 'None'
-            if int(regDict['cmd_type']) == 6: # dma_sys do not transfer data
                 regDict['Direction'] = '-'
+            transfer_bytes = 0
+            if regDict['DMA data size(B)'].isnumeric():
+                transfer_bytes = int(regDict['DMA data size(B)'])
             if regDict['Asic Cycle'].isnumeric():
                 DmaCycle += int(regDict['Asic Cycle'])
             if 'DDR' in regDict['Direction'] and regDict['DMA data size(B)'].isnumeric():
-                dmaDdrTotalDataSize += int(regDict['DMA data size(B)'])
-                dmaDdrCycle += float(regDict['Asic Cycle'])
-                dmaDdrBurstLength += int(regDict['gmem_bl_sum'])
-                dmaDdrXactCnt += int(regDict['gmem_xact_cnt'])
+                if self.dmaType != 'CDMA' or transfer_bytes:
+                    dmaDdrTotalDataSize += transfer_bytes
+                    dmaDdrCycle += float(regDict['Asic Cycle'])
+                    dmaDdrBurstLength += int(regDict['gmem_bl_sum'])
+                    dmaDdrXactCnt += int(regDict['gmem_xact_cnt'])
             elif 'L2' in regDict['Direction'] and regDict['DMA data size(B)'].isnumeric():
-                dmaL2TotalDataSize += int(regDict['DMA data size(B)'])
-                dmaL2Cycle += float(regDict['Asic Cycle'])
-            if regDict['cmd_type'] == '6' and regDict['cmd_special_function'] == '4':
+                if self.dmaType != 'CDMA' or transfer_bytes:
+                    dmaL2TotalDataSize += transfer_bytes
+                    dmaL2Cycle += float(regDict['Asic Cycle'])
+            if regDict['cmd_type'] == self.sys_cmd_id and regDict['cmd_special_function'] in self.sys_wait_id:
                 dmaWaitMsgTotalTime += eval(regDict['Asic Cycle'])
             if int(regDict['gmem_xact_cnt']) > 0:
                 regDict['AvgBurstLength'] = Decimal(

python/PerfAI/PerfAI.web/src/summary.py

@@ -68,7 +68,7 @@ def _process_dma(self, dmaProcessor, add_l2=True):
         if len(temp_timelist) >= len(self.total_time_list):
             self.total_time_list = temp_timelist
         dmaProcessor.dma_cycle_list.append(max(dmaProcessor.dma_cycle_list))
-        last_row = len(self.total_time_list) - 1
+        last_row = len(dmaProcessor.dma_cycle_list) - 1
         dmaProcessor.dma_cycle_list[last_row] = str(
             (Decimal(dmaProcessor.dma_cycle_list[last_row] / dmaProcessor.frequency)).quantize(Decimal("0.00"))) + 'us'
         dmaProcessor.dma_ddr_total_datasize_list.append(sum(dmaProcessor.dma_ddr_total_datasize_list))
@@ -157,6 +157,10 @@ def make_summary(self):
         self.data[0] = CoreIdList
         self.data[1] = ParallelismList
         self.data[2] = self.total_time_list
+        max_len = max([len(i)for i in self.data])
+        for i in self.data:
+            if len(i) < max_len:
+                i[1:1] = [0] * (max_len - len(i))
         summaryData = transpose(self.data).tolist()
         summaryDf = pd.DataFrame(summaryData, columns=self.columns, index=None)
         return summaryDf

python/PerfAI/PerfAI.web/src/tiu.py

@@ -29,7 +29,7 @@ def __init__(self, dirpath):
         self.total_alg_ops_list = []
         self.uArach_rate_list = []
         self.total_uarch_ops_list = []
-        self.columns = ['Engine Id', 'Core Id', 'Cmd Id', 'Layer Id', 'Layer Name', 'Subnet Id', 'Subnet Type', 'File Line',
+        self.columns = ['Engine Id', 'Core Id', 'Global Idx', 'Cmd Id', 'Layer Id', 'Layer Name', 'Subnet Id', 'Subnet Type', 'File Line',
                         'Function Type', 'Function Name',
                         'Alg Cycle', 'Alg Ops','Asic Cycle', 'Start Cycle', 'End Cycle',
                         'uArch Ops', 'uArch Rate', 'Bank Conflict Ratio',