Kernel profiler based NoC event tracing (#16541)

### Problem Description
This PR adds support for recording detailed traces of all NoC activity
initiated by Tensix worker cores. This data is then written to a JSON
format file. These JSON traces can be either analyzed directly or 
consumed by tt-npe (noc performance estimator). 

### What's Changed
NoC tracing builds on top of the timestamped data packets
(`PacketType::TS_DATA`) feature in the kernel profiler.

NoC tracing can be enabled on top of normal kernel profiling by setting
the ENV variable `TT_METAL_DEVICE_PROFILER_NOC_EVENTS=1`.

The directory path where noc traces are written can be set using
ENV variable `TT_METAL_DEVICE_PROFILER_NOC_EVENTS_RPT_PATH`.

A timestamped packet is recorded on each worker core for each call to
`dataflow_api.h:noc_async_*` functions. All arguments to the
`dataflow_api.h` function are bit-packed into the `TS_DATA` payload.
This is done by adding new stripped-by-default macros to each noc_async*
call of interest in `dataflow_api.h`.

```cpp
  // example of RECORD macro within dataflow_api.h
  inline void noc_async_read(...) {
      // if TT_METAL_DEVICE_PROFILER_NOC_EVENTS != 1, the following macro is stripped out
      RECORD_NOC_EVENT_WITH_ADDR(...);
      // noc_async_read continues unmodified here ... 
  }
```

### Checklist
- [X] Post commit CI passes 
- [X] Device Perf Regression Tests Pass
- [X] uBenchmark Regression Tests Pass
- [X] multi-device (T3K) Regression Tests Pass

tests/tt_metal/tools/profiler/test_device_profiler.py

@@ -351,6 +351,24 @@ def test_timestamped_events():
         assert eventCount in REF_COUNT_DICT[ENV_VAR_ARCH_NAME], "Wrong event count"
 
 
+def test_noc_event_profiler():
+    ENV_VAR_ARCH_NAME = os.getenv("ARCH_NAME")
+    assert ENV_VAR_ARCH_NAME in ["grayskull", "wormhole_b0", "blackhole"]
+
+    testCommand = f"build/{PROG_EXMP_DIR}/test_noc_event_profiler"
+    clear_profiler_runtime_artifacts()
+    nocEventProfilerEnv = "TT_METAL_DEVICE_PROFILER_NOC_EVENTS=1"
+    profilerRun = os.system(f"cd {TT_METAL_HOME} && {nocEventProfilerEnv} {testCommand}")
+    assert profilerRun == 0
+
+    expected_trace_file = f"{PROFILER_LOGS_DIR}/noc_trace_dev0_ID0.json"
+    assert os.path.isfile(expected_trace_file)
+
+    with open(expected_trace_file, "r") as nocTraceJson:
+        noc_trace_data = json.load(nocTraceJson)
+        assert len(noc_trace_data) == 8
+
+
 def test_sub_device_profiler():
     ARCH_NAME = os.getenv("ARCH_NAME")
     run_gtest_profiler_test(

tt_metal/api/tt-metalium/profiler.hpp

@@ -14,9 +14,12 @@
 #include "program_impl.hpp"
 #include "profiler_state.hpp"
 #include "common.hpp"
+#include "profiler_optional_metadata.hpp"
 #include "tracy/TracyTTDevice.hpp"
 #include "common/TracyTTDeviceData.hpp"
 
+#include <nlohmann/json.hpp>
+
 using std::chrono::duration;
 using std::chrono::duration_cast;
 using std::chrono::nanoseconds;
@@ -61,21 +64,74 @@ class DeviceProfiler {
     // Iterate through all zone source locations and generate hash
     void generateZoneSourceLocationsHashes();
 
+    // serialize all noc trace data into per-op json trace files
+    void serializeJsonNocTraces(
+        const nlohmann::ordered_json& noc_trace_json_log, const std::filesystem::path& output_dir, chip_id_t device_id);
+
+    void emitCSVHeader(
+        std::ofstream& log_file_ofs, const tt::ARCH& device_architecture, int device_core_frequency) const;
+
+    // translates potentially-virtual coordinates recorded on Device into physical coordinates
+    CoreCoord getPhysicalAddressFromVirtual(chip_id_t device_id, const CoreCoord& c) const;
+
     // Dumping profile result to file
-    void dumpResultToFile(
+    void logPacketData(
+        std::ofstream& log_file_ofs,
+        nlohmann::ordered_json& noc_trace_json_log,
         uint32_t runID,
         uint32_t runHostID,
-        int device_id,
+        const std::string& opname,
+        chip_id_t device_id,
         CoreCoord core,
         int core_flat,
         int risc_num,
         uint64_t stat_value,
         uint32_t timer_id,
         uint64_t timestamp);
 
+    // logs packet data to CSV file
+    void logPacketDataToCSV(
+        std::ofstream& log_file_ofs,
+        chip_id_t device_id,
+        int core_x,
+        int core_y,
+        const std::string_view risc_name,
+        uint32_t timer_id,
+        uint64_t timestamp,
+        uint64_t data,
+        uint32_t run_id,
+        uint32_t run_host_id,
+        const std::string_view opname,
+        const std::string_view zone_name,
+        kernel_profiler::PacketTypes packet_type,
+        uint64_t source_line,
+        const std::string_view source_file);
+
+    // dump noc trace related profile data to json file
+    void logNocTracePacketDataToJson(
+        nlohmann::ordered_json& noc_trace_json_log,
+        chip_id_t device_id,
+        int core_x,
+        int core_y,
+        const std::string_view risc_name,
+        uint32_t timer_id,
+        uint64_t timestamp,
+        uint64_t data,
+        uint32_t run_id,
+        uint32_t run_host_id,
+        const std::string_view opname,
+        const std::string_view zone_name,
+        kernel_profiler::PacketTypes packet_type,
+        uint64_t source_line,
+        const std::string_view source_file);
+
     // Helper function for reading risc profile results
     void readRiscProfilerResults(
-        IDevice* device, CoreCoord& worker_core);
+        IDevice* device,
+        const CoreCoord& worker_core,
+        const std::optional<ProfilerOptionalMetadata>& metadata,
+        std::ofstream& log_file_ofs,
+        nlohmann::ordered_json& noc_trace_json_log);
 
     // Push device results to tracy
     void pushTracyDeviceResults();
@@ -100,21 +156,19 @@ class DeviceProfiler {
     // DRAM Vector
     std::vector<uint32_t> profile_buffer;
 
-    //Device events
+    // Device events
     std::set<tracy::TTDeviceEvent> device_events;
 
     std::set<tracy::TTDeviceEvent> device_sync_events;
 
     std::set<tracy::TTDeviceEvent> device_sync_new_events;
 
-    //shift
+    // shift
     int64_t shift = 0;
 
-    //frequency scale
+    // frequency scale
     double freqScale = 1.0;
 
-    uint32_t my_device_id = 0;
-
     // Freshen device logs
     void freshDeviceLog();
 
@@ -128,7 +182,8 @@ class DeviceProfiler {
     void dumpResults(
         IDevice* device,
         const std::vector<CoreCoord>& worker_cores,
-        ProfilerDumpState state = ProfilerDumpState::NORMAL);
+        ProfilerDumpState state = ProfilerDumpState::NORMAL,
+        const std::optional<ProfilerOptionalMetadata>& metadata = {});
 };
 
 }  // namespace tt_metal

tt_metal/api/tt-metalium/profiler_optional_metadata.hpp

@@ -0,0 +1,28 @@
+// SPDX-FileCopyrightText: © 2025 Tenstorrent Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "map"
+
+class ProfilerOptionalMetadata {
+    using RuntimeID = uint32_t;
+
+public:
+    ProfilerOptionalMetadata(std::map<std::pair<chip_id_t, RuntimeID>, std::string>&& runtime_map) :
+        runtime_id_to_opname_(std::move(runtime_map)) {}
+
+    const std::string& get_op_name(chip_id_t device_id, RuntimeID runtime_id) const {
+        static const std::string empty_string;
+        auto key = std::make_pair(device_id, runtime_id);
+        auto it = runtime_id_to_opname_.find(key);
+        if (it != runtime_id_to_opname_.end()) {
+            return it->second;
+        }
+        return empty_string;
+    }
+
+private:
+    std::map<std::pair<chip_id_t, RuntimeID>, std::string> runtime_id_to_opname_;
+};

tt_metal/api/tt-metalium/rtoptions.hpp

@@ -111,6 +111,8 @@ class RunTimeOptions {
     bool profile_dispatch_cores = false;
     bool profiler_sync_enabled = false;
     bool profiler_buffer_usage_enabled = false;
+    bool profiler_noc_events_enabled = false;
+    std::string profiler_noc_events_report_path;
 
     bool null_kernels = false;
 
@@ -287,6 +289,8 @@ class RunTimeOptions {
     inline bool get_profiler_do_dispatch_cores() { return profile_dispatch_cores; }
     inline bool get_profiler_sync_enabled() { return profiler_sync_enabled; }
     inline bool get_profiler_buffer_usage_enabled() { return profiler_buffer_usage_enabled; }
+    inline bool get_profiler_noc_events_enabled() { return profiler_noc_events_enabled; }
+    inline std::string get_profiler_noc_events_report_path() { return profiler_noc_events_report_path; }
 
     inline void set_kernels_nullified(bool v) { null_kernels = v; }
     inline bool get_kernels_nullified() { return null_kernels; }

tt_metal/api/tt-metalium/tt_metal.hpp

@@ -14,6 +14,7 @@
 #include "buffer.hpp"
 #include "profiler.hpp"
 #include "llrt/tt_cluster.hpp"
+#include "profiler_optional_metadata.hpp"
 
 namespace tt::tt_metal {
 inline namespace v0 {
@@ -220,7 +221,10 @@ void ProfilerSync(ProfilerSyncState state);
  * | satate        | Dumpprofiler various states                       | ProfilerDumpState |                  | False |
  * */
 void DumpDeviceProfileResults(
-    IDevice* device, std::vector<CoreCoord>& worker_cores, ProfilerDumpState = ProfilerDumpState::NORMAL);
+    IDevice* device,
+    std::vector<CoreCoord>& worker_cores,
+    ProfilerDumpState = ProfilerDumpState::NORMAL,
+    const std::optional<ProfilerOptionalMetadata>& metadata = {});
 
 /**
  * Traverse all cores and read device side profiler data and dump results into device side CSV log
@@ -232,7 +236,7 @@ void DumpDeviceProfileResults(
  * | device        | The device holding the program being profiled.    | Device * |                           | True |
  * | satate        | Dumpprofiler various states                       | ProfilerDumpState |                  | False |
  * */
-void DumpDeviceProfileResults(IDevice* device, ProfilerDumpState = ProfilerDumpState::NORMAL);
+void DumpDeviceProfileResults(IDevice* device, ProfilerDumpState = ProfilerDumpState::NORMAL, const std::optional<ProfilerOptionalMetadata>& metadata = {});
 
 /**
  * Set the directory for device-side CSV logs produced by the profiler instance in the tt-metal module

tt_metal/hw/firmware/src/brisc.cc

@@ -7,6 +7,7 @@
 #include <cstdint>
 
 // clang-format off
+#undef PROFILE_NOC_EVENTS
 #include "risc_common.h"
 #include "tensix.h"
 #include "tensix_types.h"