diff --git a/tests/scripts/run_tt_metal.py b/tests/scripts/run_tt_metal.py index a34cdf723d8..b855f8bfd5c 100644 --- a/tests/scripts/run_tt_metal.py +++ b/tests/scripts/run_tt_metal.py @@ -112,6 +112,7 @@ ) TT_METAL_FAST_DISPATCH_TEST_ENTRIES = ( + TestEntry("tt_metal/tests/unit_tests_noc", "unit_tests_noc"), TestEntry("tt_metal/tests/test_eltwise_binary", "test_eltwise_binary"), # TestEntry( # "tt_metal/tests/test_matmul_multi_core_multi_dram", diff --git a/tt_metal/hw/firmware/src/brisc.cc b/tt_metal/hw/firmware/src/brisc.cc index 03e78e1be33..861c8da513b 100644 --- a/tt_metal/hw/firmware/src/brisc.cc +++ b/tt_metal/hw/firmware/src/brisc.cc @@ -368,6 +368,21 @@ inline void wait_ncrisc_trisc() { inline void trigger_sync_register_init() { mailboxes->slave_sync.trisc0 = RUN_SYNC_MSG_INIT_SYNC_REGISTERS; } +template +inline __attribute__((always_inline)) void signal_dispatch_done( + uint8_t noc, uint8_t cmd_buf, uint64_t dispatch_addr, uint32_t atomic_ret_val, bool posted) { + noc_fast_atomic_increment( + noc, + cmd_buf, + dispatch_addr, + NOC_UNICAST_WRITE_VC, + 1, + 31 /*wrap*/, + false /*linked*/, + posted /*posted*/, + atomic_ret_val); +} + int main() { configure_l1_data_cache(); DIRTY_STACK_MEMORY(); @@ -451,6 +466,9 @@ int main() { 31 /*wrap*/, false /*linked*/, post_atomic_increments /*posted*/); +#if defined(ARCH_BLACKHOLE) + while (!ncrisc_noc_nonposted_atomics_flushed(noc_index)); +#endif } } @@ -592,15 +610,18 @@ int main() { // messages in the ring buffer. Must be executed before the atomic increment, as after that the launch // message is no longer owned by us. CLEAR_PREVIOUS_LAUNCH_MESSAGE_ENTRY_FOR_WATCHER(); - noc_fast_atomic_increment( - noc_index, - NCRISC_AT_CMD_BUF, - dispatch_addr, - NOC_UNICAST_WRITE_VC, - 1, - 31 /*wrap*/, - false /*linked*/, - post_atomic_increments /*posted*/); +#if defined(ARCH_BLACKHOLE) + if (noc_mode == DM_DEDICATED_NOC) { + signal_dispatch_done(noc_index, cmd_buf, dispatch_addr, 0, post_atomic_increments); + while (!ncrisc_noc_nonposted_atomics_flushed(noc_index)); + } else { + signal_dispatch_done( + noc_index, cmd_buf, dispatch_addr, MEM_NOC_ATOMIC_RET_VAL_ADDR, post_atomic_increments); + while (!ncrisc_dynamic_noc_nonposted_atomics_flushed(noc_index)); + } +#else + signal_dispatch_done(noc_index, cmd_buf, dispatch_addr, 0, post_atomic_increments); +#endif mailboxes->launch_msg_rd_ptr = (launch_msg_rd_ptr + 1) & (launch_msg_buffer_num_entries - 1); } } diff --git a/tt_metal/hw/inc/blackhole/noc_nonblocking_api.h b/tt_metal/hw/inc/blackhole/noc_nonblocking_api.h index f8752cefa3d..d0ab288d90b 100644 --- a/tt_metal/hw/inc/blackhole/noc_nonblocking_api.h +++ b/tt_metal/hw/inc/blackhole/noc_nonblocking_api.h @@ -134,6 +134,9 @@ inline __attribute__((always_inline)) bool noc_cmd_buf_ready(uint32_t noc, uint3 template inline __attribute__((always_inline)) void ncrisc_noc_fast_read( uint32_t noc, uint32_t cmd_buf, uint64_t src_addr, uint32_t dest_addr, uint32_t len_bytes) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } if constexpr (noc_mode == DM_DYNAMIC_NOC) { uint32_t noc_rd_cmd_field = NOC_CMD_CPY | NOC_CMD_RD | NOC_CMD_RESP_MARKED | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(1); @@ -146,17 +149,16 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_read( noc, cmd_buf, NOC_TARG_ADDR_COORDINATE, (uint32_t)(src_addr >> NOC_ADDR_COORD_SHIFT) & NOC_COORDINATE_MASK); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_reads_flushed(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_RD_RESP_RECEIVED); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::READS_NUM_ISSUED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_RD_RESP_RECEIVED) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_reads_flushed(uint32_t noc) { @@ -182,6 +184,14 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write( bool multicast_path_reserve, bool posted = false, uint32_t trid = 0) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + if (posted) { + inc_noc_counter_val(noc, 1); + } else { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, num_dests); + } + } uint32_t noc_cmd_field = NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(vc) | (linked ? NOC_CMD_VC_LINKED : 0x0) | (mcast ? ((multicast_path_reserve ? NOC_CMD_PATH_RESERVE : 0) | NOC_CMD_BRCST_PACKET) : 0x0) | @@ -201,16 +211,9 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if (posted) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { + if (posted) { noc_posted_writes_num_issued[noc] += 1; - } - } else { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, num_dests); } else { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += num_dests; @@ -230,6 +233,10 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_loopback_src( bool linked, uint32_t num_dests, bool multicast_path_reserve) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, num_dests); + } uint32_t noc_cmd_field = NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(vc) | (linked ? NOC_CMD_VC_LINKED : 0x0) | (mcast ? ((multicast_path_reserve ? NOC_CMD_PATH_RESERVE : 0) | NOC_CMD_BRCST_PACKET) : 0x0) | @@ -244,10 +251,7 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_loopback_src( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_BRCST_EXCLUDE, 0); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, num_dests); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += num_dests; } @@ -266,6 +270,10 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_exclude_region( uint32_t num_dests, bool multicast_path_reserve, uint32_t exclude_region) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, num_dests); + } uint32_t noc_cmd_field = NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(vc) | (linked ? NOC_CMD_VC_LINKED : 0x0) | (mcast ? ((multicast_path_reserve ? NOC_CMD_PATH_RESERVE : 0) | NOC_CMD_BRCST_PACKET) : 0x0) | @@ -280,10 +288,7 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_exclude_region( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_BRCST_EXCLUDE, exclude_region); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, num_dests); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += num_dests; } @@ -292,6 +297,10 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_exclude_region( template inline __attribute__((always_inline)) void ncrisc_noc_blitz_write_setup( uint32_t noc, uint32_t cmd_buf, uint64_t dest_addr, uint32_t len_bytes, uint32_t vc, uint32_t num_times_to_write) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, num_times_to_write); + inc_noc_counter_val(noc, num_times_to_write); + } uint32_t noc_cmd_field = NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(vc) | NOC_CMD_RESP_MARKED; while (!noc_cmd_buf_ready(noc, cmd_buf)); @@ -300,19 +309,17 @@ inline __attribute__((always_inline)) void ncrisc_noc_blitz_write_setup( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_RET_ADDR_MID, (uint32_t)(dest_addr >> 32) & 0x1000000F); NOC_CMD_BUF_WRITE_REG( noc, cmd_buf, NOC_RET_ADDR_COORDINATE, (uint32_t)(dest_addr >> NOC_ADDR_COORD_SHIFT) & NOC_COORDINATE_MASK); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, num_times_to_write); - inc_noc_counter_val(noc, num_times_to_write); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += num_times_to_write; noc_nonposted_writes_acked[noc] += num_times_to_write; } } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_nonposted_writes_sent(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_NONPOSTED_WR_REQ_SENT); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::NONPOSTED_WRITES_NUM_ISSUED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_NONPOSTED_WR_REQ_SENT) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_writes_sent(uint32_t noc) { @@ -320,9 +327,10 @@ inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_writes_sent(uint } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_posted_writes_sent(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_POSTED_WR_REQ_SENT); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::POSTED_WRITES_NUM_ISSUED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_POSTED_WR_REQ_SENT) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_posted_writes_sent(uint32_t noc) { @@ -330,9 +338,10 @@ inline __attribute__((always_inline)) bool ncrisc_noc_posted_writes_sent(uint32_ } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_nonposted_writes_flushed(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_WR_ACK_RECEIVED); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::NONPOSTED_WRITES_ACKED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_WR_ACK_RECEIVED) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_writes_flushed(uint32_t noc) { @@ -350,9 +359,10 @@ inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_write_with_trans } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_nonposted_atomics_flushed(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_ATOMIC_RESP_RECEIVED); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::NONPOSTED_ATOMICS_ACKED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_ATOMIC_RESP_RECEIVED) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_atomics_flushed(uint32_t noc) { @@ -676,6 +686,14 @@ inline __attribute__((always_inline)) void noc_fast_write_dw_inline( uint32_t static_vc, bool mcast, bool posted = false) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + if (posted) { + inc_noc_counter_val(noc, 1); + } else { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, 1); + } + } bool static_vc_alloc = true; uint32_t noc_cmd_field = (static_vc_alloc ? NOC_CMD_VC_STATIC : 0x0) | NOC_CMD_STATIC_VC(static_vc) | NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_WR_INLINE | @@ -696,17 +714,9 @@ inline __attribute__((always_inline)) void noc_fast_write_dw_inline( noc, cmd_buf, NOC_TARG_ADDR_COORDINATE, (uint32_t)(dest_addr >> NOC_ADDR_COORD_SHIFT) & NOC_COORDINATE_MASK); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, be32); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - - if (posted) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { + if (posted) { noc_posted_writes_num_issued[noc] += 1; - } - } else { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, 1); } else { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += 1; @@ -725,6 +735,11 @@ inline __attribute__((always_inline)) void noc_fast_atomic_increment( bool linked, bool posted = false, uint32_t atomic_ret_val = 0) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + if (!posted) { + inc_noc_counter_val(noc, 1); + } + } while (!noc_cmd_buf_ready(noc, cmd_buf)); if constexpr (noc_mode == DM_DYNAMIC_NOC || program_ret_addr == true) { uint32_t noc_id_reg = NOC_CMD_BUF_READ_REG(noc, 0, NOC_NODE_ID); @@ -750,10 +765,8 @@ inline __attribute__((always_inline)) void noc_fast_atomic_increment( NOC_AT_INS(NOC_AT_INS_INCR_GET) | NOC_AT_WRAP(wrap) | NOC_AT_IND_32((addr >> 2) & 0x3) | NOC_AT_IND_32_SRC(0)); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_DATA, incr); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, 0x1); - if (!posted) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { + if (!posted) { noc_nonposted_atomics_acked[noc] += 1; } } @@ -763,6 +776,9 @@ inline __attribute__((always_inline)) void noc_fast_atomic_increment( template inline __attribute__((always_inline)) void ncrisc_noc_fast_read_with_transaction_id( uint32_t noc, uint32_t cmd_buf, uint32_t src_base_addr, uint32_t src_addr, uint32_t dest_addr, uint32_t trid) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } uint32_t src_addr_; src_addr_ = src_base_addr + src_addr; @@ -772,9 +788,7 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_read_with_transaction NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_RET_ADDR_LO, dest_addr); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_TARG_ADDR_LO, src_addr_); // (uint32_t)src_addr NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } diff --git a/tt_metal/hw/inc/dataflow_api.h b/tt_metal/hw/inc/dataflow_api.h index cd1e33c417c..3058ff764a1 100644 --- a/tt_metal/hw/inc/dataflow_api.h +++ b/tt_metal/hw/inc/dataflow_api.h @@ -440,6 +440,9 @@ void noc_async_read_one_packet( Read requests - use static VC Read responses - assigned VCs dynamically */ + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } WAYPOINT("RP2W"); while (!noc_cmd_buf_ready(noc, read_cmd_buf)); @@ -466,9 +469,7 @@ void noc_async_read_one_packet( (uint32_t)(src_noc_addr >> NOC_ADDR_COORD_SHIFT) & NOC_COORDINATE_MASK); NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_AT_LEN_BE, size); NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } @@ -553,6 +554,11 @@ FORCE_INLINE void noc_async_read_one_packet_with_state( Read requests - use static VC Read responses - assigned VCs dynamically */ + if constexpr (inc_num_issued) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } + } WAYPOINT("RP4W"); while (!noc_cmd_buf_ready(noc, read_cmd_buf)); @@ -568,9 +574,7 @@ FORCE_INLINE void noc_async_read_one_packet_with_state( NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); if constexpr (inc_num_issued) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } @@ -624,6 +628,11 @@ FORCE_INLINE void noc_async_read_with_state( DEBUG_SANITIZE_NOC_READ_TRANSACTION_WITH_ADDR_STATE(noc, src_noc_addr, dst_local_l1_addr, size); while (size > NOC_MAX_BURST_SIZE) { + if constexpr (inc_num_issued) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } + } WAYPOINT("RP6W"); while (!noc_cmd_buf_ready(noc, read_cmd_buf)); WAYPOINT("RP6D"); @@ -636,14 +645,17 @@ FORCE_INLINE void noc_async_read_with_state( src_noc_addr += NOC_MAX_BURST_SIZE; dst_local_l1_addr += NOC_MAX_BURST_SIZE; if constexpr (inc_num_issued) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } } + if constexpr (inc_num_issued) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } + } // left-over packet WAYPOINT("RP7W"); while (!noc_cmd_buf_ready(noc, read_cmd_buf)); @@ -654,9 +666,7 @@ FORCE_INLINE void noc_async_read_with_state( NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_AT_LEN_BE, size); NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); if constexpr (inc_num_issued) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } @@ -678,6 +688,10 @@ void noc_async_read_inc_num_issued(std::uint32_t num_issued_reads_inc, uint8_t n FORCE_INLINE void noc_async_write_one_packet( std::uint32_t src_local_l1_addr, std::uint64_t dst_noc_addr, std::uint32_t size, uint8_t noc = noc_index) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, 1); + } WAYPOINT("NWPW"); DEBUG_SANITIZE_NOC_WRITE_TRANSACTION(noc, dst_noc_addr, src_local_l1_addr, size); while (!noc_cmd_buf_ready(noc, write_cmd_buf)); @@ -702,10 +716,7 @@ void noc_async_write_one_packet( (uint32_t)(dst_noc_addr >> NOC_ADDR_COORD_SHIFT) & NOC_COORDINATE_MASK); NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_AT_LEN_BE, size); NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += 1; // num_dests } @@ -722,6 +733,10 @@ void noc_async_write_multicast_one_packet( bool linked = false, bool multicast_path_reserve = true, uint8_t noc = noc_index) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, num_dests); + } WAYPOINT("NWPW"); DEBUG_SANITIZE_NOC_MULTI_WRITE_TRANSACTION(noc, dst_noc_addr_multicast, src_local_l1_addr, size); while (!noc_cmd_buf_ready(noc, write_cmd_buf)); @@ -746,10 +761,7 @@ void noc_async_write_multicast_one_packet( (uint32_t)(dst_noc_addr_multicast >> NOC_ADDR_COORD_SHIFT) & NOC_COORDINATE_MASK); NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_AT_LEN_BE, size); NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, num_dests); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += num_dests; } @@ -787,6 +799,12 @@ FORCE_INLINE void noc_async_write_one_packet_set_state( template FORCE_INLINE void noc_async_write_one_packet_with_state( std::uint32_t src_local_l1_addr, std::uint32_t dst_noc_addr, uint8_t noc = noc_index) { + if constexpr (non_posted) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, 1); + } + } WAYPOINT("NWPW"); while (!noc_cmd_buf_ready(noc, write_cmd_buf)); WAYPOINT("NWPD"); @@ -799,10 +817,7 @@ FORCE_INLINE void noc_async_write_one_packet_with_state( NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); if constexpr (non_posted) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += 1; // num_dests } @@ -1484,7 +1499,9 @@ FORCE_INLINE void noc_async_read_tile_dram_sharded_with_state( uint32_t src_base_addr, uint32_t src_addr, uint32_t dest_addr, uint32_t trid = 0, uint8_t noc = noc_index) { uint32_t src_addr_; - + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } src_addr_ = src_base_addr + src_addr; WAYPOINT("NRTW"); @@ -1494,9 +1511,7 @@ void noc_async_read_tile_dram_sharded_with_state( NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_RET_ADDR_LO, dest_addr); NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_TARG_ADDR_LO, src_addr_); // (uint32_t)src_addr NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } diff --git a/tt_metal/hw/inc/dataflow_api_addrgen.h b/tt_metal/hw/inc/dataflow_api_addrgen.h index afec1f14fb1..e0944917d0f 100644 --- a/tt_metal/hw/inc/dataflow_api_addrgen.h +++ b/tt_metal/hw/inc/dataflow_api_addrgen.h @@ -337,6 +337,9 @@ struct InterleavedAddrGenFast { FORCE_INLINE void noc_async_read_tile( const uint32_t id, uint32_t dest_addr, const uint32_t offset = 0, uint8_t noc = noc_index) const { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } uint32_t bank_offset_index = interleaved_addr_gen::get_bank_offset_index(id); uint32_t bank_index = interleaved_addr_gen::get_bank_index(id, bank_offset_index); uint32_t src_addr = this->get_addr(id, bank_offset_index, bank_index, offset); @@ -357,15 +360,17 @@ struct InterleavedAddrGenFast { NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_TARG_ADDR_COORDINATE, src_noc_xy); // src_addr >> 32 NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_AT_LEN_BE, this->page_size); // len_bytes NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } FORCE_INLINE void noc_async_write_tile(const uint32_t id, uint32_t src_addr, uint8_t noc = noc_index) const { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, 1); + } uint32_t bank_offset_index = interleaved_addr_gen::get_bank_offset_index(id); uint32_t bank_index = interleaved_addr_gen::get_bank_index(id, bank_offset_index); uint32_t dest_addr = this->get_addr(id, bank_offset_index, bank_index); @@ -389,10 +394,7 @@ struct InterleavedAddrGenFast { NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_RET_ADDR_COORDINATE, dest_noc_xy); // dest_addr >> 32 NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_AT_LEN_BE, this->page_size); // len_bytes NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += 1; // num_dests } @@ -435,6 +437,9 @@ struct InterleavedPow2AddrGenFast { FORCE_INLINE void noc_async_read_page( const uint32_t id, uint32_t dest_addr, const uint32_t offset = 0, uint8_t noc = noc_index) const { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } uint32_t bank_offset_index = interleaved_addr_gen::get_bank_offset_index(id); uint32_t bank_index = interleaved_addr_gen::get_bank_index(id, bank_offset_index); uint32_t src_addr = this->get_addr(id, bank_offset_index, bank_index, offset); @@ -457,9 +462,7 @@ struct InterleavedPow2AddrGenFast { NOC_CMD_BUF_WRITE_REG( noc, read_cmd_buf, NOC_AT_LEN_BE, 1 << this->aligned_log_base_2_of_page_size); // len_bytes NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } @@ -471,6 +474,9 @@ struct InterleavedPow2AddrGenFast { const uint32_t size, const uint32_t offset, uint8_t noc = noc_index) const { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } uint32_t bank_offset_index = interleaved_addr_gen::get_bank_offset_index(id); uint32_t bank_index = interleaved_addr_gen::get_bank_index(id, bank_offset_index); uint32_t src_addr = this->get_addr(id, bank_offset_index, bank_index, offset); @@ -491,9 +497,7 @@ struct InterleavedPow2AddrGenFast { NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_TARG_ADDR_COORDINATE, src_noc_xy); // src_addr >> 32 NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_AT_LEN_BE, size); // len_bytes NOC_CMD_BUF_WRITE_REG(noc, read_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } @@ -505,6 +509,10 @@ struct InterleavedPow2AddrGenFast { const uint32_t write_size_bytes, const uint32_t offset = 0, uint8_t noc = noc_index) const { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, 1); + } uint32_t bank_offset_index = interleaved_addr_gen::get_bank_offset_index(id); uint32_t bank_index = interleaved_addr_gen::get_bank_index(id, bank_offset_index); uint32_t dest_addr = this->get_addr(id, bank_offset_index, bank_index, offset); @@ -528,10 +536,7 @@ struct InterleavedPow2AddrGenFast { NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_RET_ADDR_COORDINATE, dest_noc_xy); // dest_addr >> 32 NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_AT_LEN_BE, write_size_bytes); // len_bytes NOC_CMD_BUF_WRITE_REG(noc, write_cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += 1; // num_dests } diff --git a/tt_metal/hw/inc/wormhole/noc_nonblocking_api.h b/tt_metal/hw/inc/wormhole/noc_nonblocking_api.h index 3fd17b969b5..d635a61380b 100644 --- a/tt_metal/hw/inc/wormhole/noc_nonblocking_api.h +++ b/tt_metal/hw/inc/wormhole/noc_nonblocking_api.h @@ -135,6 +135,9 @@ inline __attribute__((always_inline)) bool noc_cmd_buf_ready(uint32_t noc, uint3 template inline __attribute__((always_inline)) void ncrisc_noc_fast_read( uint32_t noc, uint32_t cmd_buf, uint64_t src_addr, uint32_t dest_addr, uint32_t len_bytes) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } if constexpr (noc_mode == DM_DYNAMIC_NOC) { uint32_t noc_rd_cmd_field = NOC_CMD_CPY | NOC_CMD_RD | NOC_CMD_RESP_MARKED | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(1); @@ -145,17 +148,16 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_read( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_TARG_ADDR_COORDINATE, (uint32_t)(src_addr >> NOC_ADDR_COORD_SHIFT)); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_reads_flushed(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_RD_RESP_RECEIVED); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::READS_NUM_ISSUED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_RD_RESP_RECEIVED) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_reads_flushed(uint32_t noc) { @@ -181,6 +183,14 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write( bool multicast_path_reserve, bool posted = false, uint32_t trid = 0) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + if (posted) { + inc_noc_counter_val(noc, 1); + } else { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, num_dests); + } + } uint32_t noc_cmd_field = NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(vc) | (linked ? NOC_CMD_VC_LINKED : 0x0) | (mcast ? ((multicast_path_reserve ? NOC_CMD_PATH_RESERVE : 0) | NOC_CMD_BRCST_PACKET) : 0x0) | @@ -197,16 +207,9 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if (posted) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { + if (posted) { noc_posted_writes_num_issued[noc] += 1; - } - } else { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, num_dests); } else { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += num_dests; @@ -226,6 +229,10 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_loopback_src( bool linked, uint32_t num_dests, bool multicast_path_reserve) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, num_dests); + } uint32_t noc_cmd_field = NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(vc) | (linked ? NOC_CMD_VC_LINKED : 0x0) | (mcast ? ((multicast_path_reserve ? NOC_CMD_PATH_RESERVE : 0) | NOC_CMD_BRCST_PACKET) : 0x0) | @@ -237,10 +244,7 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_loopback_src( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_RET_ADDR_COORDINATE, (uint32_t)(dest_addr >> NOC_ADDR_COORD_SHIFT)); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, num_dests); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += num_dests; } @@ -249,25 +253,27 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_write_loopback_src( template inline __attribute__((always_inline)) void ncrisc_noc_blitz_write_setup( uint32_t noc, uint32_t cmd_buf, uint64_t dest_addr, uint32_t len_bytes, uint32_t vc, uint32_t num_times_to_write) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, num_times_to_write); + inc_noc_counter_val(noc, num_times_to_write); + } uint32_t noc_cmd_field = NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_VC_STATIC | NOC_CMD_STATIC_VC(vc) | NOC_CMD_RESP_MARKED; while (!noc_cmd_buf_ready(noc, cmd_buf)); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CTRL, noc_cmd_field); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, len_bytes); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_RET_ADDR_COORDINATE, (uint32_t)(dest_addr >> NOC_ADDR_COORD_SHIFT)); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, num_times_to_write); - inc_noc_counter_val(noc, num_times_to_write); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_nonposted_writes_num_issued[noc] += num_times_to_write; noc_nonposted_writes_acked[noc] += num_times_to_write; } } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_nonposted_writes_sent(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_NONPOSTED_WR_REQ_SENT); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::NONPOSTED_WRITES_NUM_ISSUED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_NONPOSTED_WR_REQ_SENT) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_writes_sent(uint32_t noc) { @@ -275,9 +281,10 @@ inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_writes_sent(uint } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_posted_writes_sent(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_POSTED_WR_REQ_SENT); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::POSTED_WRITES_NUM_ISSUED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_POSTED_WR_REQ_SENT) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_posted_writes_sent(uint32_t noc) { @@ -285,9 +292,10 @@ inline __attribute__((always_inline)) bool ncrisc_noc_posted_writes_sent(uint32_ } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_nonposted_writes_flushed(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_WR_ACK_RECEIVED); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::NONPOSTED_WRITES_ACKED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_WR_ACK_RECEIVED) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_writes_flushed(uint32_t noc) { @@ -305,9 +313,10 @@ inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_write_with_trans } inline __attribute__((always_inline)) bool ncrisc_dynamic_noc_nonposted_atomics_flushed(uint32_t noc) { + uint32_t status_reg_val = NOC_STATUS_READ_REG(noc, NIU_MST_ATOMIC_RESP_RECEIVED); uint32_t self_risc_acked = get_noc_counter_val(noc); uint32_t other_risc_acked = get_noc_counter_val<1 - proc_type, NocBarrierType::NONPOSTED_ATOMICS_ACKED>(noc); - return (NOC_STATUS_READ_REG(noc, NIU_MST_ATOMIC_RESP_RECEIVED) == (self_risc_acked + other_risc_acked)); + return (status_reg_val == (self_risc_acked + other_risc_acked)); } inline __attribute__((always_inline)) bool ncrisc_noc_nonposted_atomics_flushed(uint32_t noc) { @@ -569,6 +578,14 @@ inline __attribute__((always_inline)) void noc_fast_write_dw_inline( uint32_t static_vc, bool mcast, bool posted = false) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + if (posted) { + inc_noc_counter_val(noc, 1); + } else { + inc_noc_counter_val(noc, 1); + inc_noc_counter_val(noc, 1); + } + } bool static_vc_alloc = true; uint32_t noc_cmd_field = (static_vc_alloc ? NOC_CMD_VC_STATIC : 0x0) | NOC_CMD_STATIC_VC(static_vc) | NOC_CMD_CPY | NOC_CMD_WR | NOC_CMD_WR_INLINE | @@ -588,16 +605,9 @@ inline __attribute__((always_inline)) void noc_fast_write_dw_inline( NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_LEN_BE, be32); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if (posted) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { + if (posted) { noc_posted_writes_num_issued[noc] += 1; - } - } else { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - inc_noc_counter_val(noc, 1); } else { noc_nonposted_writes_num_issued[noc] += 1; noc_nonposted_writes_acked[noc] += 1; @@ -616,6 +626,11 @@ inline __attribute__((always_inline)) void noc_fast_atomic_increment( bool linked, bool posted = false, uint32_t atomic_ret_val = 0) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + if (!posted) { + inc_noc_counter_val(noc, 1); + } + } while (!noc_cmd_buf_ready(noc, cmd_buf)); if constexpr (noc_mode == DM_DYNAMIC_NOC || program_ret_addr == true) { uint32_t noc_id_reg = NOC_CMD_BUF_READ_REG(noc, 0, NOC_NODE_ID); @@ -639,10 +654,8 @@ inline __attribute__((always_inline)) void noc_fast_atomic_increment( NOC_AT_INS(NOC_AT_INS_INCR_GET) | NOC_AT_WRAP(wrap) | NOC_AT_IND_32((addr >> 2) & 0x3) | NOC_AT_IND_32_SRC(0)); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_AT_DATA, incr); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, 0x1); - if (!posted) { - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { + if (!posted) { noc_nonposted_atomics_acked[noc] += 1; } } @@ -652,6 +665,9 @@ inline __attribute__((always_inline)) void noc_fast_atomic_increment( template inline __attribute__((always_inline)) void ncrisc_noc_fast_read_with_transaction_id( uint32_t noc, uint32_t cmd_buf, uint32_t src_base_addr, uint32_t src_addr, uint32_t dest_addr, uint32_t trid) { + if constexpr (noc_mode == DM_DYNAMIC_NOC) { + inc_noc_counter_val(noc, 1); + } uint32_t src_addr_; src_addr_ = src_base_addr + src_addr; @@ -661,9 +677,7 @@ inline __attribute__((always_inline)) void ncrisc_noc_fast_read_with_transaction NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_RET_ADDR_LO, dest_addr); NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_TARG_ADDR_LO, src_addr_); // (uint32_t)src_addr NOC_CMD_BUF_WRITE_REG(noc, cmd_buf, NOC_CMD_CTRL, NOC_CTRL_SEND_REQ); - if constexpr (noc_mode == DM_DYNAMIC_NOC) { - inc_noc_counter_val(noc, 1); - } else { + if constexpr (noc_mode == DM_DEDICATED_NOC) { noc_reads_num_issued[noc] += 1; } }