From 2b438523e08aabcb87b5b6269b8b3377851c6c0a Mon Sep 17 00:00:00 2001 From: jumao Date: Fri, 1 Sep 2023 17:27:06 -0400 Subject: [PATCH 1/8] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=09Saidump=20for?= =?UTF-8?q?=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimage/issu?= =?UTF-8?q?es/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, #1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null #2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" #3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null #4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" #5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- scripts/generate_dump | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index dd98302a27..1e4c6a0dd4 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -874,11 +874,11 @@ save_redis() { save_saidump() { trap 'handle_error $? $LINENO' ERR if [[ ( "$NUM_ASICS" == 1 ) ]] ; then - save_cmd "docker exec syncd saidump" "saidump" + save_cmd "saidump.sh" "saidump" else for (( i=0; i<$NUM_ASICS; i++ )) do - save_cmd "docker exec syncd$i saidump" "saidump$i" + save_cmd "saidump.sh $i" "saidump$i" done fi } From 62aab2df7e6d99f51a0f80db8ece015bc920a8e8 Mon Sep 17 00:00:00 2001 From: jumao Date: Fri, 1 Sep 2023 17:27:06 -0400 Subject: [PATCH 2/8] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=09Saidump=20for?= =?UTF-8?q?=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimage/issu?= =?UTF-8?q?es/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- scripts/generate_dump | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index dd98302a27..1e4c6a0dd4 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -874,11 +874,11 @@ save_redis() { save_saidump() { trap 'handle_error $? $LINENO' ERR if [[ ( "$NUM_ASICS" == 1 ) ]] ; then - save_cmd "docker exec syncd saidump" "saidump" + save_cmd "saidump.sh" "saidump" else for (( i=0; i<$NUM_ASICS; i++ )) do - save_cmd "docker exec syncd$i saidump" "saidump$i" + save_cmd "saidump.sh $i" "saidump$i" done fi } From 3504fdca51cf0333cd0ae316f95dca4dfa8fe8df Mon Sep 17 00:00:00 2001 From: jumao Date: Tue, 5 Sep 2023 17:19:56 -0400 Subject: [PATCH 3/8] =?UTF-8?q?=20=20=20=20*=20[saidump]=20=20=20=20=20?= =?UTF-8?q?=E2=80=A2=20=20=20=20=20=20=20Saidump=20for=20DNX-SAI=20https:/?= =?UTF-8?q?/github.com/sonic-net/sonic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- scripts/generate_dump | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/scripts/generate_dump b/scripts/generate_dump index 1e4c6a0dd4..0c98749bb8 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -872,6 +872,27 @@ save_redis() { # None ############################################################################### save_saidump() { + trap 'handle_error $? $LINENO' ERR + if [[ ( "$NUM_ASICS" == 1 ) ]] ; then + save_cmd "docker exec syncd saidump" "saidump" + else + for (( i=0; i<$NUM_ASICS; i++ )) + do + save_cmd "docker exec syncd$i saidump" "saidump$i" + done + fi +} + +############################################################################### +# SAI DUMP from syncd by Redis Save command +# Globals: +# NUM_ASICS +# Arguments: +# None +# Returns: +# None +############################################################################### +save_saidump_by_save_cmd() { trap 'handle_error $? $LINENO' ERR if [[ ( "$NUM_ASICS" == 1 ) ]] ; then save_cmd "saidump.sh" "saidump" @@ -1791,6 +1812,8 @@ main() { if [[ "$device_type" != "SpineRouter" ]]; then save_saidump + else + save_saidump_by_save_cmd fi if [ "$asic" = "barefoot" ]; then From 2450c4879a8640fa83325ddcf34ce4ad3d4c5531 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 12:06:16 -0400 Subject: [PATCH 4/8] =?UTF-8?q?=20=20=20=20=20=20=20=20*=20[saidump]=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=E2=80=A2=20=20=20=20=20=20=20Saidump=20fo?= =?UTF-8?q?r=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimage/iss?= =?UTF-8?q?ues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- scripts/generate_dump | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index 0c98749bb8..b5342b9233 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -892,7 +892,7 @@ save_saidump() { # Returns: # None ############################################################################### -save_saidump_by_save_cmd() { +save_saidump_by_redis_save_cmd() { trap 'handle_error $? $LINENO' ERR if [[ ( "$NUM_ASICS" == 1 ) ]] ; then save_cmd "saidump.sh" "saidump" @@ -1813,7 +1813,7 @@ main() { if [[ "$device_type" != "SpineRouter" ]]; then save_saidump else - save_saidump_by_save_cmd + save_saidump_by_redis_save_cmd fi if [ "$asic" = "barefoot" ]; then From 9fa769e846bd995c284c203f15bd1606a73f4a39 Mon Sep 17 00:00:00 2001 From: jumao Date: Tue, 19 Sep 2023 00:04:51 -0400 Subject: [PATCH 5/8] Fixup based on the below PR comments. https://github.com/sonic-net/sonic-utilities/pull/2972 SAI DUMP based on the route table size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [saidump] • Saidump for DNX-SAI https://github.com/sonic-net/sonic-buildimage/issues/13561 Solution and modification: To use the Redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2, install Python library rdbtools into the syncd containter. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Updated sonic-buildimage/build_debian.sh, to add a new script file: files/scripts/saidump.sh into the host. This shell file does the below steps: For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files docker exec syncd$1 sh -c "rdb --command json /var/run/redis$1/dump.rdb | tee /var/run/redis$1/dump.json > /dev/null" 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json -m 100" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, to check the asic db size and if it is larger than xxx entries, then do with REDIS SAVE, otherwise, to do with old method: looping through each entry of Redis DB. --- scripts/generate_dump | 117 ++++++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 28 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index b5342b9233..f2be984b13 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -50,6 +50,7 @@ SKIP_BCMCMD=0 SAVE_STDERR=true RETURN_CODE=$EXT_SUCCESS DEBUG_DUMP=false +ROUTE_TAB_LIMIT_DIRECT_ITERATION=6400 # lock dirs/files LOCKDIR="/tmp/techsupport-lock" @@ -863,45 +864,109 @@ save_redis() { } ############################################################################### -# SAI DUMP from syncd +# GET ROUTE table size by ASIC id and ip version # Globals: -# NUM_ASICS +# TIMEOUT_MIN +# TIMEOUT_EXIT_CODE # Arguments: -# None +# asic id +# IP version # Returns: -# None -############################################################################### -save_saidump() { - trap 'handle_error $? $LINENO' ERR - if [[ ( "$NUM_ASICS" == 1 ) ]] ; then - save_cmd "docker exec syncd saidump" "saidump" +# route +############################################################################### +get_route_table_size_by_asic_id_and_ipver() { + local asic_id="$1" + local ip_ver="$2" + local filepath="/tmp/route_summary.txt" + RC=0 + + if [ $ip_ver = "ipv4" ]; then + cmd="vtysh -n $asic_id -c 'show ip route summary json'" + elif [ $ip_ver = "ipv6" ]; then + cmd="vtysh -n $asic_id -c 'show ipv6 route summary json'" else - for (( i=0; i<$NUM_ASICS; i++ )) - do - save_cmd "docker exec syncd$i saidump" "saidump$i" - done + echo "Wrong argument $ip_ver." + return 255 + fi + + + local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m" + local cmds="$cmd > '$filepath'" + + eval "${timeout_cmd} bash -c \"${cmds}\"" || RC=$? + if [ $RC -eq $TIMEOUT_EXIT_CODE ]; then + echo "Command: $cmds timedout after ${TIMEOUT_MIN} minutes." + return $RC + elif [ $RC -ne 0 ]; then + echo "Command: $cmds failed with RC $RC" + return $RC fi + + local route_tab_size=$(python3 -c "\ +import json +with open('$filepath') as json_file: + data = json.load(json_file) +print(data['routesTotal'])") + rm $filepath + echo "$route_tab_size" } ############################################################################### -# SAI DUMP from syncd by Redis Save command +# SAI DUMP based on the route table size +# if the route table has more than ROUTE_TAB_LIMIT_DIRECT_ITERATION +# then dump by Redis Save command, +# otherwize, dump it by directly iteration the Redis +# # Globals: # NUM_ASICS +# ROUTE_TAB_LIMIT_DIRECT_ITERATION # Arguments: # None # Returns: # None ############################################################################### -save_saidump_by_redis_save_cmd() { +save_saidump_by_route_size() { trap 'handle_error $? $LINENO' ERR - if [[ ( "$NUM_ASICS" == 1 ) ]] ; then - save_cmd "saidump.sh" "saidump" - else - for (( i=0; i<$NUM_ASICS; i++ )) - do - save_cmd "saidump.sh $i" "saidump$i" - done - fi + + for (( i=0; i<$NUM_ASICS; i++ )) + do + route_size_ipv4=`get_route_table_size_by_asic_id_and_ipver $i ipv4` + ret=$? + + if [ $ret -ne 0 ]; then + echo "Get route table's size by asicid $i and ipv4 failed." + return $ret + fi + + route_size_ipv6=`get_route_table_size_by_asic_id_and_ipver $i ipv6` + ret=$? + + if [ $ret -ne 0 ]; then + echo "Get route table's size by asicid $i and ipv6 failed." + return $ret + fi + + route_size=`expr $route_size_ipv4 + $route_size_ipv6` + echo "The route table's size is $route_size(ipv4 $route_size_ipv4, ipv6 $route_size_ipv6)" + + if [[ $route_size -gt $ROUTE_TAB_LIMIT_DIRECT_ITERATION ]]; then + echo "Dump by using Redis SAVE." + + if [[ ( "$NUM_ASICS" == 1 ) ]] ; then + save_cmd "saidump.sh" "saidump" + else + save_cmd "saidump.sh $i" "saidump$i" + fi + else + echo "Dump by using direct iteration of Redis DB." + + if [[ ( "$NUM_ASICS" == 1 ) ]] ; then + save_cmd "docker exec syncd saidump" "saidump" + else + save_cmd "docker exec syncd$i saidump" "saidump$i" + fi + fi + done } ############################################################################### @@ -1810,11 +1875,7 @@ main() { save_cmd "ps -AwwL -o user,pid,lwp,ppid,nlwp,pcpu,pri,nice,vsize,rss,tty,stat,wchan:12,start,bsdtime,command" "ps.extended" & wait - if [[ "$device_type" != "SpineRouter" ]]; then - save_saidump - else - save_saidump_by_redis_save_cmd - fi + save_saidump_by_route_size if [ "$asic" = "barefoot" ]; then collect_barefoot From 8a3f93b1c000af3e71d4d88b0e1f81a8c3cad675 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 20 Sep 2023 15:32:13 -0400 Subject: [PATCH 6/8] According to the testing group's advice, change the default ROUTE_TAB_LIMIT_DIRECT_ITERATION to 24000 --- scripts/generate_dump | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index f2be984b13..194c332bfd 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -50,7 +50,7 @@ SKIP_BCMCMD=0 SAVE_STDERR=true RETURN_CODE=$EXT_SUCCESS DEBUG_DUMP=false -ROUTE_TAB_LIMIT_DIRECT_ITERATION=6400 +ROUTE_TAB_LIMIT_DIRECT_ITERATION=24000 # lock dirs/files LOCKDIR="/tmp/techsupport-lock" From 32f56078a5c12e9a7c8f8191bb0ac4cc51b06855 Mon Sep 17 00:00:00 2001 From: jumao Date: Sat, 30 Sep 2023 01:20:23 -0400 Subject: [PATCH 7/8] https://github.com/sonic-net/sonic-buildimage/pull/16466 fixing based on review comments: To add the rdbtools into base docker. Move saidump.sh from host to syncd docker container. --- scripts/generate_dump | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index 194c332bfd..780997bd6e 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -872,7 +872,7 @@ save_redis() { # asic id # IP version # Returns: -# route +# Status: 0 success, otherwise failure ############################################################################### get_route_table_size_by_asic_id_and_ipver() { local asic_id="$1" @@ -953,9 +953,9 @@ save_saidump_by_route_size() { echo "Dump by using Redis SAVE." if [[ ( "$NUM_ASICS" == 1 ) ]] ; then - save_cmd "saidump.sh" "saidump" + save_cmd "docker exec syncd saidump.sh" "saidump" else - save_cmd "saidump.sh $i" "saidump$i" + save_cmd "docker exec syncd$i saidump.sh" "saidump$i" fi else echo "Dump by using direct iteration of Redis DB." From 867a6d1251f2dfb14dcaf40b8ab4715110ad3805 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 4 Oct 2023 11:17:56 -0400 Subject: [PATCH 8/8] To address the below review comments: https://github.com/sonic-net/sonic-utilities/pull/2972#discussion_r1345011573 --- scripts/generate_dump | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index 42604c6026..2179c17ca8 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -878,22 +878,27 @@ get_route_table_size_by_asic_id_and_ipver() { local asic_id="$1" local ip_ver="$2" local filepath="/tmp/route_summary.txt" + local ns="" RC=0 + if [[ $NUM_ASICS -gt 1 ]] ; then + ns="-n ${asic_id}" + fi + if [ $ip_ver = "ipv4" ]; then - cmd="vtysh -n $asic_id -c 'show ip route summary json'" + cmd="vtysh ${ns} -c 'show ip route summary json'" elif [ $ip_ver = "ipv6" ]; then - cmd="vtysh -n $asic_id -c 'show ipv6 route summary json'" + cmd="vtysh ${ns} -c 'show ipv6 route summary json'" else echo "Wrong argument $ip_ver." return 255 fi - local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m" local cmds="$cmd > '$filepath'" eval "${timeout_cmd} bash -c \"${cmds}\"" || RC=$? + if [ $RC -eq $TIMEOUT_EXIT_CODE ]; then echo "Command: $cmds timedout after ${TIMEOUT_MIN} minutes." return $RC @@ -906,7 +911,7 @@ get_route_table_size_by_asic_id_and_ipver() { import json with open('$filepath') as json_file: data = json.load(json_file) -print(data['routesTotal'])") + print(data['routesTotal'])") rm $filepath echo "$route_tab_size" }