Migrate the bm_diff benchmarks to python3 (grpc#25619)

* Migrate the bm_diff benchmarks to python3

Includes a requirements.txt pinned at ~2017 versions, when this script
was first written.

* Replace p2 with p3 scipy/numpy dependencies.

* py2->3 for benchmark setup scripts

* upgrade pip to resolve python3 cryptography/setuptools-rust problem

* re-add jobset import (accidentally removed)

* re-add six's urllib import. This file is still used in py2 tests

* force py3 in run_if_c_cpp_modified.sh

* Fix another instance of subprocess.check_output binary mode

* Use the requirements.txt for CI perf environment setup

* Try to upgrade PyJWT. (v2.0.0 was problematic, grpc#25053)

v2.x makes encode return strs from jwt.encode in both py2 and py3.
Previously, py3 would return bytes, and py2 a str.

* upgate cryptography lib version requirements for jwt.

* Wrap pip requirements specifier in quotes '>=x,<y'

* Decode subprocess output once instead of for every line

* Revert "Decode subprocess output once instead of for every line"

This reverts commit 28d1402.

py2 doesn't support the `text` argument to subprocess.check_output.

* Address reviewer requests

* Pin a valid scipy version

* Remove scipy and tabulate dependencies from macos tests

Author: drfloob

tools/gce/linux_kokoro_performance_worker_init.sh

@@ -200,7 +200,7 @@ echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb
 git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph
 
 # Install scipy and numpy for benchmarking scripts
-sudo apt-get install -y python-scipy python-numpy
+sudo apt-get install -y python3-scipy python3-numpy
 
 # Install docker
 curl -sSL https://get.docker.com/ | sh

tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc

@@ -19,9 +19,10 @@
 ulimit -n 32768
 ulimit -c unlimited
 
-sudo pip install tabulate
+python3 -m pip install pip==19.3.1
 
 # Python dependencies for tools/run_tests/python_utils/check_on_pr.py
-time python2.7 -m pip install pyjwt cryptography requests --user
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+time python3 -m pip install --user -r $DIR/requirements.linux_perf.txt
 
 git submodule update --init

tools/internal_ci/helper_scripts/prepare_build_macos_rc

@@ -37,6 +37,7 @@ brew config
 # Add GCP credentials for BQ access
 pip install --user google-api-python-client oauth2client six==1.15.0
 export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
 # If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests
 if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then
@@ -67,7 +68,7 @@ then
   time git clone --depth 1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master
 
   # Needed for ios-binary-size
-  time pip install --user pyyaml pyjwt==1.7.1 pyOpenSSL cryptography requests
+  time pip install --user -r $DIR/requirements.macos.txt
 
   # Store intermediate build files of ObjC tests into /tmpfs
   # TODO(jtattermusch): this has likely been done to avoid running
@@ -84,8 +85,8 @@ fi
 if [ "${PREPARE_BUILD_INSTALL_DEPS_PYTHON}" == "true" ]
 then
   # python
-  time pip install --user virtualenv
-  time pip install --user --upgrade Mako tox setuptools==44.1.1 twisted pyyaml pyjwt==1.7.1 pyOpenSSL cryptography requests
+  time pip install --user -r $DIR/requirements.macos.txt
+  time pip install --user --upgrade virtualenv Mako tox setuptools==44.1.1 twisted
 
   # Install Python 3.7 if it doesn't exist
   if [ ! -f "/usr/local/bin/python3.7" ]; then

tools/internal_ci/helper_scripts/requirements.linux_perf.txt

@@ -0,0 +1,5 @@
+cryptography==3.4.6
+PyJWT==2.0.1
+requests==2.25.1
+scipy==1.5.4
+tabulate==0.8.9

tools/internal_ci/helper_scripts/requirements.macos.txt

@@ -0,0 +1,5 @@
+cryptography==3.4.6
+PyJWT==2.0.1
+pyOpenSSL==20.0.1
+PyYAML==5.4.1
+requests==2.25.1

tools/internal_ci/linux/grpc_performance_profile_daily.sh

@@ -20,7 +20,7 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
 
-CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'`
+CPUS=`python3 -c 'import multiprocessing; print(multiprocessing.cpu_count())'`
 
 ./tools/run_tests/start_port_server.py || true
 

tools/internal_ci/linux/run_if_c_cpp_modified.sh

@@ -20,7 +20,7 @@ set -ex
 # Enter the gRPC repo root
 cd $(dirname $0)/../../..
 
-AFFECTS_C_CPP=`python -c 'import os; \
+AFFECTS_C_CPP=`python3 -c 'import os; \
                import sys; \
                sys.path.insert(0, "tools/run_tests/python_utils"); \
                import filter_pull_request_tests as filter; \

tools/profiling/bloat/bloat_diff.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #

tools/profiling/microbenchmarks/bm2bq.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #
@@ -41,7 +41,7 @@
 }
 
 if sys.argv[1] == '--schema':
-    print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns)
+    print(',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns))
     sys.exit(0)
 
 with open(sys.argv[1]) as f:

tools/profiling/microbenchmarks/bm_diff/bm_build.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #
@@ -15,13 +15,13 @@
 # limitations under the License.
 """ Python utility to build opt and counters benchmarks """
 
-import bm_constants
-
 import argparse
-import subprocess
 import multiprocessing
 import os
 import shutil
+import subprocess
+
+import bm_constants
 
 
 def _args():

tools/profiling/microbenchmarks/bm_diff/bm_constants.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #

tools/profiling/microbenchmarks/bm_diff/bm_diff.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #
@@ -15,20 +15,19 @@
 # limitations under the License.
 """ Computes the diff between two bm runs and outputs significant results """
 
-import bm_constants
-import bm_speedup
-
-import sys
+import argparse
+import collections
+import json
 import os
+import subprocess
+import sys
 
 sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
-import bm_json
 
-import json
+import bm_constants
+import bm_json
+import bm_speedup
 import tabulate
-import argparse
-import collections
-import subprocess
 
 verbose = False
 
@@ -38,9 +37,9 @@ def _median(ary):
     ary = sorted(ary)
     n = len(ary)
     if n % 2 == 0:
-        return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
+        return (ary[(n - 1) // 2] + ary[(n - 1) // 2 + 1]) / 2.0
     else:
-        return ary[n / 2]
+        return ary[n // 2]
 
 
 def _args():
@@ -91,7 +90,7 @@ def _args():
 
 def _maybe_print(str):
     if verbose:
-        print str
+        print(str)
 
 
 class Benchmark:
@@ -136,14 +135,14 @@ def _read_json(filename, badjson_files, nonexistant_files):
         with open(filename) as f:
             r = f.read()
             return json.loads(r)
-    except IOError, e:
+    except IOError as e:
         if stripped in nonexistant_files:
             nonexistant_files[stripped] += 1
         else:
             nonexistant_files[stripped] = 1
         return None
-    except ValueError, e:
-        print r
+    except ValueError as e:
+        print(r)
         if stripped in badjson_files:
             badjson_files[stripped] += 1
         else:
@@ -166,6 +165,7 @@ def diff(bms, loops, regex, track, old, new, counters):
                     'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests',
                     '--benchmark_filter=%s' % regex
             ]).splitlines():
+                line = line.decode('UTF-8')
                 stripped_line = line.strip().replace("/", "_").replace(
                     "<", "_").replace(">", "_").replace(", ", "_")
                 js_new_opt = _read_json(

tools/profiling/microbenchmarks/bm_diff/bm_main.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #
@@ -15,26 +15,26 @@
 # limitations under the License.
 """ Runs the entire bm_*.py pipeline, and possible comments on the PR """
 
-import bm_constants
-import bm_build
-import bm_run
-import bm_diff
-
-import sys
-import os
-import random
 import argparse
 import multiprocessing
+import os
+import random
 import subprocess
+import sys
 
 sys.path.append(
     os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests',
                  'python_utils'))
-import check_on_pr
 
 sys.path.append(
     os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
                  'python_utils'))
+
+import bm_build
+import bm_constants
+import bm_diff
+import bm_run
+import check_on_pr
 import jobset
 
 

tools/profiling/microbenchmarks/bm_diff/bm_run.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #
@@ -15,20 +15,20 @@
 # limitations under the License.
 """ Python utility to run opt and counters benchmarks and save json output """
 
-import bm_constants
-
 import argparse
-import subprocess
+import itertools
 import multiprocessing
+import os
 import random
-import itertools
+import subprocess
 import sys
-import os
+
+import bm_constants
+import jobset
 
 sys.path.append(
     os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
                  'python_utils'))
-import jobset
 
 
 def _args():
@@ -70,7 +70,8 @@ def _args():
     args = argp.parse_args()
     assert args.name
     if args.loops < 3:
-        print "WARNING: This run will likely be noisy. Increase loops to at least 3."
+        print("WARNING: This run will likely be noisy. Increase loops to at "
+              "least 3.")
     return args
 
 
@@ -80,6 +81,7 @@ def _collect_bm_data(bm, cfg, name, regex, idx, loops):
             'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_list_tests',
             '--benchmark_filter=%s' % regex
     ]).splitlines():
+        line = line.decode('UTF-8')
         stripped_line = line.strip().replace("/",
                                              "_").replace("<", "_").replace(
                                                  ">", "_").replace(", ", "_")

tools/profiling/microbenchmarks/bm_diff/bm_speedup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # Copyright 2017 gRPC authors.
 #
@@ -14,9 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from scipy import stats
 import math
 
+from scipy import stats
+
 _DEFAULT_THRESHOLD = 1e-10
 
 
@@ -63,5 +64,5 @@ def speedup(new, old, threshold=_DEFAULT_THRESHOLD):
 if __name__ == "__main__":
     new = [0.0, 0.0, 0.0, 0.0]
     old = [2.96608e-06, 3.35076e-06, 3.45384e-06, 3.34407e-06]
-    print speedup(new, old, 1e-5)
-    print speedup(old, new, 1e-5)
+    print(speedup(new, old, 1e-5))
+    print(speedup(old, new, 1e-5))

tools/profiling/microbenchmarks/bm_json.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

tools/run_tests/python_utils/check_on_pr.py

@@ -55,7 +55,7 @@ def _access_token():
                 url='https://api.github.com/app/installations/%s/access_tokens'
                 % _INSTALLATION_ID,
                 headers={
-                    'Authorization': 'Bearer %s' % _jwt_token().decode('ASCII'),
+                    'Authorization': 'Bearer %s' % _jwt_token(),
                     'Accept': 'application/vnd.github.machine-man-preview+json',
                 })
 

tools/run_tests/python_utils/filter_pull_request_tests.py

@@ -18,7 +18,7 @@
 
 import re
 import six
-from subprocess import check_output
+import subprocess
 
 
 class TestSuite:
@@ -126,10 +126,11 @@ def _get_changed_files(base_branch):
   """
     # Get file changes between branch and merge-base of specified branch
     # Not combined to be Windows friendly
-    base_commit = check_output(["git", "merge-base", base_branch,
-                                "HEAD"]).rstrip()
-    return check_output(["git", "diff", base_commit, "--name-only",
-                         "HEAD"]).splitlines()
+    base_commit = subprocess.check_output(
+        ["git", "merge-base", base_branch, "HEAD"]).decode("UTF-8").rstrip()
+    return subprocess.check_output(
+        ["git", "diff", base_commit, "--name-only",
+         "HEAD"]).decode("UTF-8").splitlines()
 
 
 def _can_skip_tests(file_names, triggers):

tools/run_tests/python_utils/start_port_server.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # Copyright 2015 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

tools/run_tests/run_microbenchmark.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -96,6 +96,7 @@ def collect_latency(bm_name, args):
             'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
             '--benchmark_list_tests'
     ]).splitlines():
+        line = line.decode('UTF-8')
         link(line, '%s.txt' % fnize(line))
         benchmarks.append(
             jobset.JobSpec([
@@ -150,6 +151,7 @@ def collect_perf(bm_name, args):
             'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
             '--benchmark_list_tests'
     ]).splitlines():
+        line = line.decode('UTF-8')
         link(line, '%s.svg' % fnize(line))
         benchmarks.append(
             jobset.JobSpec([