Merge pull request #15885 from kubernetes/revert-15761-python3

Revert "Move hack job to python3 and enable PY3 linting"

Author: k8s-ci-robot

WORKSPACE

@@ -68,22 +68,11 @@ load("@npm_bazel_typescript//:index.bzl", "ts_setup_workspace")
 
 ts_setup_workspace()
 
-load("@rules_python//python:pip.bzl", "pip_import")
+load("@io_bazel_rules_python//python:pip.bzl", "pip_import")
 
 pip_import(
     name = "py_deps",
-    python_interpreter = "python2",
-    requirements = "//:requirements2.txt",
-)
-
-load("@py_deps//:requirements.bzl", "pip_install")
-
-pip_install()
-
-pip_import(
-    name = "py3_deps",
-    python_interpreter = "python3",
-    requirements = "//:requirements3.txt",
+    requirements = "//:requirements.txt",
 )
 
 load("//:py.bzl", "python_repos")

experiment/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@py3_deps//:requirements.bzl", "requirement")
+load("@py_deps//:requirements.bzl", "requirement")
 
 py_binary(
     name = "flakedetector",

experiment/find_developers.py

@@ -43,7 +43,7 @@ def load_content(data):
 
 
 @functools.total_ordering
-class User:  # pylint: disable=too-few-public-methods
+class User(object):  # pylint: disable=too-few-public-methods
     """Store .user and number of .total and .recent commits."""
     def __init__(self, blob):
         self.user = blob['author']['login']

experiment/graphql_issue_example.py

@@ -16,6 +16,9 @@
 
 # USAGE: find_issues.py <github_token>
 
+# Required for pylint: 1.9.4 to tokenize the python3 print function.
+from __future__ import print_function
+
 import sys
 import json
 import argparse

experiment/parse_build_log.py

@@ -29,7 +29,7 @@
 _CURRENT_YEAR = datetime.datetime.utcnow().year
 
 
-class TestOutput:
+class TestOutput(object):
     def __init__(self):
         self._lines = []
         self._start = None
@@ -81,7 +81,8 @@ def _get_tests(log):
             current_test = TestOutput()
             if len(ended_test) <= 1:
                 continue
-            yield ended_test
+            else:
+                yield ended_test
         else:
             current_test.append(match.group(1))
     yield current_test

hack/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@py3_deps//:requirements.bzl", "requirement")
+load("@py_deps//:requirements.bzl", "requirement")
 load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary", "nodejs_test")
 
 package(default_visibility = ["//visibility:public"])
@@ -226,13 +226,14 @@ test_suite(
 py_binary(
     name = "pylint_bin",
     srcs = ["pylint_bin.py"],
-    python_version = "PY3",
+    python_version = "PY2",
     tags = ["lint"],
     # NOTE: this should only contain direct third party imports and pylint
     deps = [
         requirement("astroid"),
         requirement("backports.functools_lru_cache"),
         requirement("configparser"),
+        requirement("enum34"),
         requirement("influxdb"),
         requirement("isort"),
         requirement("lazy-object-proxy"),

hack/pylint_bin.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python2
 
 # Copyright 2017 The Kubernetes Authors.
 #

hack/verify-pylint.sh

@@ -36,5 +36,6 @@ export PYLINTHOME=$TEST_TMPDIR
 
 shopt -s extglob globstar
 
-# TODO(clarketm): remove `boskos` exclusion after upgrading to PY3.
-"$DIR/pylint_bin" !(gubernator|external|vendor|jenkins|scenarios|triage|boskos|bazel-*)/**/*.py
+# TODO(clarketm) there is no version of `pylint` that supports "both" PY2 and PY3
+# I am disabling pylint checks for python3 files until migration complete
+"$DIR/pylint_bin" !(kettle|metrics|triage|velodrome|hack|gubernator|external|vendor|testgrid|bazel-*)/**/*.py

kettle/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@py3_deps//:requirements.bzl", "requirement")
+load("@py_deps//:requirements.bzl", "requirement")
 
 py_test(
     name = "make_db_test",

kettle/make_json.py

@@ -81,16 +81,14 @@ def make_result(name, time, failure_text):
 
 
 def buckets_yaml():
-    import ruamel.yaml as yaml  # pylint: disable=import-outside-toplevel
+    import ruamel.yaml as yaml  # does not support pypy
     with open(os.path.dirname(os.path.abspath(__file__))+'/buckets.yaml') as fp:
         return yaml.safe_load(fp)
 
 # pypy compatibility hack
 def python_buckets_yaml(python='python3'):
     return json.loads(subprocess.check_output(
-        [python, '-c',
-         'import json, ruamel.yaml as yaml; print(json.dumps(yaml.safe_load(open("buckets.yaml"))))'
-         ],
+        [python, '-c', 'import json, ruamel.yaml as yaml; print(json.dumps(yaml.safe_load(open("buckets.yaml"))))'],
         cwd=os.path.dirname(os.path.abspath(__file__))).decode("utf-8"))
 
 for attempt in [python_buckets_yaml, buckets_yaml, lambda: python_buckets_yaml(python='python')]:

kettle/make_json_test.py

@@ -135,7 +135,7 @@ def expect(args, needles, negneedles, expected_ret=None):
             for needle in negneedles:
                 # Only match negative needles in the middle of a word, to avoid
                 # failures on timestamps that happen to contain a short number.
-                self.assertNotRegexpMatches(result, r'\b%s\b' % needle) # pylint: disable=deprecated-method
+                self.assertNotRegexpMatches(result, r'\b%s\b' % needle)
 
         add_build('some-job/123', last_month, last_month + 10, 'SUCCESS', junits)
         add_build('some-job/456', now - 10, now, 'FAILURE', junits)

kettle/stream.py

@@ -69,16 +69,15 @@ def get_started_finished(gcs_client, db, todo):
     pool = multiprocessing.pool.ThreadPool(16)
     try:
         for ack_id, (build_dir, started, finished) in pool.imap_unordered(
-                lambda ack_id_job_build: (ack_id_job_build[0], gcs_client.get_started_finished(
-                    ack_id_job_build[1], ack_id_job_build[2])),
+                lambda ack_id_job_build: (ack_id_job_build[0], gcs_client.get_started_finished(ack_id_job_build[1], ack_id_job_build[2])),
                 todo):
             if finished:
                 if not db.insert_build(build_dir, started, finished):
                     print('already present??')
                 start = time.localtime(started.get('timestamp', 0) if started else 0)
                 print((build_dir, bool(started), bool(finished),
-                       time.strftime('%F %T %Z', start),
-                       finished and finished.get('result')))
+                      time.strftime('%F %T %Z', start),
+                      finished and finished.get('result')))
                 build_dirs.append(build_dir)
                 acks.append(ack_id)
             else:

load.bzl

@@ -48,10 +48,10 @@ def repositories():
     # Python setup
     # pip_import() calls must live in WORKSPACE, otherwise we get a load() after non-load() error
     git_repository(
-        name = "rules_python",
-        commit = "94677401bc56ed5d756f50b441a6a5c7f735a6d4",
+        name = "io_bazel_rules_python",
+        commit = "9d68f24659e8ce8b736590ba1e4418af06ec2552",
         remote = "https://github.com/bazelbuild/rules_python.git",
-        shallow_since = "1573842889 -0500",
+        shallow_since = "1565801665 -0400",
     )
 
     # TODO(fejta): get this to work

metrics/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@py3_deps//:requirements.bzl", "requirement")
+load("@py_deps//:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

metrics/bigquery.py

@@ -65,7 +65,7 @@ def do_jq(jq_filter, data_filename, out_filename, jq_bin='jq'):
         check([jq_bin, jq_filter, data_filename], stdout=out_file)
 
 
-class BigQuerier:
+class BigQuerier(object):
     def __init__(self, project, bucket_path, backfill_days, influx_client):
         if not project:
             raise ValueError('project', project)

py.bzl

@@ -15,7 +15,7 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
 load("@io_bazel_rules_appengine//appengine:py_appengine.bzl", "py_appengine_repositories")
 load(
-    "@py3_deps//:requirements.bzl",
+    "@py_deps//:requirements.bzl",
     "pip_install",
 )
 

releng/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@py3_deps//:requirements.bzl", "requirement")
+load("@py_deps//:requirements.bzl", "requirement")
 
 py_binary(
     name = "generate_tests",

releng/generate_tests.py

@@ -107,7 +107,7 @@ def apply_job_overrides(envs_or_args, job_envs_or_args):
         envs_or_args.append(job_env_or_arg)
 
 
-class E2ENodeTest:
+class E2ENodeTest(object):
 
     def __init__(self, job_name, job, config):
         self.job_name = job_name
@@ -214,10 +214,10 @@ def generate(self):
         return job_config, prow_config, None
 
 
-class E2ETest:
+class E2ETest(object):
 
     def __init__(self, output_dir, job_name, job, config):
-        self.env_filename = os.path.join(output_dir, '%s.env' % job_name)
+        self.env_filename = os.path.join(output_dir, '%s.env' % job_name),
         self.job_name = job_name
         self.job = job
         self.common = config['common']

requirements.txt

@@ -0,0 +1,28 @@
+astroid==1.6.6
+backports.functools_lru_cache==1.5
+certifi==2019.6.16
+chardet==3.0.4
+configparser==3.8.1
+enum34==1.1.6
+google-cloud-bigquery==1.18.0
+google-cloud-pubsub==0.45.0
+idna==2.8
+influxdb==5.2.2
+isort==4.3.21
+jinja2==2.10.1
+lazy-object-proxy==1.4.1
+MarkupSafe==1.1.1
+mccabe==0.6.1
+pylint==1.9.4
+python-dateutil==2.8.0
+pytz==2019.2
+PyYAML==5.1.1
+requests==2.22.0
+ruamel.yaml==0.15.98
+setuptools==41.1.0
+sh==1.12.14
+singledispatch==3.4.0.3
+six==1.12.0
+urllib3==1.25.3
+webapp2==2.5.2
+wrapt==1.11.2

requirements2.txt

@@ -32,6 +32,8 @@
 #
 # Usage: see README.md
 
+# Required for pylint: 1.9.4 to tokenize the python3 print function.
+from __future__ import print_function
 
 import re
 import sys
@@ -110,7 +112,7 @@ def parse_e2e_logfile(file_handle, year):
         if passed is False:
             # if we already have found a failure, ignore subsequent pass/fails
             continue
-        if E2E_LOG_SUCCESS_RE.match(line):
+        elif E2E_LOG_SUCCESS_RE.match(line):
             passed = True
         elif E2E_LOG_FAIL_RE.match(line):
             passed = False

requirements3.txt

@@ -1,4 +1,4 @@
-load("@py3_deps//:requirements.bzl", "requirement")
+load("@py_deps//:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])