Again - move hack job to python3 and enable PY3 linting

matthyx · matthyx · commit 596fe77f2a00 · 2020-01-13T22:10:22.000+01:00
This reverts commit ed64df7.
diff --git a/WORKSPACE b/WORKSPACE
@@ -68,11 +68,22 @@ load("@npm_bazel_typescript//:index.bzl", "ts_setup_workspace")
 
 ts_setup_workspace()
 
-load("@io_bazel_rules_python//python:pip.bzl", "pip_import")
+load("@rules_python//python:pip.bzl", "pip_import")
 
 pip_import(
     name = "py_deps",
-    requirements = "//:requirements.txt",
+    python_interpreter = "python2",
+    requirements = "//:requirements2.txt",
+)
+
+load("@py_deps//:requirements.bzl", "pip_install")
+
+pip_install()
+
+pip_import(
+    name = "py3_deps",
+    python_interpreter = "python3",
+    requirements = "//:requirements3.txt",
 )
 
 load("//:py.bzl", "python_repos")
diff --git a/experiment/BUILD.bazel b/experiment/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@py_deps//:requirements.bzl", "requirement")
+load("@py3_deps//:requirements.bzl", "requirement")
 
 py_binary(
     name = "flakedetector",
diff --git a/experiment/find_developers.py b/experiment/find_developers.py
@@ -43,7 +43,7 @@ def load_content(data):
 
 
 @functools.total_ordering
-class User(object):  # pylint: disable=too-few-public-methods
+class User:  # pylint: disable=too-few-public-methods
     """Store .user and number of .total and .recent commits."""
     def __init__(self, blob):
         self.user = blob['author']['login']
diff --git a/experiment/graphql_issue_example.py b/experiment/graphql_issue_example.py
@@ -16,9 +16,6 @@
 
 # USAGE: find_issues.py <github_token>
 
-# Required for pylint: 1.9.4 to tokenize the python3 print function.
-from __future__ import print_function
-
 import sys
 import json
 import argparse
diff --git a/experiment/parse_build_log.py b/experiment/parse_build_log.py
@@ -29,7 +29,7 @@
 _CURRENT_YEAR = datetime.datetime.utcnow().year
 
 
-class TestOutput(object):
+class TestOutput:
     def __init__(self):
         self._lines = []
         self._start = None
@@ -81,8 +81,7 @@ def _get_tests(log):
             current_test = TestOutput()
             if len(ended_test) <= 1:
                 continue
-            else:
-                yield ended_test
+            yield ended_test
         else:
             current_test.append(match.group(1))
     yield current_test
diff --git a/hack/BUILD.bazel b/hack/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@py_deps//:requirements.bzl", "requirement")
+load("@py3_deps//:requirements.bzl", "requirement")
 load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary", "nodejs_test")
 
 package(default_visibility = ["//visibility:public"])
@@ -226,14 +226,13 @@ test_suite(
 py_binary(
     name = "pylint_bin",
     srcs = ["pylint_bin.py"],
-    python_version = "PY2",
+    python_version = "PY3",
     tags = ["lint"],
     # NOTE: this should only contain direct third party imports and pylint
     deps = [
         requirement("astroid"),
         requirement("backports.functools_lru_cache"),
         requirement("configparser"),
-        requirement("enum34"),
         requirement("influxdb"),
         requirement("isort"),
         requirement("lazy-object-proxy"),
diff --git a/hack/pylint_bin.py b/hack/pylint_bin.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2017 The Kubernetes Authors.
 #
diff --git a/hack/verify-pylint.sh b/hack/verify-pylint.sh
@@ -36,6 +36,5 @@ export PYLINTHOME=$TEST_TMPDIR
 
 shopt -s extglob globstar
 
-# TODO(clarketm) there is no version of `pylint` that supports "both" PY2 and PY3
-# I am disabling pylint checks for python3 files until migration complete
-"$DIR/pylint_bin" !(kettle|metrics|triage|velodrome|hack|gubernator|external|vendor|testgrid|bazel-*)/**/*.py
+# TODO(clarketm): remove `boskos` exclusion after upgrading to PY3.
+"$DIR/pylint_bin" !(gubernator|external|vendor|jenkins|scenarios|triage|boskos|bazel-*)/**/*.py
diff --git a/kettle/BUILD.bazel b/kettle/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@py_deps//:requirements.bzl", "requirement")
+load("@py3_deps//:requirements.bzl", "requirement")
 
 py_test(
     name = "make_db_test",
diff --git a/kettle/make_json.py b/kettle/make_json.py
@@ -81,14 +81,16 @@ def make_result(name, time, failure_text):
 
 
 def buckets_yaml():
-    import ruamel.yaml as yaml  # does not support pypy
+    import ruamel.yaml as yaml  # pylint: disable=import-outside-toplevel
     with open(os.path.dirname(os.path.abspath(__file__))+'/buckets.yaml') as fp:
         return yaml.safe_load(fp)
 
 # pypy compatibility hack
 def python_buckets_yaml(python='python3'):
     return json.loads(subprocess.check_output(
-        [python, '-c', 'import json, ruamel.yaml as yaml; print(json.dumps(yaml.safe_load(open("buckets.yaml"))))'],
+        [python, '-c',
+         'import json, ruamel.yaml as yaml; print(json.dumps(yaml.safe_load(open("buckets.yaml"))))'
+         ],
         cwd=os.path.dirname(os.path.abspath(__file__))).decode("utf-8"))
 
 for attempt in [python_buckets_yaml, buckets_yaml, lambda: python_buckets_yaml(python='python')]:
diff --git a/kettle/make_json_test.py b/kettle/make_json_test.py
@@ -135,7 +135,7 @@ def expect(args, needles, negneedles, expected_ret=None):
             for needle in negneedles:
                 # Only match negative needles in the middle of a word, to avoid
                 # failures on timestamps that happen to contain a short number.
-                self.assertNotRegexpMatches(result, r'\b%s\b' % needle)
+                self.assertNotRegexpMatches(result, r'\b%s\b' % needle) # pylint: disable=deprecated-method
 
         add_build('some-job/123', last_month, last_month + 10, 'SUCCESS', junits)
         add_build('some-job/456', now - 10, now, 'FAILURE', junits)
diff --git a/kettle/stream.py b/kettle/stream.py
@@ -69,15 +69,16 @@ def get_started_finished(gcs_client, db, todo):
     pool = multiprocessing.pool.ThreadPool(16)
     try:
         for ack_id, (build_dir, started, finished) in pool.imap_unordered(
-                lambda ack_id_job_build: (ack_id_job_build[0], gcs_client.get_started_finished(ack_id_job_build[1], ack_id_job_build[2])),
+                lambda ack_id_job_build: (ack_id_job_build[0], gcs_client.get_started_finished(
+                    ack_id_job_build[1], ack_id_job_build[2])),
                 todo):
             if finished:
                 if not db.insert_build(build_dir, started, finished):
                     print('already present??')
                 start = time.localtime(started.get('timestamp', 0) if started else 0)
                 print((build_dir, bool(started), bool(finished),
-                      time.strftime('%F %T %Z', start),
-                      finished and finished.get('result')))
+                       time.strftime('%F %T %Z', start),
+                       finished and finished.get('result')))
                 build_dirs.append(build_dir)
                 acks.append(ack_id)
             else:
diff --git a/load.bzl b/load.bzl
@@ -48,10 +48,10 @@ def repositories():
     # Python setup
     # pip_import() calls must live in WORKSPACE, otherwise we get a load() after non-load() error
     git_repository(
-        name = "io_bazel_rules_python",
-        commit = "9d68f24659e8ce8b736590ba1e4418af06ec2552",
+        name = "rules_python",
+        commit = "94677401bc56ed5d756f50b441a6a5c7f735a6d4",
         remote = "https://github.com/bazelbuild/rules_python.git",
-        shallow_since = "1565801665 -0400",
+        shallow_since = "1573842889 -0500",
     )
 
     # TODO(fejta): get this to work
diff --git a/metrics/BUILD.bazel b/metrics/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@py_deps//:requirements.bzl", "requirement")
+load("@py3_deps//:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 
diff --git a/metrics/bigquery.py b/metrics/bigquery.py
@@ -65,7 +65,7 @@ def do_jq(jq_filter, data_filename, out_filename, jq_bin='jq'):
         check([jq_bin, jq_filter, data_filename], stdout=out_file)
 
 
-class BigQuerier(object):
+class BigQuerier:
     def __init__(self, project, bucket_path, backfill_days, influx_client):
         if not project:
             raise ValueError('project', project)
diff --git a/py.bzl b/py.bzl
@@ -15,7 +15,7 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
 load("@io_bazel_rules_appengine//appengine:py_appengine.bzl", "py_appengine_repositories")
 load(
-    "@py_deps//:requirements.bzl",
+    "@py3_deps//:requirements.bzl",
     "pip_install",
 )
 
diff --git a/releng/BUILD.bazel b/releng/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@py_deps//:requirements.bzl", "requirement")
+load("@py3_deps//:requirements.bzl", "requirement")
 
 py_binary(
     name = "generate_tests",
diff --git a/releng/generate_tests.py b/releng/generate_tests.py
@@ -107,7 +107,7 @@ def apply_job_overrides(envs_or_args, job_envs_or_args):
         envs_or_args.append(job_env_or_arg)
 
 
-class E2ENodeTest(object):
+class E2ENodeTest:
 
     def __init__(self, job_name, job, config):
         self.job_name = job_name
@@ -214,10 +214,10 @@ def generate(self):
         return job_config, prow_config, None
 
 
-class E2ETest(object):
+class E2ETest:
 
     def __init__(self, output_dir, job_name, job, config):
-        self.env_filename = os.path.join(output_dir, '%s.env' % job_name),
+        self.env_filename = os.path.join(output_dir, '%s.env' % job_name)
         self.job_name = job_name
         self.job = job
         self.common = config['common']
diff --git a/requirements.txt b/requirements.txt
diff --git a/requirements2.txt b/requirements2.txt
@@ -0,0 +1 @@
+PyYAML==5.1.1
diff --git a/requirements3.txt b/requirements3.txt
@@ -0,0 +1,11 @@
+astroid==2.3.3
+backports.functools_lru_cache==1.6.1
+configparser==4.0.2
+influxdb==5.2.3
+isort==4.3.21
+pylint==2.4.4
+PyYAML==5.3
+ruamel.yaml==0.16.5
+setuptools==44.0.0
+sh==1.12.14
+singledispatch==3.4.0.3
diff --git a/testgrid/conformance/upload_e2e.py b/testgrid/conformance/upload_e2e.py
@@ -32,8 +32,6 @@
 #
 # Usage: see README.md
 
-# Required for pylint: 1.9.4 to tokenize the python3 print function.
-from __future__ import print_function
 
 import re
 import sys
@@ -112,7 +110,7 @@ def parse_e2e_logfile(file_handle, year):
         if passed is False:
             # if we already have found a failure, ignore subsequent pass/fails
             continue
-        elif E2E_LOG_SUCCESS_RE.match(line):
+        if E2E_LOG_SUCCESS_RE.match(line):
             passed = True
         elif E2E_LOG_FAIL_RE.match(line):
             passed = False
diff --git a/velodrome/BUILD.bazel b/velodrome/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@py_deps//:requirements.bzl", "requirement")
+load("@py3_deps//:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-load("@py_deps//:requirements.bzl", "requirement")`
	`1`	`+load("@py3_deps//:requirements.bzl", "requirement")`
`2`	`2`
`3`	`3`	`py_binary(`
`4`	`4`	`name = "flakedetector",`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#!/usr/bin/env python2`
	`1`	`+#!/usr/bin/env python3`
`2`	`2`
`3`	`3`	`# Copyright 2017 The Kubernetes Authors.`
`4`	`4`	`#`