Porting python2 to 3 modules: top level scripts 1

Author: gudrutis

add-externals-gh-labels.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from github import Github
 from os.path import expanduser, dirname, abspath, join
 from githublabels import LABEL_TYPES, COMMON_LABELS, COMPARISON_LABELS, CMSSW_BUILD_LABELS, LABEL_COLORS
-from categories import COMMON_CATEGORIES, EXTERNAL_CATEGORIES, EXTERNAL_REPOS, CMSSW_REPOS, CMSDIST_REPOS, CMSSW_CATEGORIES
+from categories import COMMON_CATEGORIES, EXTERNAL_CATEGORIES, EXTERNAL_REPOS, CMSSW_REPOS, CMSSW_CATEGORIES
 from cms_static import VALID_CMS_SW_REPOS_FOR_TESTS, GH_CMSSW_ORGANIZATION
-from datetime import datetime
 from socket import setdefaulttimeout
-from github_utils import api_rate_limits, api_rate_limits_repo
+from github_utils import api_rate_limits
 from cmsutils import get_config_map_properties
 from sys import argv
 setdefaulttimeout(120)
@@ -24,29 +24,29 @@ def setRepoLabels (gh, repo_name, all_labels, dryRun=False, ignore=[]):
         if repo.name not in VALID_CMS_SW_REPOS_FOR_TESTS:
           skip = True
       if skip:
-        print "Ignoring repo:",repo.full_name
+        print("Ignoring repo:",repo.full_name)
         continue
       repos.append(repo)
   else:
     repos.append(gh.get_repo(repo_name))
   api_rate_limits(gh)
   for repo in repos:
-    print "Checking repository ", repo.full_name, ", DryRun:",dryRun
+    print("Checking repository ", repo.full_name, ", DryRun:",dryRun)
     cur_labels = {}
     for lab in repo.get_labels():
       cur_labels [lab.name]=lab
     api_rate_limits(gh)
     for lab in all_labels:
       if not lab in cur_labels:
-        print "  Creating new label ",lab,"=>",all_labels[lab]
+        print("  Creating new label ",lab,"=>",all_labels[lab])
         if not dryRun:
           repo.create_label(lab, all_labels[lab])
           api_rate_limits(gh)
       elif cur_labels[lab].color != all_labels[lab]:
         if not dryRun:
           cur_labels[lab].edit(lab, all_labels[lab])
           api_rate_limits(gh)
-        print "  Label ",lab," color updated: ",cur_labels[lab].color ," => ",all_labels[lab]
+        print("  Label ",lab," color updated: ",cur_labels[lab].color ," => ",all_labels[lab])
 
 if __name__ == "__main__":
   from optparse import OptionParser
@@ -70,7 +70,7 @@ def setRepoLabels (gh, repo_name, all_labels, dryRun=False, ignore=[]):
 
   if opts.cmssw or opts.externals:
     all_labels = COMMON_LABELS
-    for cat in COMMON_CATEGORIES+EXTERNAL_CATEGORIES+CMSSW_CATEGORIES.keys():
+    for cat in COMMON_CATEGORIES+EXTERNAL_CATEGORIES+list(CMSSW_CATEGORIES.keys()):
       for lab in LABEL_TYPES:
         all_labels[cat+"-"+lab]=LABEL_TYPES[lab]
     for lab in COMPARISON_LABELS:
@@ -101,14 +101,14 @@ def setRepoLabels (gh, repo_name, all_labels, dryRun=False, ignore=[]):
     from glob import glob
     for rconf in glob(join(SCRIPT_DIR,"repos","*","*","repo_config.py")):
       repo_data = rconf.split("/")[-4:-1]
-      exec 'from '+".".join(repo_data)+' import categories, repo_config'
-      print repo_config.GH_TOKEN, repo_config.GH_REPO_FULLNAME
+      exec('from '+".".join(repo_data)+' import categories, repo_config')
+      print(repo_config.GH_TOKEN, repo_config.GH_REPO_FULLNAME)
       if not repo_config.ADD_LABELS: continue
       gh = Github(login_or_token=open(expanduser(repo_config.GH_TOKEN)).read().strip())
       all_labels = COMMON_LABELS
       for lab in COMPARISON_LABELS:
         all_labels[lab] = COMPARISON_LABELS[lab]
-      for cat in categories.COMMON_CATEGORIES+categories.CMSSW_CATEGORIES.keys():
+      for cat in categories.COMMON_CATEGORIES+list(categories.CMSSW_CATEGORIES.keys()):
         for lab in LABEL_TYPES:
           all_labels[cat+"-"+lab]=LABEL_TYPES[lab]
       setRepoLabels (gh, repo_config.GH_REPO_FULLNAME, all_labels, opts.dryRun)

auto-create-pull-requests

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # Creates pull requests to automatically forward port from one release to another.
+from __future__ import print_function
 from github import Github, GithubException
 from sys import exit
 from os.path import expanduser
@@ -31,7 +32,7 @@ if __name__ == "__main__":
     try:
       gh_repo = gh.get_organization("cms-sw").get_repo(repo)
     except:
-      print "Could not find repository."
+      print("Could not find repository.")
       exit(0)
     for (head, base) in MAPPINGS[repo]:
       title = "Forward port %s into %s" % (head, base)
@@ -41,13 +42,13 @@ if __name__ == "__main__":
                                  body="",
                                  base=base,
                                  head=head)
-      except GithubException, e:
+      except GithubException as e:
         if e.data[u"errors"][0][u"message"].startswith("No commits between"):
-          print "%s already in %s" % (head, base)
+          print("%s already in %s" % (head, base))
         elif e.data[u"errors"][0][u"message"].startswith("A pull request already exists"):
-          print "%s is already pending for merge in %s" % (head, base)
+          print("%s is already pending for merge in %s" % (head, base))
         else:
-          print e.data
+          print(e.data)
           exit(1)
       if pr:
-        print pr
+        print(pr)

backport-pr.py

@@ -1,39 +1,40 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from github import Github, GithubException
 from sys import exit
 from os.path import expanduser
 from argparse import ArgumentParser
 from cms_static import GH_CMSSW_ORGANIZATION
 from cms_static import GH_CMSSW_REPO
-from commands import getstatusoutput as run_cmd
+from _py2with3compatibility import run_cmd
 CMSSW_GIT_REF="/cvmfs/cms.cern.ch/cmssw.git.daily"
 
 def backport_pull (repo, pr, branch):
   pr_branch = pr.base.label.split(":")[1]
-  print "Source Branch:",pr_branch
+  print("Source Branch:",pr_branch)
   if pr_branch == branch: return "Warning: Can not backport, same branch %s vs %s" % (pr_branch, branch),False
   br = gh_repo.get_branch(branch)
   commits = []
   for c in pr.get_commits().reversed: commits.insert(0,"git cherry-pick %s" % c.sha)
   if not commits: return "There are no commits to backport",False
-  print "Cherry-pick commands:"
-  print "  "+"\n  ".join(commits)
+  print("Cherry-pick commands:")
+  print("  "+"\n  ".join(commits))
   if len(commits)>=250:
     return "Error: Too many commits in PR %s\nBot can only handle max 250 commits." % len(commits),False
   new_branch = "backport-%s-%s" % (branch.replace("/","_"), pr.number)
-  print "New Branch:",new_branch
+  print("New Branch:",new_branch)
   git_ref = ""
   if repo.name == "cmssw": git_ref = "--reference "+CMSSW_GIT_REF
-  print "GIT REF:",git_ref
+  print("GIT REF:",git_ref)
   e , o = run_cmd("rm -rf pr_backport; git clone --branch %s %s [email protected]:%s pr_backport && cd pr_backport && git checkout -b %s" % (branch, git_ref, repo.full_name, new_branch))
   if e:
-    print o
+    print(o)
     exit(1)
   e, o = run_cmd('cd pr_backport; %s' % ";".join(commits))
   if e: return "Error: Failed to cherry-pick commits. Please backport this PR yourself.\n```"+o+"\n```",False
   e, o = run_cmd("cd pr_backport; git push origin %s" % new_branch)
   if e:
-    print o
+    print(o)
     exit(1)
   run_cmd("rm -rf pr_backport")
   newBody = "backport of #%s\n\n%s" %(pr.number, pr.body)
@@ -56,5 +57,5 @@ def backport_pull (repo, pr, branch):
   res = backport_pull (gh_repo, pr, args.branch)
   status = "done"
   if not res[1]: status = "failed\n**Reason:**\n"
-  print res
+  print(res)
   pr.create_issue_comment("backport %s\n%s" % (status, res[0]))

build-node-stats

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
+from __future__ import print_function
 import sys
-sys.exit(0)
+sys.exit(0)  # TODO script not run anymore?
 
-from commands import getstatusoutput
+from _py2with3compatibility import run_cmd
 from socket import gethostname
 from argparse import ArgumentParser
 import time
@@ -11,7 +12,7 @@ import struct
 import socket
 
 def chunks(l, n):
-  for i in xrange(0, len(l), n):
+  for i in range(0, len(l), n):
     yield l[i:i+n]
 
 def format(s, **kwds):
@@ -24,25 +25,25 @@ CARBON_PORT=2004
 def sendMetrics(metrics):
   for l in chunks(metrics, 100):
     payload = pickle.dumps(l)
-    print len(payload)
+    print(len(payload))
     header = struct.pack("!L", len(payload))
     message = header + payload
     sock = socket.socket()
-    print CARBON_SERVER, CARBON_PORT
+    print(CARBON_SERVER, CARBON_PORT)
     sock.connect((CARBON_SERVER, CARBON_PORT))
     sock.sendall(message)
     sock.close()
 
 def aggregate(metrics):
   aggregated = {}
   for x in metrics:
-    print x[0]
-    if x[0] in aggregated.keys():
+    print(x[0])
+    if x[0] in list(aggregated.keys()):
       aggregated[x[0]][1][1] += x[1][1]
-      print x[0],"now", aggregated[x[0]][1][1]
+      print(x[0],"now", aggregated[x[0]][1][1])
     else:
       aggregated[x[0]] = x
-  return [x for x in aggregated.values()]
+  return [x for x in list(aggregated.values())]
 
 if __name__ == "__main__":
   parser = ArgumentParser()
@@ -56,15 +57,15 @@ if __name__ == "__main__":
 
   fields = ("pid", "comm", "pcpu", "rss", "vsize")
   cmd = 'ps -U cmsbuild -o %s= -o %s= -o %s= -o %s= -o %s=' % fields
-  err, out = getstatusoutput(cmd)
+  err, out = run_cmd(cmd)
   if err:
-    print "Error while reading process data"
+    print("Error while reading process data")
     exit(1)
-  print out
+  print(out)
   metrics = []
   timestamp = time.time()
   for l in out.split("\n"):
-    data = dict(zip(fields, [x for x in l.split(" ") if x]))
+    data = dict(list(zip(fields, [x for x in l.split(" ") if x])))
     try:
       data["pid"] = int(data["pid"])
       data["pcpu"] = float(data["pcpu"])

buildLogAnalyzer.py

@@ -6,10 +6,12 @@
 Created by Andreas Pfeiffer on 2008-08-05.
 Copyright (c) 2008 CERN. All rights reserved.
 """
-
+from __future__ import print_function
+from past.builtins import cmp
 import sys, os, re, time
 import getopt
 
+
 def pkgCmp(a,b):
     if a.subsys == b.subsys: return cmp(a.pkg, b.pkg)
     else: return cmp(a.subsys, b.subsys)
@@ -65,7 +67,7 @@ def __init__(self, topDirIn='.', topUrlIn='', verbose = -1, pkgsList = None, rel
         if not pkgsList:  pkgsList = "../../../../../src/PackageList.cmssw"
         if self.topURL != '' :
             if self.topURL[-1] != '/' : self.topURL += '/'
-            if not release: release = self.topURL.split('/')[-3]
+            if not release: release = self.topURL.split('/')[-3]  # TODO no error catching
         self.release = release
         self.pkgsList = pkgsList
         self.verbose = verbose
@@ -138,7 +140,7 @@ def analyze(self):
         start = time.time()
         packageList = glob.glob('*/*/build.log')
 
-        if self.verbose > 0: print "going to analyze ", len(packageList), 'files.'
+        if self.verbose > 0: print("going to analyze ", len(packageList), 'files.')
 
         for logFile in packageList:
             self.analyzeFile(logFile)
@@ -151,7 +153,7 @@ def analyze(self):
                 for key in self.errorKeys:
                     if key in pkg.errSummary.keys() :
                         self.errMapAll[key].append(pkg)
-                    if key in pkg.errSummary.keys() and pkg not in pkgDone: 
+                    if key in pkg.errSummary.keys() and pkg not in pkgDone:
                         self.errMap[key].append(pkg)
                         pkgDone.append(pkg)
             else:    
@@ -164,17 +166,17 @@ def analyze(self):
     def report(self):
         """show collected info"""
 
-        print 'analyzed ', len(self.packageList), 'log files in', str(self.anaTime), 'sec.'
+        print('analyzed ', len(self.packageList), 'log files in', str(self.anaTime), 'sec.')
         totErr = 0
         for key, val in self.nErrorInfo.items():
             totErr += int(val)
 
-        print 'found ', totErr, ' errors and warnings in total, by type:'
+        print('found ', totErr, ' errors and warnings in total, by type:')
         for key, val in self.nErrorInfo.items():
-            print '\t', key, ' : ', val, ' in ', len(self.errMapAll[key]), 'packages'
+            print('\t', key, ' : ', val, ' in ', len(self.errMapAll[key]), 'packages')
 
-        print 'found ', len(self.pkgOK),  'packages without errors/warnings.'
-        print 'found ', len(self.pkgErr), 'packages with errors or warnings, ', len(self.pkgWarn), ' with warnings only.'
+        print('found ', len(self.pkgOK),  'packages without errors/warnings.')
+        print('found ', len(self.pkgErr), 'packages with errors or warnings, ', len(self.pkgWarn), ' with warnings only.')
 #        for pkg in pkgErr:
 #            print '\t',pkg.name(), ' : ',
 #            for key in ['dictError', 'compError', 'linkError']:
@@ -201,7 +203,7 @@ def report(self):
         for pkg in self.pkgOK:
             self.makeHTMLLogFile(pkg)
         stop = time.time()
-        print "creating html pages took ", str(stop-start), 'sec.'
+        print("creating html pages took ", str(stop-start), 'sec.')
 
     def makeHTMLSummaryPage(self):
 
@@ -356,10 +358,9 @@ def analyzeFile(self, fileNameIn):
         """read in file and check for errors"""
         subsys, pkg, logFile = fileNameIn.split('/')
 
-        if self.verbose > 5 : print "analyzing file : ", fileNameIn
+        if self.verbose > 5 : print("analyzing file : ", fileNameIn)
 
         fileIn = open(fileNameIn, 'r')
-        lines = fileIn.xreadlines()
         shLib = 'so'
         if os.uname()[0] == 'Darwin' :
             shLib = 'dylib'
@@ -414,7 +415,7 @@ def analyzeFile(self, fileNameIn):
 
         pkgInfo = PackageInfo(subsys, pkg)
         lineNo = -1
-        for line in lines:
+        for line in fileIn:
             lineNo += 1
             errFound = False
             for errI in errors:
@@ -488,7 +489,7 @@ def main(argv=None):
     try:
         try:
             opts, args = getopt.getopt(argv[1:], "hv:l:t:p:r:", ["help", "verbose=", "logDir=", "topURL=", "pkgList=", "release="])
-        except getopt.error, msg:
+        except getopt.error as msg:
             raise Usage(msg)
 
         # option processing
@@ -515,10 +516,11 @@ def main(argv=None):
         lfa.analyze()
         lfa.report()
 
-    except Usage, err:
-        print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg)
-        print >> sys.stderr, "\t for help use --help"
+    except Usage as err:
+        print(sys.argv[0].split("/")[-1] + ": " + str(err.msg), file=sys.stderr)
+        print("\t for help use --help", file=sys.stderr)
         return 2
 
+
 if __name__ == "__main__":
     sys.exit(main(sys.argv))