[WIP] Feature/pylint-cleanup

.github/workflows/industrial_ci_action.yml

@@ -13,5 +13,5 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
-      - uses: 'ros-industrial/industrial_ci@master'
+      - uses: 'fmessmer/industrial_ci@master_pylint'
         env: ${{matrix.env}}

.travis.yml

@@ -10,14 +10,17 @@ env:
   global:
     - CATKIN_LINT=pedantic
     - NOT_TEST_DOWNSTREAM=true
-    - PARALLEL_TESTS=true
+    - PYLINT_ARGS="--output-format=parseable --errors-only --ignored-modules=tf2_ros"
+    - PYLINT2_CHECK=true
+    - PYLINT3_CHECK=false
+    - PARALLEL_TESTS=false
     - ROS_REPO=ros
     - VERBOSE_OUTPUT=false
     - VERBOSE_TESTS=true
   matrix:
     - ROS_DISTRO=kinetic
     - ROS_DISTRO=melodic
 install:
-  - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci -b master
+  - git clone --quiet --depth 1 https://github.com/fmessmer/industrial_ci.git .industrial_ci -b master_pylint
 script:
   - .industrial_ci/travis.sh

atf/package.xml

@@ -15,6 +15,7 @@
     <exec_depend>atf_core</exec_depend>
     <exec_depend>atf_metrics</exec_depend>
     <exec_depend>atf_msgs</exec_depend>
+    <exec_depend>atf_plotter</exec_depend>
     <exec_depend>atf_recorder_plugins</exec_depend>
     <exec_depend>atf_test</exec_depend>
     <exec_depend>atf_test_tools</exec_depend>

atf_core/scripts/analyser.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-import copy
 import json
 import os
 import progressbar
@@ -12,8 +11,9 @@
 import unittest
 import yaml
 
-from atf_core import ATFConfigurationParser
-from atf_msgs.msg import AtfResult, TestResult, TestblockResult, MetricResult, TestblockStatus, KeyValue, DataStamped, Groundtruth
+from atf_core.configuration_parser import ATFConfigurationParser
+from atf_metrics.error import ATFAnalyserError
+from atf_msgs.msg import AtfResult, TestResult, TestblockResult, MetricResult, TestblockStatus, Groundtruth
 from atf_metrics import metrics_helper
 
 class Analyser:

atf_core/scripts/cleaner.py

@@ -5,7 +5,7 @@
 import os
 import sys
 
-from atf_core import ATFConfigurationParser
+from atf_core.configuration_parser import ATFConfigurationParser
 
 
 class Cleaner():

atf_core/scripts/generate_tests.py

@@ -1,8 +1,5 @@
 #!/usr/bin/env python
 import yaml
-import json
-import re
-import itertools as it
 import rospkg
 import os
 import lxml.etree
@@ -13,9 +10,8 @@
 from xml.etree import ElementTree
 from xml.dom import minidom
 
-from copy import deepcopy, copy
-
 import atf_core
+from atf_core.configuration_parser import ATFConfigurationParser
 
 class GenerateTests:
     def __init__(self, arguments):
@@ -31,7 +27,7 @@ def __init__(self, arguments):
         self.test_generation_config_file = arguments[2]
         self.package_src_path = arguments[3]
         self.package_bin_path = arguments[4]
-        self.atf_configuration_parser = atf_core.ATFConfigurationParser(self.package_src_path, self.test_generation_config_file, skip_metrics=True)
+        self.atf_configuration_parser = ATFConfigurationParser(self.package_src_path, self.test_generation_config_file, skip_metrics=True)
         self.tests = self.atf_configuration_parser.get_tests()
 
         self.test_list = {}
@@ -168,7 +164,7 @@ def generate_tests(self):
                 xml_test({'test-name': "uploading_data",
                         'pkg': "atf_core",
                         'type': "test_dropbox_uploader.py",
-                        'time-limit': str(self.time_limit_uploading),
+                        'time-limit': str(self.atf_configuration_parser.generation_config["time_limit_uploading"]),
                         'args': "-f " + os.path.join(self.package_src_path, "atf/.dropbox_uploader_config") + " upload " + self.atf_configuration_parser.generation_config["bagfile_output"] + " " + os.path.join(self.package_name, "data")}))
 
         if test.generation_config["upload_result"]:

atf_core/scripts/sm_test.py

@@ -4,10 +4,12 @@
 import smach
 import smach_ros
 import sys
-import threading
 import unittest
 
 import atf_core
+from atf_core.configuration_parser import ATFConfigurationParser
+from atf_core.recorder import ATFRecorder
+from atf_core.sm_atf import SmAtfTestblock
 
 class Recorder():
     def __init__(self):
@@ -24,7 +26,7 @@ def __init__(self):
         test_name = rospy.get_param("/atf/test_name")
         print "test_name:", test_name
 
-        atf_configuration_parser = atf_core.ATFConfigurationParser(package_name, test_generation_config_file)
+        atf_configuration_parser = ATFConfigurationParser(package_name, test_generation_config_file)
         tests = atf_configuration_parser.get_tests()
         for test in tests:
             #print "test.name:", test.name
@@ -40,7 +42,7 @@ def __init__(self):
         outcome_map = {'succeeded':outcome_map_succeeded,
                         'error':outcome_map_error}
 
-        recorder_handle = atf_core.ATFRecorder(test)
+        recorder_handle = ATFRecorder(test)
 
         # Open the container
         with self.sm_top:
@@ -57,7 +59,7 @@ def __init__(self):
                 # Add states to the container
                 for testblock in test.testblockset_config.keys():
                     #print "adding testblock:", testblock
-                    smach.Concurrence.add(testblock, atf_core.SmAtfTestblock(testblock, recorder_handle))
+                    smach.Concurrence.add(testblock, SmAtfTestblock(testblock, recorder_handle))
 
             # TODO preempt all other concurrent States as soon as one state returns 'error'
 

atf_core/src/atf_core/__init__.py

@@ -1,8 +0,0 @@
-from atf_core.atf import ATF
-from atf_core.error import ATFError, ATFConfigurationError, ATFRecorderError, ATFAnalyserError
-from atf_core.testblock import Testblock
-from atf_core.test import Test
-from atf_core.configuration_parser import ATFConfigurationParser
-from atf_core.recorder import ATFRecorder
-from atf_core.bagfile_helper import BagfileWriter
-from atf_core.sm_atf import SmAtfTestblock

atf_core/src/atf_core/atf.py

@@ -1,11 +1,14 @@
 #!/usr/bin/env python
 import rospy
-import atf_core
 import copy
 
+from atf_core.error import ATFError
+from atf_core.configuration_parser import ATFConfigurationParser
+from atf_metrics.error import ATFConfigurationError
 from atf_msgs.msg import MetricResult, TestblockTrigger, DataStamped
 from smach_msgs.msg import SmachContainerStatus
 
+
 ###########
 ### ATF ###
 ###########
@@ -19,7 +22,7 @@ def __init__(self):
         test_name = rospy.get_param("/atf/test_name")
         print "test_name:", test_name
 
-        atf_configuration_parser = atf_core.ATFConfigurationParser(package_name, test_generation_config_file)
+        atf_configuration_parser = ATFConfigurationParser(package_name, test_generation_config_file)
         tests = atf_configuration_parser.get_tests()
         for test in tests:
             #print "test.name:", test.name
@@ -42,7 +45,7 @@ def start(self, testblock):
         if testblock not in self.test.testblockset_config.keys():
             error_msg = "testblock \'%s\' not in list of testblocks"%testblock
             self._send_error(error_msg)
-            raise atf_core.ATFError(error_msg)
+            raise ATFError(error_msg)
         rospy.loginfo("starting testblock \'%s\'"%testblock)
         trigger = TestblockTrigger()
         trigger.stamp = rospy.Time.now()
@@ -54,7 +57,7 @@ def stop(self, testblock, metric_result = None):
         if testblock not in self.test.testblockset_config.keys():
             error_msg = "testblock \'%s\' not in list of testblocks"%testblock
             self._send_error(error_msg)
-            raise atf_core.ATFError(error_msg)
+            raise ATFError(error_msg)
 
         if metric_result != None:
 
@@ -69,25 +72,25 @@ def stop(self, testblock, metric_result = None):
                             "metric %s is not a MetricResult. data=%s, type=%s" % (
                             testblock, metric_result.name, str(metric_result), type(metric_result))
                 self._send_error(error_msg)
-                raise atf_core.ATFError(error_msg)
+                raise ATFError(error_msg)
             if not isinstance(metric_result.data, DataStamped):
                 error_msg = "metric_result.data of testblock %s for " \
                             "metric %s is not a DataStamped. data=%s, type=%s" % (
                             testblock, metric_result.name, str(metric_result.data), type(metric_result.data))
                 self._send_error(error_msg)
-                raise atf_core.ATFError(error_msg)
+                raise ATFError(error_msg)
             if not isinstance(metric_result.data.data, float) and not isinstance(metric_result.data.data, int):
                 error_msg = "metric_result.data.data of testblock %s for " \
                             "metric %s is not a float or int. data=%s, type=%s" % (
                     testblock, metric_result.name, str(metric_result.data.data), type(metric_result.data.data))
                 self._send_error(error_msg)
-                raise atf_core.ATFError(error_msg)
+                raise ATFError(error_msg)
             if type(metric_result.details) is not list:
                 error_msg = "metric_result.details of testblock %s for " \
                             "metric %s is not a list. detail=%s" % (
                     testblock, metric_result.name, str(metric_result.details))
                 self._send_error(error_msg)
-                raise atf_core.ATFError(error_msg)
+                raise ATFError(error_msg)
 
         else:
             rospy.loginfo("no user result set for testblock \'%s\'"%testblock)

atf_core/src/atf_core/configuration_parser.py

@@ -1,17 +1,15 @@
 #!/usr/bin/env python
-import atf_core
 import fnmatch
 import yaml
 import rospkg
-import rosparam
-import atf_metrics
 import os
-import itertools as it
 import json
 import rosbag
 
-from atf_core import ATFConfigurationError
-from atf_core import Test, Testblock
+from atf_metrics.error import ATFConfigurationError
+from atf_core.test import Test
+from atf_core.testblock import Testblock
+import atf_metrics
 
 class ATFConfigurationParser:
     def __init__(self, package_name = None, test_generation_config_file = None, skip_metrics = False):

atf_core/src/atf_core/error.py

@@ -2,11 +2,5 @@
 class ATFError(Exception):
     pass
 
-class ATFConfigurationError(Exception):
-    pass
-
 class ATFRecorderError(Exception):
-    pass
-
-class ATFAnalyserError(Exception):
     pass

atf_core/src/atf_core/recorder.py

@@ -1,20 +1,21 @@
 #!/usr/bin/env python
-import actionlib
 import roslib
 import rospy
 import rospkg
 import rostopic
 import rosbag
 import yaml
 import os
-import atf_recorder_plugins
-import atf_core
 import tf
 
 from threading import Lock
-from atf_msgs.msg import TestblockTrigger
-from diagnostic_msgs.msg import DiagnosticStatus
 from tf2_msgs.msg import TFMessage
+from diagnostic_msgs.msg import DiagnosticStatus
+
+from actionlib.simple_action_client import SimpleActionClient
+from atf_core.bagfile_helper import BagfileWriter
+from atf_core.error import ATFRecorderError
+import atf_recorder_plugins
 
 
 class ATFRecorder:
@@ -41,7 +42,7 @@ def __init__(self, test):
         # create bag file writer handle
         self.lock_write = Lock()
         self.bag = rosbag.Bag(self.test.generation_config["bagfile_output"] + self.test.name + ".bag", 'w')
-        self.bag_file_writer = atf_core.BagfileWriter(self.bag, self.lock_write)
+        self.bag_file_writer = BagfileWriter(self.bag, self.lock_write)
 
         # Init metric recorder
         self.recorder_plugin_list = []
@@ -82,12 +83,13 @@ def __init__(self, test):
                 topic_type = rostopic._get_topic_type(action + "/goal")[0]
 
                 # remove "Goal" string from action type
-                if topic_type == None or not "Goal" in topic_type:
+                if topic_type == None or not "Goal" in topic_type:  ## pylint: disable=unsupported-membership-test
                     msg = "Could not get type for action %s. type is %s"%(action, topic_type)
                     rospy.logerr(msg)
                     raise ATFRecorderError(msg)
-                topic_type = topic_type[0:len(topic_type)-4] # remove "Goal" from type
-                client = actionlib.SimpleActionClient(action, roslib.message.get_message_class(topic_type))
+                # remove "Goal" from type
+                topic_type = topic_type[0:len(topic_type)-4] ## pylint: disable=unsubscriptable-object
+                client = SimpleActionClient(action, roslib.message.get_message_class(topic_type))
 
                 # wait for action server
                 client.wait_for_server()

atf_core/src/atf_core/sm_atf.py

@@ -3,7 +3,6 @@
 import smach
 import smach_ros
 import threading
-import atf_core
 
 from atf_msgs.msg import TestblockTrigger, TestblockStatus
 

atf_core/src/atf_core/test.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from atf_msgs.msg import TestResult
+from atf_metrics.error import ATFAnalyserError
 
 class Test:
     def __init__(self):
@@ -18,7 +19,6 @@ def __init__(self):
         self.testblockset_name = None
 
         # testblocks with metrics
-        metrics_handle = None
         self.testblocks = []
 
     def get_result(self):

atf_core/src/atf_core/testblock.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from atf_core import ATFAnalyserError
+from atf_metrics.error import ATFAnalyserError
 from atf_msgs.msg import TestblockResult, TestblockStatus, Groundtruth
 
 class Testblock:

atf_metrics/src/atf_metrics/calculate_interface.py

@@ -4,8 +4,7 @@
 import re
 import rospy
 
-from atf_core import ATFAnalyserError, ATFConfigurationError
-from atf_msgs.msg import Api
+from atf_metrics.error import ATFConfigurationError
 from atf_msgs.msg import MetricResult, Groundtruth, KeyValue, DataStamped, TestblockStatus
 from atf_metrics import metrics_helper
 

atf_metrics/src/atf_metrics/calculate_publish_rate.py

@@ -2,7 +2,7 @@
 import math
 import rospy
 
-from atf_core import ATFAnalyserError, ATFConfigurationError
+from atf_metrics.error import ATFConfigurationError
 from atf_msgs.msg import MetricResult, Groundtruth, KeyValue, DataStamped, TestblockStatus
 from atf_metrics import metrics_helper
 

atf_metrics/src/atf_metrics/calculate_tf_acceleration_translation.py

@@ -4,10 +4,9 @@
 import rospy
 import sys
 import tf
-import tf2_py
 import tf2_ros
 
-from atf_core import ATFAnalyserError, ATFConfigurationError
+from atf_metrics.error import ATFConfigurationError
 from atf_msgs.msg import MetricResult, Groundtruth, KeyValue, DataStamped, TestblockStatus
 from atf_metrics import metrics_helper
 
@@ -122,11 +121,11 @@ def get_data(self, t):
             sys.stdout = sys.__stdout__  # restore stdout
             #print "Exception in metric '%s' %s %s"%(self.name, type(e), e)
             return None
-        except tf2_py.ExtrapolationException as e:
+        except tf2_ros.ExtrapolationException as e:
             sys.stdout = sys.__stdout__  # restore stdout
             #print "Exception in metric '%s' %s %s"%(self.name, type(e), e)
             return None
-        except tf2_py.ConnectivityException as e:
+        except tf2_ros.ConnectivityException as e:
             sys.stdout = sys.__stdout__  # restore stdout
             #print "Exception in metric '%s' %s %s"%(self.name, type(e), e)
             return None