scriptcheck.yml: re-enabled pylint and mitigated findings / added missing files to Python syntax and lint check (#6464)

Author: firewave

Diff for: .github/workflows/scriptcheck.yml

@@ -89,6 +89,7 @@ jobs:
           python -m pip install pygments
           python -m pip install requests
           python -m pip install psutil
+          python -m pip install setuptools
 
       - name: run Shellcheck
         if: matrix.python-latest
@@ -98,10 +99,8 @@ jobs:
       - name: run pylint
         if: matrix.python-latest
         run: |
-          echo "FIXME pylint is disabled for now because it fails to import files:"
-          echo "FIXME addons/runaddon.py:1:0: E0401: Unable to import 'cppcheckdata' (import-error)"
-          echo "FIXME addons/runaddon.py:1:0: E0401: Unable to import 'cppcheck' (import-error)"
-          # pylint --rcfile=pylintrc_travis --jobs $(nproc) addons/*.py htmlreport/cppcheck-htmlreport htmlreport/*.py tools/*.py
+          shopt -s globstar
+          pylint --jobs $(nproc) addons/**/*.py htmlreport/cppcheck-htmlreport htmlreport/**/*.py test/**/*.py tools/**/*.py
 
       - name: check .json files
         if: matrix.python-latest
@@ -115,10 +114,12 @@ jobs:
 
       - name: check python syntax
         run: |
-          python -m py_compile addons/*.py
+          shopt -s globstar
+          python -m py_compile addons/**/*.py
           python -m py_compile htmlreport/cppcheck-htmlreport
-          python -m py_compile htmlreport/*.py
-          python -m py_compile tools/*.py
+          python -m py_compile htmlreport/**/*.py
+          python -m py_compile test/**/*.py
+          python -m py_compile tools/**/*.py
 
       - name: compile addons
         run: |

Diff for: pylintrc_travis renamed to .pylintrc

@@ -677,7 +677,7 @@ def __init__(self, element, nestedIn):
         self.argumentId = {}
 
     def __repr__(self):
-        attrs = ["Id", "tokenId", "tokenDefId", "name", "type", "hasVirtualSpecifier", 
+        attrs = ["Id", "tokenId", "tokenDefId", "name", "type", "hasVirtualSpecifier",
                  "isImplicitlyVirtual", "access", "isInlineKeyword", "isStatic", 
                  "isAttributeNoreturn", "overriddenFunction", "nestedIn", "argumentId"]
         return "{}({})".format(
@@ -901,7 +901,7 @@ def setId(self, IdMap):
         self.symbolic = IdMap.get(self._symbolicId)
 
     def __repr__(self):
-        attrs = ["intvalue", "tokvalue", "floatvalue", "movedvalue", "uninit", 
+        attrs = ["intvalue", "tokvalue", "floatvalue", "movedvalue", "uninit",
                  "bufferSize", "containerSize", "condition", "valueKind"]
         return "{}({})".format(
             "Value",
@@ -999,7 +999,7 @@ def isMatch(self, file, line, message, errorId):
             return True
         # Other Suppression (Globaly set via suppression file or cli command)
         if ((self.fileName is None or fnmatch(file, self.fileName))
-                and (self.suppressionType is None) 
+                and (self.suppressionType is None)
                 and (self.symbolName is None or fnmatch(message, '*'+self.symbolName+'*'))
                 and fnmatch(errorId, self.errorId)):
             return True

Diff for: addons/cppcheckdata.py

@@ -2105,7 +2105,7 @@ def misra_8_4(self, cfg):
                 else:
                     self.insert_in_dict(extern_var_without_def, tok.str, tok)
             else:
-                 self.insert_in_dict(extern_var_without_def, var.nameToken.str, var.nameToken)
+                self.insert_in_dict(extern_var_without_def, var.nameToken.str, var.nameToken)
 
         for var in extern_var_with_def:
             if var not in extern_var_without_def:

Diff for: addons/misra.py

@@ -146,6 +146,11 @@ class Config:
     if have_error:
         sys.exit(1)
 
+    # pylint: disable-next=no-member - TODO: fix this
+    if config.include_guard:
+        # pylint: disable-next=no-member - TODO: fix this
+        config.include_guard_header_re = config.include_guard.get('RE_HEADERFILE',"[^/].*\\.h\\Z")
+
     return config
 
 
@@ -200,14 +205,15 @@ def check_include_guards(conf,cfg,unguarded_include_files):
     # - test whether include guards are in place
     max_linenr = conf.include_guard.get('max_linenr', 5)
 
-    def report(directive,msg,errorId,column=0):
-        reportNamingError(directive,msg,errorId,column=column)
+    def report(directive,msg,errorId,severity='style',column=0):
+        reportNamingError(directive,msg,errorId,severity=severity,column=column)
 
     def report_pending_ifndef(directive,column):
         report(directive,'include guard #ifndef is not followed by #define','includeGuardIncomplete',column=column)
 
     last_fn = None
     pending_ifndef = None
+    guard_column = None
     phase = 0
     for directive in cfg.directives:
         if last_fn != directive.file:
@@ -219,7 +225,7 @@ def report_pending_ifndef(directive,column):
         if phase == -1:
             # ignore (the remainder of) this file
             continue
-        if not re.match(include_guard_header_re,directive.file):
+        if not re.match(conf.include_guard_header_re,directive.file):
             phase = -1
             continue
 
@@ -263,20 +269,16 @@ def report_pending_ifndef(directive,column):
     if pending_ifndef:
         report_pending_ifndef(pending_ifndef,guard_column)
 
-def process(dumpfiles, configfile):
+def process(dumpfiles, configfile, cli, debugprint):
     conf = loadConfig(configfile)
 
-    if conf.include_guard:
-        global include_guard_header_re
-        include_guard_header_re = conf.include_guard.get('RE_HEADERFILE',"[^/].*\\.h\\Z")
-
     for afile in dumpfiles:
         if not afile[-5:] == '.dump':
             continue
-        if not args.cli:
+        if not cli:
             print('Checking ' + afile + '...')
         data = cppcheckdata.CppcheckData(afile)
-        process_data(conf,data)
+        process_data(conf,data,cli,debugprint)
 
 def check_file_naming(conf,data):
     for source_file in data.files:
@@ -297,7 +299,7 @@ def check_namespace_naming(conf,data):
         for exp in conf.namespace:
             evalExpr(conf.namespace, exp, mockToken, 'Namespace')
 
-def check_variable_naming(conf,cfg):
+def check_variable_naming(conf,cfg,debugprint):
     for var in cfg.variables:
         if not var.nameToken:
             continue
@@ -309,7 +311,7 @@ def check_variable_naming(conf,cfg):
             prev = prev.previous
             varType = prev.str + varType
 
-        if args.debugprint:
+        if debugprint:
             print("Variable Name: " + str(var.nameToken.str))
             print("original Type Name: " + str(var.nameToken.valueType.originalTypeName))
             print("Type Name: " + var.nameToken.valueType.type)
@@ -342,7 +344,7 @@ def check_gpp_naming(conf_list,cfg,access,message):
         for exp in conf_list:
             evalExpr(conf_list, exp, mockToken, message)
 
-def check_function_naming(conf,cfg):
+def check_function_naming(conf,cfg,debugprint):
     for token in cfg.tokenlist:
         if not token.function:
             continue
@@ -353,7 +355,7 @@ def check_function_naming(conf,cfg):
         while "*" in retval and len(retval.replace("*", "")) == 0:
             prev = prev.previous
             retval = prev.str + retval
-        if args.debugprint:
+        if debugprint:
             print("\t:: {} {}".format(retval, token.function.name))
 
         if retval and retval in conf.function_prefixes:
@@ -373,7 +375,7 @@ def check_class_naming(conf,cfg):
         for exp in conf.class_name:
             evalExpr(conf.class_name, exp, mockToken, msgType)
 
-def process_data(conf,data):
+def process_data(conf,data,cli,debugprint):
     if conf.file:
         check_file_naming(conf,data)
 
@@ -382,21 +384,21 @@ def process_data(conf,data):
 
     unguarded_include_files = []
     if conf.include_guard and conf.include_guard.get('required',1):
-        unguarded_include_files = [fn for fn in data.files if re.match(include_guard_header_re,fn)]
+        unguarded_include_files = [fn for fn in data.files if re.match(conf.include_guard_header_re,fn)]
 
     for cfg in data.configurations:
-        if not args.cli:
+        if not cli:
             print('Checking config %s...' % cfg.name)
         if conf.variable:
-            check_variable_naming(conf,cfg)
+            check_variable_naming(conf,cfg,debugprint)
         if conf.private_member:
             check_gpp_naming(conf.private_member,cfg,'Private','Private member variable')
         if conf.public_member:
             check_gpp_naming(conf.public_member,cfg,'Public','Public member variable')
         if conf.global_variable:
             check_gpp_naming(conf.global_variable,cfg,'Global','Global variable')
         if conf.function_name:
-            check_function_naming(conf,cfg)
+            check_function_naming(conf,cfg,debugprint)
         if conf.class_name:
             check_class_naming(conf,cfg)
         if conf.include_guard:
@@ -414,6 +416,6 @@ def process_data(conf,data):
                         help="Naming check config file")
 
     args = parser.parse_args()
-    process(args.dumpfile, args.configfile)
+    process(args.dumpfile, args.configfile, args.cli, args.debugprint)
 
     sys.exit(0)

Diff for: addons/namingng.py

@@ -422,6 +422,7 @@ def git_blame(errors, path, file, blame_options):
     full_path = os.path.join(path, file)
     path, filename = os.path.split(full_path)
 
+    cwd = os.getcwd()
     if path:
         os.chdir(path)
 
@@ -887,11 +888,12 @@ def main() -> None:
                 except KeyError:
                     pass
 
+                cwe_url = ""
                 try:
                     if error['cwe']:
                         cwe_url = "<a href=\"https://cwe.mitre.org/data/definitions/" + error['cwe'] + ".html\">" + error['cwe'] + "</a>"
                 except KeyError:
-                    cwe_url = ""
+                    pass
 
                 if error['severity'] in ['error', 'warning']:
                     message_class = error['severity']

Diff for: htmlreport/cppcheck-htmlreport

@@ -1463,7 +1463,7 @@ def test_filelist(tmpdir):
     ]
     assert len(expected), len(lines)
     for i in range(1, len(expected)+1):
-        lines.remove('{}/11 files checked 0% done'.format(i, len(expected)))
+        lines.remove('{}/{} files checked 0% done'.format(i, len(expected)))
     assert lines == expected
 
 

Diff for: test/cli/other_test.py

@@ -185,7 +185,7 @@ def test_slow_initlist_varchanged(tmpdir):
                     }
                 }""")
     cppcheck([filename]) # should not take more than ~1 second
-    
+
 
 @pytest.mark.timeout(10)
 def test_slow_many_scopes(tmpdir):

Diff for: test/cli/performance_test.py

@@ -31,8 +31,8 @@ def copy_cppcheck_premium(tmpdir):
                     "safety": false
                 }
                 """.replace('NAME', PRODUCT_NAME))
- 
-    return exe    
+
+    return exe
 
 
 def test_misra_c_builtin_style_checks(tmpdir):

Diff for: test/cli/premium_test.py

@@ -78,11 +78,11 @@ def __lookup_cppcheck_exe():
 def __run_subprocess_tty(args, env=None, cwd=None, timeout=None):
     import pty
     mo, so = pty.openpty()
-    me, se = pty.openpty() 
+    me, se = pty.openpty()
     p = subprocess.Popen(args, stdout=so, stderr=se, env=env, cwd=cwd)
     for fd in [so, se]:
         os.close(fd)
-        
+
     select_timeout = 0.04  # seconds
     readable = [mo, me]
     result = {mo: b'', me: b''}
@@ -110,15 +110,15 @@ def __run_subprocess_tty(args, env=None, cwd=None, timeout=None):
         if p.poll() is None:
             p.kill()
         return_code = p.wait()
-        
+
     stdout = result[mo]
-    stderr = result[me]        
+    stderr = result[me]
     return return_code, stdout, stderr
 
-        
+
 def __run_subprocess(args, env=None, cwd=None, timeout=None):
     p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=cwd)
-    
+
     try:
         comm = p.communicate(timeout=timeout)
         return_code = p.returncode
@@ -143,9 +143,9 @@ def __run_subprocess(args, env=None, cwd=None, timeout=None):
             #os.killpg(os.getpgid(p.pid), signal.SIGTERM)  # Send the signal to all the process groups
             p.terminate()
             comm = p.communicate()
-            
+
     stdout = comm[0]
-    stderr = comm[1]    
+    stderr = comm[1]
     return return_code, stdout, stderr
 
 
@@ -190,13 +190,13 @@ def cppcheck(args, env=None, remove_checkers_report=True, cwd=None, cppcheck_exe
             args.append(arg_executor)
 
     logging.info(exe + ' ' + ' '.join(args))
-    
+
     run_subprocess = __run_subprocess_tty if tty else __run_subprocess
     return_code, stdout, stderr = run_subprocess([exe] + args, env=env, cwd=cwd, timeout=timeout)
-        
+
     stdout = stdout.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n')
     stderr = stderr.decode(encoding='utf-8', errors='ignore').replace('\r\n', '\n')
-    
+
     if remove_checkers_report:
         if stderr.find('[checkersReport]\n') > 0:
             start_id = stderr.find('[checkersReport]\n')

Diff for: test/cli/testutils.py

@@ -26,7 +26,7 @@
 # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
 # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
 # changes)
-SERVER_VERSION = "1.3.53"
+SERVER_VERSION = "1.3.54"
 
 OLD_VERSION = '2.14.0'
 
@@ -1204,7 +1204,7 @@ def run(self):
                 text = check_library_function_name(self.resultPath, var_name, queryParams, nonfunc_id='unknownMacro')
                 httpGetResponse(self.connection, text, 'text/plain')
             else:
-                filename = resultPath + url
+                filename = self.resultPath + url
                 if not os.path.isfile(filename):
                     print_ts('HTTP/1.1 404 Not Found')
                     self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n')

Diff for: tools/donate-cpu-server.py

@@ -691,6 +691,8 @@ def convertFile(self, srcname, destname, line_directive):
         for line in srclines:
             if not modified:
                 line_orig = line
+            else:
+                line_orig = None
 
             linenr += 1
             # Compile Token::Match and Token::simpleMatch