fix bugs introduced for versions of clang < 7.0

Author: andrew-taylor

Makefile

@@ -20,7 +20,10 @@ dcc.1: dcc help2man_include.txt
 	help2man --include=help2man_include.txt ./dcc >dcc.1
 
 tests: dcc
-	tests/do_tests.sh
+	tests/do_tests.sh ./dcc
+	
+tests_all_clang_versions: dcc
+	set -x ; for compiler in /usr/bin/clang-[1-24-9]* ; do tests/do_tests.sh ./dcc $$compiler; done
 
 debian: dcc
 	rm -rf debian

compile.py

@@ -10,8 +10,9 @@
 COMMON_WARNING_ARGS = "-Wall -Wno-unused -Wunused-variable -Wunused-value -Wno-unused-result".split()
 COMMON_COMPILER_ARGS = COMMON_WARNING_ARGS + "-std=gnu11 -g -lm".split()
 
-EXTRA_C_COMPILER_ARGS = COMMON_COMPILER_ARGS + "-Wunused-comparison -fcolor-diagnostics  -fno-omit-frame-pointer -fno-common -funwind-tables -fno-optimize-sibling-calls -Qunused-arguments".split()
-GCC_ARGS = COMMON_COMPILER_ARGS + "-Wunused-but-set-variable -O -fdiagnostics-color -o /dev/null".split()
+CLANG_ONLY_ARGS = "-Wunused-comparison -fno-omit-frame-pointer -fno-common -funwind-tables -fno-optimize-sibling-calls -Qunused-arguments".split()
+
+GCC_ARGS = COMMON_COMPILER_ARGS + "-Wunused-but-set-variable -O  -o /dev/null".split()
 
 MAXIMUM_SOURCE_FILE_EMBEDDED_BYTES = 1000000
 
@@ -25,7 +26,13 @@
 def compile(debug=False):
 	os.environ['PATH'] = os.path.dirname(os.path.realpath(sys.argv[0])) + ':/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:' + os.environ.get('PATH', '') 
 	args = parse_args(sys.argv[1:])
-
+	
+	# we have to set these explicitly because 
+	clang_args = COMMON_COMPILER_ARGS + CLANG_ONLY_ARGS
+	if args.colorize_output:
+		clang_args += ['-fcolor-diagnostics']
+		clang_args += ['-fdiagnostics-color']
+		
 	if args.which_sanitizer == "memory" and platform.architecture()[0][0:2] == '32':
 		if search_path('valgrind'):
 			# -fsanitize=memory requires 64-bits so we fallback to embedding valgrind
@@ -37,18 +44,21 @@ def compile(debug=False):
 
 	clang_version = None
 	try:
-		clang_version = subprocess.check_output(["clang", "--version"], universal_newlines=True)
+		clang_version = subprocess.check_output([args.c_compiler, "--version"], universal_newlines=True)
 		if debug:
 			print("clang version:", clang_version)
-		m = re.search("clang version (\d+\.\d+\.\d+)", clang_version, flags=re.I)
+		m = re.search("clang version ((\d+)\.(\d+)\.\d+)", clang_version, flags=re.I)
 		if m is not None:
 			clang_version = m.group(1)
+			clang_version_major = m.group(2)
+			clang_version_minor = m.group(3)
+			clang_version_float = float(m.group(2) + "." + m.group(3))
 	except OSError as e:
 		if debug:
 			print(e)
 
 	if not clang_version:
-		print("Can not get clang version", file=sys.stderr)
+		print("Can not get version information for '%s'" % args.c_compiler, file=sys.stderr)
 		sys.exit(1)
 
 	if args.which_sanitizer == "address" and platform.architecture()[0][0:2] == '32':
@@ -66,7 +76,7 @@ def compile(debug=False):
 			if debug:
 				print(e)
 
-		if clang_version and libc_version and clang_version[0] in "345" and libc_version >= 2.27:
+		if  libc_version and clang_version_float < 6 and libc_version >= 2.27:
 			print("incompatible clang libc versions, disabling error detection by sanitiziers", file=sys.stderr)
 			sanitizer_args = []
 
@@ -79,8 +89,6 @@ def compile(debug=False):
 		tar_n_bytes, tar_source = source_for_embedded_tarfile(args) 
 
 	if args.which_sanitizer == "valgrind":
-		sanitizer_args = []
-		sanitizer_args = ['-fsanitize=undefined', '-fno-sanitize-recover=undefined,integer']
 		wrapper_source = wrapper_source.replace('__DCC_SANITIZER_IS_VALGRIND__', '1')
 		if args.embed_source:
 			watcher = fr"python3 -E -c \"import os,sys,tarfile,tempfile\n\
@@ -92,27 +100,36 @@ def compile(debug=False):
 		else:
 			watcher = dcc_path +  "--watch-stdin-for-valgrind-errors"
 		wrapper_source = wrapper_source.replace('__DCC_MONITOR_VALGRIND__', watcher)
+		sanitizer_args = []
 	elif args.which_sanitizer == "memory":
 		wrapper_source = wrapper_source.replace('__DCC_SANITIZER_IS_MEMORY__', '1')
-		# FIXME if we enable  '-fsanitize=undefined', '-fno-sanitize-recover=undefined,integer'
-		# which would be preferable here we get uninitialized variable error message for undefined errors
+		args.which_sanitizer = "memory"
 		sanitizer_args = ['-fsanitize=memory']
 	else:
 		wrapper_source = wrapper_source.replace('__DCC_SANITIZER_IS_ADDRESS__', '1')
 		# fixme add code to check version supports these
-		sanitizer_args = ['-fsanitize=address', '-fsanitize=undefined', '-fno-sanitize-recover=undefined,integer']
+		sanitizer_args = ['-fsanitize=address']
 		args.which_sanitizer = "address"
+		
+	if args.which_sanitizer != "memory":
+		# FIXME if we enable  '-fsanitize=undefined', '-fno-sanitize-recover=undefined,integer' for memory
+		# which would be preferable here we get uninitialized variable error message for undefined errors
+		sanitizer_args += ['-fsanitize=undefined']
+		if clang_version_float >= 3.6:
+			sanitizer_args += ['-fno-sanitize-recover=undefined,integer']
 
 	wrapper_source = wrapper_source.replace('__DCC_LEAK_CHECK_YES_NO__', "yes" if args.leak_check else "no")
 	wrapper_source = wrapper_source.replace('__DCC_LEAK_CHECK_1_0__', "1" if args.leak_check else "0")
 	wrapper_source = wrapper_source.replace('__DCC_SUPRESSIONS_FILE__', args.suppressions_file)
 	wrapper_source = wrapper_source.replace('__DCC_STACK_USE_AFTER_RETURN__', "1" if args.stack_use_after_return else "0")
 	wrapper_source = wrapper_source.replace('__DCC_NO_WRAP_MAIN__', "1" if args.no_wrap_main else "0")
+	wrapper_source = wrapper_source.replace('__DCC_CLANG_VERSION_MAJOR__', clang_version_major)
+	wrapper_source = wrapper_source.replace('__DCC_CLANG_VERSION_MINOR__', clang_version_minor)
 
 	# shared_libasan breaks easily ,e.g if there are libraries in  /etc/ld.so.preload
 	# and we can't override with verify_asan_link_order=0 for clang version < 5
 	# and with clang-6 on debian __asan_default_options not called with shared_libasan
-	if args.shared_libasan is None and clang_version[0] not in "3456":
+	if args.shared_libasan is None and clang_version_float >= 7.0:
 		args.shared_libasan = True
 
 	if args.shared_libasan and args.which_sanitizer == "address":
@@ -124,15 +141,15 @@ def compile(debug=False):
 		wrapper_source = wrapper_source.replace('__DCC_EMBED_SOURCE__', '1')
 		wrapper_source = tar_source + wrapper_source
 
-	incremental_compilation_args = sanitizer_args + EXTRA_C_COMPILER_ARGS + args.user_supplied_compiler_args
+	incremental_compilation_args = sanitizer_args + clang_args + args.user_supplied_compiler_args
 	command = [args.c_compiler] + incremental_compilation_args
 	if args.incremental_compilation:
 		if args.debug:
 			print('incremental compilation, running: ', " ".join(command), file=sys.stderr)
 		sys.exit(subprocess.call(command))
 
 	# -x - must come after any filenames but before ld options
-	command =  [args.c_compiler] + args.user_supplied_compiler_args + ['-x', 'c', '-', ] + sanitizer_args + EXTRA_C_COMPILER_ARGS
+	command =  [args.c_compiler] + args.user_supplied_compiler_args + ['-x', 'c', '-', ] + sanitizer_args + clang_args
 	if args.ifdef_main:
 		command +=  ['-Dmain=__real_main']
 		wrapper_source = wrapper_source.replace('__wrap_main', 'main')
@@ -141,10 +158,16 @@ def compile(debug=False):
 
 	if args.debug:
 		print(" ".join(command), file=sys.stderr)
+
 	if args.debug > 1:
-		print(" ".join(command), '<<eof', file=sys.stderr)
-		print(wrapper_source)
-		print("eof", file=sys.stderr)
+		debug_wrapper_file = "dcc_main_wrapper.c"
+		print("Leaving main_wrapper in", debug_wrapper_file, "compile with this command:", file=sys.stderr)
+		print(" ".join(command).replace('-x c -', debug_wrapper_file), file=sys.stderr)
+		try:
+			with open(debug_wrapper_file,"w") as f:
+				f.write(wrapper_source)
+		except OSError as e:
+			print(e)
 	process = subprocess.run(command, input=wrapper_source, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
 
 	# workaround for  https://github.com/android-ndk/ndk/issues/184
@@ -199,7 +222,8 @@ class Args(object):
 	user_supplied_compiler_args = []
 	explanations = True
 	max_explanations = 3
-	embed_source = True 
+	embed_source = True
+	# FIXME - check terminal actually support ANSI
 	colorize_output = sys.stderr.isatty() or os.environ.get('DCC_COLORIZE_OUTPUT', False)
 	debug = int(os.environ.get('DCC_DEBUG', '0'))
 	source_files = set()
@@ -259,6 +283,10 @@ def parse_arg(arg, next_arg, args):
 		args.no_wrap_main = True
 	elif arg.startswith('--c-compiler='):
 		args.c_compiler = arg[arg.index('=') + 1:]
+	elif arg == '-fcolor-diagnostics':
+		args.colorize_output = True
+	elif arg == '-fno-color-diagnostics':
+		args.colorize_output = False
 	elif arg == '-v' or arg == '--version':
 		print('dcc version', VERSION)
 		sys.exit(0)
@@ -282,10 +310,6 @@ def parse_clang_arg(arg, next_arg, args):
 	args.user_supplied_compiler_args.append(arg)
 	if arg  == '-c':
 		args.incremental_compilation = True
-	elif arg == '-fcolor-diagnostics':
-		args.colorize_output = True
-	elif arg == '-fno-color-diagnostics':
-		args.colorize_output = False
 	elif arg == '-o' and next_arg:
 		object_filename = next_arg
 		if object_filename.endswith('.c') and os.path.exists(object_filename):

main_wrapper.c

@@ -16,9 +16,17 @@
 
 static int debug = 0;
 
-void __dcc_start(void) __attribute__((constructor));
+void __dcc_start(void) __attribute__((constructor))
+#if __has_attribute(no_sanitize)
+__attribute__((no_sanitize("address", "memory", "undefined")))
+#endif
+;
 
-static void _dcc_exit(void);
+static void _dcc_exit(void)
+#if __has_attribute(no_sanitize)
+__attribute__((no_sanitize("address", "memory", "undefined")))
+#endif
+;
 
 static void _signal_handler(int signum)
 #if __has_attribute(no_sanitize)
@@ -63,6 +71,12 @@ __attribute__((optnone))
 
 #undef main
 
+int __wrap_main(int argc, char *argv[], char *envp[])
+#if __has_attribute(no_sanitize)
+__attribute__((no_sanitize("address", "memory", "undefined")))
+#endif
+;
+
 #if !__DCC_SANITIZER_IS_VALGRIND__
 
 // wrapping ASAN
@@ -83,7 +97,6 @@ int __wrap_main(int argc, char *argv[], char *envp[]) {
 #else
 
 // wrapping valgrind
-
 int __wrap_main(int argc, char *argv[], char *envp[]) {
 	extern int __real_main(int argc, char *argv[], char *envp[]);
 	char mypath[PATH_MAX];
@@ -217,7 +230,7 @@ void __asan_on_error() {
 	if (debug) fprintf(stderr, "__asan_on_error\n");
 
 	char *report = "";
-#if __DCC_SANITIZER_IS_ADDRESS__
+#if __DCC_SANITIZER_IS_ADDRESS__ && __DCC_CLANG_VERSION_MAJOR__ >= 6
 	extern char *__asan_get_report_description();
 	extern  int __asan_report_present();
 	if (__asan_report_present()) {
@@ -244,17 +257,18 @@ char *__msan_default_options() {
 	return "verbosity=0:print_stacktrace=1:halt_on_error=1:detect_leaks=__DCC_LEAK_CHECK_1_0__";
 }
 
-extern void __ubsan_get_current_report_data(char **OutIssueKind, char **OutMessage, char **OutFilename, unsigned int *OutLine, unsigned int *OutCol, char **OutMemoryAddr);
-
 void __ubsan_on_report(void) {
 	if (debug) fprintf(stderr, "__ubsan_on_report\n");
 
+#if __DCC_CLANG_VERSION_MAJOR__ >= 7 && !__DCC_SANITIZER_IS_MEMORY__
 	char *OutIssueKind;
 	char *OutMessage;
 	char *OutFilename;
 	unsigned int OutLine;
 	unsigned int OutCol;
 	char *OutMemoryAddr;
+	extern void __ubsan_get_current_report_data(char **OutIssueKind, char **OutMessage, char **OutFilename, unsigned int *OutLine, unsigned int *OutCol, char **OutMemoryAddr);
+
 	__ubsan_get_current_report_data(&OutIssueKind, &OutMessage, &OutFilename, &OutLine, &OutCol, &OutMemoryAddr);
 
 	// buffer + putenv is ugly - but safer?
@@ -267,7 +281,7 @@ void __ubsan_on_report(void) {
 	snprintf(buffer[5], sizeof buffer[5], "DCC_UBSAN_ERROR_MEMORYADDR=%s", OutMemoryAddr);
 	for (int i = 0; i < sizeof buffer/sizeof buffer[0]; i++) 
 		putenv(buffer[i]);
-
+#endif
 	_explain_error();
 	// not reached
 }

packaging/debian/debian/control

@@ -7,7 +7,7 @@ Build-Depends: debhelper (>= 9)
 Package: dcc
 Architecture: all
 Depends: python3 (>= 3.6), gdb(>= 7.12), clang(>= 4.0)
-Recommends: valgrind(>= 1:3.13)
+Recommends: valgrind(>= 1:3.13), clang(>= 7.0)
 Homepage: https://github.com/COMP1511UNSW/dcc
 Description: compiler for novice C programmers
  dcc compiles C programs using clang and adds explanations suitable

tests/do_tests.sh

@@ -39,15 +39,16 @@ for src_file in tests/extracted_compile_time_tests/*.c tests/compile_time/*.c te
 do
 	rm -f a.out
 
-	compile_options_list=$(egrep '^//dcc_flags=' "$src_file"|sed 's?//??;s? ?#?g')
+	compile_options_list=$(egrep '^//dcc_flags=' "$src_file"|sed 's?//??;s/ /#/g')
 	compile_options_list=${compile_options_list:-'dcc_flags=""'}
 
 	for compile_options in $compile_options_list
 	do
+		compile_options=$(echo "$compile_options"|sed 's/#/ /g')
 		case "$src_file" in
 		*.c)
 			dcc_flags=
-			suffix=`echo $compile_options|sed 's/^dcc_flags=//;s/["$]//g;s/src_file//'`
+			suffix=`echo $compile_options|sed 's/^dcc_flags=//;s/ /_/g;s/["$]//g;s/src_file//'`
 			eval $compile_options
 			expected_stderr_file="$expected_output_dir/`basename $src_file .c`$suffix.txt"
 			#echo "$dcc" --c-compiler=$c_compiler $dcc_flags "$src_file"

tests/expected_output/clang-6.0-x86_64-pc-linux-gnu/add_int_to_string.txt

@@ -1,10 +1,10 @@
-tests/compile_time/add_int_to_string.c:4:17: warning: adding 'int' to a string does not append to the string [-Wstring-plus-int]
+tests/compile_time/add_int_to_string.c:4:17: warning: adding 'int' to a string does not append to the string [-Wstring-plus-int]
         printf("hello" + argc);
-               ~~~~~~~~^~~~~~
-tests/compile_time/add_int_to_string.c:4:17: note: use array indexing to silence this warning
+               ~~~~~~~~^~~~~~
+tests/compile_time/add_int_to_string.c:4:17: note: use array indexing to silence this warning
         printf("hello" + argc);
-                       ^
-               &       [     ]
+                       ^
+               &       [     ]
 dcc explanation: Careful, you can't concatenate values and strings in C using the `+` operator, as you seem to be trying to do on line 4 of `tests/compile_time/add_int_to_string.c`.
  Odds are you want to provide `printf` with a format code for that value and pass that value to `printf` as an argument.
 

tests/expected_output/clang-6.0-x86_64-pc-linux-gnu/array_static_illegal_index.txt

@@ -1,9 +1,9 @@
-tests/compile_time/array_static_illegal_index.c:3:2: warning: array index 5 is past the end of the array (which contains 5 elements) [-Warray-bounds]
+tests/compile_time/array_static_illegal_index.c:3:2: warning: array index 5 is past the end of the array (which contains 5 elements) [-Warray-bounds]
         a[5] = 0;
-        ^ ~
-tests/compile_time/array_static_illegal_index.c:2:2: note: array 'a' declared here
+        ^ ~
+tests/compile_time/array_static_illegal_index.c:2:2: note: array 'a' declared here
         int a[5];
-        ^
+        ^
 dcc explanation: Careful, on line 3 of `tests/compile_time/array_static_illegal_index.c`, it looks like you're trying to access location 5 of `a`, which doesn't exist; `a` isn't that long.
  Keep in mind that arrays are 0-indexed.
 

tests/expected_output/clang-6.0-x86_64-pc-linux-gnu/array_string_index.txt

@@ -1,6 +1,6 @@
-tests/compile_time/array_string_index.c:3:3: error: array subscript is not an integer
+tests/compile_time/array_string_index.c:3:3: error: array subscript is not an integer
         a["0"] = 0;
-         ^~~~
+         ^~~~
 dcc explanation: Looks like you're trying to access an element of the array `a` on line 3 of `tests/compile_time/array_string_index.c`, but your index (`"0"`) is not of type `int`.
  Right now, your index is of type `string` instead.
  Make sure your index (the value between square brackets) is an `int`.

tests/expected_output/clang-6.0-x86_64-pc-linux-gnu/assert_without_closing_parenthesis.txt

@@ -1,4 +1,4 @@
-tests/extracted_compile_time_tests/assert_without_closing_parenthesis.c:5:2: error: unterminated function-like macro invocation
+tests/extracted_compile_time_tests/assert_without_closing_parenthesis.c:5:2: error: unterminated function-like macro invocation
         assert(argc == 1;
-        ^
+        ^
 dcc explanation: it looks like there is a missing closing bracket on the assert on line 5 of tests/extracted_compile_time_tests/assert_without_closing_parenthesis.c

tests/expected_output/clang-6.0-x86_64-pc-linux-gnu/assign_array_to_int.txt

@@ -1,5 +1,5 @@
-tests/extracted_compile_time_tests/assign_array_to_int.c:4:10: warning: incompatible pointer to integer conversion assigning to 'int' from 'int [3]' [-Wint-conversion]
+tests/extracted_compile_time_tests/assign_array_to_int.c:4:10: warning: incompatible pointer to integer conversion assigning to 'int' from 'int [3]' [-Wint-conversion]
         a[0][0] = a[1];
-                ^ ~~~~
+                ^ ~~~~
 dcc explanation: you are attempting to assign a[1] which is an array to an int variable.