testparser: make it compile too and make a py3 comparison

Author: ncw

parser/testparser/py3compile.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+Compile any files passed in on the command line
+"""
+
+import sys
+for path in sys.argv[1:]:
+    print("Compiling %s" % path)
+    with open(path) as f:
+        try:
+            data = f.read()
+            compile(data, path, "exec")
+        except Exception as e:
+            print("Failed: %s" % e)

parser/testparser/test.sh

@@ -1,8 +1,15 @@
 #!/bin/bash
 # Run testparser over python3 source
+#
+# Pass in args to be passed to testparser, eg -c, -l
+# Parses by default
 
 PY3SOURCE=~/Code/cpython
 
 go install
 
-find $PY3SOURCE -type f -name \*.py | grep -v "lib2to3/tests" | xargs testparser
+# Grep out python2 source code which we can't parse and files with deliberate syntax errors
+find $PY3SOURCE -type f -name \*.py | egrep -v "Lib/(lib2to3/tests|test/bad.*py)|Tools/(hg|msi|test2to3)/" | xargs testparser "$@"
+
+#find $PY3SOURCE -type f -name \*.py | egrep -v "Lib/(lib2to3/tests|test/bad.*py)|Tools/(hg|msi|test2to3)/" | xargs ./py3compile.py "$@"
+

parser/testparser/testparser.go

@@ -3,37 +3,57 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 
+	"github.com/ncw/gpython/compile"
 	"github.com/ncw/gpython/parser"
 )
 
 var (
-	lex        = flag.Bool("l", false, "Lex the file only")
-	debugLevel = flag.Int("d", 0, "Debug level 0-4")
+	lexFile     = flag.Bool("l", false, "Lex the file only")
+	compileFile = flag.Bool("c", false, "Lex, Parse and compile the file")
+	debugLevel  = flag.Int("d", 0, "Debug level 0-4")
 )
 
 func main() {
 	flag.Parse()
 	parser.SetDebug(*debugLevel)
+	if len(flag.Args()) == 0 {
+		log.Printf("Need files to parse")
+		os.Exit(1)
+	}
 	for _, path := range flag.Args() {
-		if *lex {
+		if *lexFile {
 			fmt.Printf("Lexing %q\n", path)
+		} else if *compileFile {
+			fmt.Printf("Compiling %q\n", path)
 		} else {
 			fmt.Printf("Parsing %q\n", path)
 		}
 		in, err := os.Open(path)
 		if err != nil {
 			log.Fatal(err)
 		}
-		fmt.Printf("-----------------\n")
-		if *lex {
-			_, err = parser.Lex(in, "exec")
+		if *debugLevel > 0 {
+			fmt.Printf("-----------------\n")
+		}
+		if *lexFile {
+			_, err = parser.Lex(in, path, "exec")
+		} else if *compileFile {
+			var input []byte
+			input, err = ioutil.ReadAll(in)
+			if err != nil {
+				log.Fatalf("Failed to read %q: %v", path, err)
+			}
+			_, err = compile.Compile(string(input), path, "exec", 0, false)
 		} else {
-			_, err = parser.Parse(in, "exec")
+			_, err = parser.Parse(in, path, "exec")
+		}
+		if *debugLevel > 0 {
+			fmt.Printf("-----------------\n")
 		}
-		fmt.Printf("-----------------\n")
 		closeErr := in.Close()
 		if err != nil {
 			log.Fatalf("Failed on %q: %v", path, err)