Rename fieldMeta into fieldDesc (#1436)

* Update

* Update

* Update

Author: wangkuiyi

cmd/repl/repl_test.go

@@ -267,7 +267,7 @@ INTO sqlflow_models.mymodel;`
 	a.NoError(err)
 	pbTrain := &irpb.TrainStmt{}
 	proto.UnmarshalText(pbtxt, pbTrain)
-	a.Equal("class", pbTrain.GetLabel().GetNc().GetFieldMeta().GetName())
+	a.Equal("class", pbTrain.GetLabel().GetNc().GetFieldDesc().GetName())
 
 	// run one train SQL to save the model then test predict/analyze use the model
 	sess := &irpb.Session{DbConnStr: dataSourceStr}

doc/design/codegen_couler_use_ir.md

@@ -19,10 +19,10 @@ To implement the single `codegen_couler.go` to support generate code that can ru
             tfFiller := TFFiller{
                 Estimator: generateTFEstimatorCode(ir),
                 FeatureColumns: generateFeatureColumnsCode(ir),
-                FieldMetas: generateFieldMetasCode(ir),
+                FieldDescs: generateFieldDescsCode(ir),
                 ...
             }
-            tfTrainTemplate = `couler.tensorflow.train(estimator="{{.Estimator}}", FeatureColumns="""{{.FeatureColumns}}""", FieldMetas={{.FieldMetas}})`
+            tfTrainTemplate = `couler.tensorflow.train(estimator="{{.Estimator}}", FeatureColumns="""{{.FeatureColumns}}""", FieldDescs={{.FieldDescs}})`
             // Do template rendering here.
         else if ir.ModelType == "XGBoost":
             ...

doc/design/couler_sqlflow.md

@@ -49,7 +49,7 @@ From the above Couler function:
 protobuf text format, the protobuf definition is as follows:
 
 ```protobuf
-message FieldMeta {
+message FieldDesc {
     required string name = 1;
     required FieldType dtype = 2;
     optional string delimiter = 3;
@@ -60,7 +60,7 @@ message FieldMeta {
 }
 
 message NumericColumn {
-    FieldMeta field_meta = 1;
+    FieldDesc field_meta = 1;
 }
 
 message BucketColumn {

pkg/proto/intermediate_representation.proto

@@ -8,7 +8,7 @@ import "sqlflow.proto";
 // All structs should be exactly the same as structs defined in `pkg/sql/ir/ir.go`
 // You may refer to `ir.go` and `feature_column.go` for the meaning of each field.
 
-message FieldMeta {
+message FieldDesc {
     string name = 1;
     string dtype = 2;
     string delimiter = 3;
@@ -19,7 +19,7 @@ message FieldMeta {
 }
 
 message NumericColumn {
-    FieldMeta field_meta = 1;
+    FieldDesc field_desc = 1;
 }
 
 message BucketColumn {
@@ -33,12 +33,12 @@ message CrossColumn {
 }
 
 message CategoryIDColumn {
-    FieldMeta field_meta = 1;
+    FieldDesc field_desc = 1;
     int32 bucket_size = 2;
 }
 
 message SeqCategoryIDColumn {
-    FieldMeta field_meta = 1;
+    FieldDesc field_desc = 1;
     int32 bucket_size = 2;
 }
 

pkg/sql/codegen/couler/codegen_test.go

@@ -70,11 +70,11 @@ INTO sqlflow_models.my_xgboost_model;
 				"model.n_classes":    3},
 			Features: map[string][]ir.FeatureColumn{
 				"feature_columns": {
-					&ir.NumericColumn{&ir.FieldMeta{"sepal_length", ir.Float, "", []int{1}, false, nil, 0}},
-					&ir.NumericColumn{&ir.FieldMeta{"sepal_width", ir.Float, "", []int{1}, false, nil, 0}},
-					&ir.NumericColumn{&ir.FieldMeta{"petal_length", ir.Float, "", []int{1}, false, nil, 0}},
-					&ir.NumericColumn{&ir.FieldMeta{"petal_width", ir.Float, "", []int{1}, false, nil, 0}}}},
-			Label: &ir.NumericColumn{&ir.FieldMeta{"class", ir.Int, "", []int{1}, false, nil, 0}}},
+					&ir.NumericColumn{&ir.FieldDesc{"sepal_length", ir.Float, "", []int{1}, false, nil, 0}},
+					&ir.NumericColumn{&ir.FieldDesc{"sepal_width", ir.Float, "", []int{1}, false, nil, 0}},
+					&ir.NumericColumn{&ir.FieldDesc{"petal_length", ir.Float, "", []int{1}, false, nil, 0}},
+					&ir.NumericColumn{&ir.FieldDesc{"petal_width", ir.Float, "", []int{1}, false, nil, 0}}}},
+			Label: &ir.NumericColumn{&ir.FieldDesc{"class", ir.Int, "", []int{1}, false, nil, 0}}},
 	}
 }
 

pkg/sql/codegen/tensorflow/codegen.go

@@ -61,8 +61,8 @@ func generateFeatureColumnCode(fc ir.FeatureColumn) (string, error) {
 	case *ir.NumericColumn:
 		nc := fc.(*ir.NumericColumn)
 		return fmt.Sprintf("tf.feature_column.numeric_column(\"%s\", shape=%s)",
-			nc.FieldMeta.Name,
-			intArrayToJSONString(nc.FieldMeta.Shape)), nil
+			nc.FieldDesc.Name,
+			intArrayToJSONString(nc.FieldDesc.Shape)), nil
 	case *ir.BucketColumn:
 		bc := fc.(*ir.BucketColumn)
 		sourceCode, err := generateFeatureColumnCode(bc.SourceColumn)
@@ -76,11 +76,11 @@ func generateFeatureColumnCode(fc ir.FeatureColumn) (string, error) {
 	case *ir.CategoryIDColumn:
 		cc := fc.(*ir.CategoryIDColumn)
 		return fmt.Sprintf("tf.feature_column.categorical_column_with_identity(key=\"%s\", num_buckets=%d)",
-			cc.FieldMeta.Name, cc.BucketSize), nil
+			cc.FieldDesc.Name, cc.BucketSize), nil
 	case *ir.SeqCategoryIDColumn:
 		cc := fc.(*ir.SeqCategoryIDColumn)
 		return fmt.Sprintf("tf.feature_column.sequence_categorical_column_with_identity(key=\"%s\", num_buckets=%d)",
-			cc.FieldMeta.Name, cc.BucketSize), nil
+			cc.FieldDesc.Name, cc.BucketSize), nil
 	case *ir.CrossColumn:
 		cc := fc.(*ir.CrossColumn)
 		var keysGenerated = make([]string, len(cc.Keys))
@@ -226,26 +226,25 @@ func setValidateParamDefaultValues(validateParams map[string]interface{}) {
 	}
 }
 
-func deriveFeatureColumnCode(trainStmt *ir.TrainStmt) (featureColumnsCode []string, fieldMetas []*ir.FieldMeta, err error) {
+func deriveFeatureColumnCode(trainStmt *ir.TrainStmt) (featureColumnsCode []string, fieldDescs []*ir.FieldDesc, err error) {
 	perTargetFeatureColumnsCode := []string{}
-
 	for target, fcList := range trainStmt.Features {
 		for _, fc := range fcList {
 			fcCode, err := generateFeatureColumnCode(fc)
 			if err != nil {
 				return nil, nil, err
 			}
 			perTargetFeatureColumnsCode = append(perTargetFeatureColumnsCode, fcCode)
-			if len(fc.GetFieldMeta()) > 0 {
-				for _, fm := range fc.GetFieldMeta() {
-					fieldMetas = append(fieldMetas, fm)
+			if len(fc.GetFieldDesc()) > 0 {
+				for _, fm := range fc.GetFieldDesc() {
+					fieldDescs = append(fieldDescs, fm)
 				}
 			}
 		}
 		featureColumnsCode = append(featureColumnsCode,
 			fmt.Sprintf("\"%s\": [%s]", target, strings.Join(perTargetFeatureColumnsCode, ",\n")))
 	}
-	return featureColumnsCode, fieldMetas, nil
+	return featureColumnsCode, fieldDescs, nil
 }
 
 // Train generates a Python program for train a TensorFlow model.
@@ -259,7 +258,7 @@ func Train(trainStmt *ir.TrainStmt) (string, error) {
 	setTrainParamDefaultValues(trainParams)
 	setValidateParamDefaultValues(validateParams)
 
-	featureColumnsCode, fieldMetas, err := deriveFeatureColumnCode(trainStmt)
+	featureColumnsCode, fieldDescs, err := deriveFeatureColumnCode(trainStmt)
 	if err != nil {
 		return "", err
 	}
@@ -286,9 +285,9 @@ func Train(trainStmt *ir.TrainStmt) (string, error) {
 		ValidationSelect:  trainStmt.ValidationSelect,
 		Estimator:         estimatorStr,
 		IsKerasModel:      isKeras,
-		FieldMetas:        fieldMetas,
+		FieldDescs:        fieldDescs,
 		FeatureColumnCode: fmt.Sprintf("{%s}", strings.Join(featureColumnsCode, ",\n")),
-		Y:                 trainStmt.Label.GetFieldMeta()[0], // TODO(typhoonzero): label only support numericColumn.
+		Y:                 trainStmt.Label.GetFieldDesc()[0], // TODO(typhoonzero): label only support numericColumn.
 		ModelParams:       modelParams,
 		TrainParams:       trainParams,
 		ValidationParams:  validateParams,
@@ -319,29 +318,29 @@ func Pred(predStmt *ir.PredictStmt, session *pb.Session) (string, error) {
 	}
 	featureColumnsCode := []string{}
 	perTargetFeatureColumnsCode := []string{}
-	fieldMetas := []*ir.FieldMeta{}
+	fieldDescs := []*ir.FieldDesc{}
 	for target, fcList := range predStmt.TrainStmt.Features {
 		for _, fc := range fcList {
 			fcCode, err := generateFeatureColumnCode(fc)
 			if err != nil {
 				return "", err
 			}
 			perTargetFeatureColumnsCode = append(perTargetFeatureColumnsCode, fcCode)
-			if len(fc.GetFieldMeta()) > 0 {
-				for _, fm := range fc.GetFieldMeta() {
-					fieldMetas = append(fieldMetas, fm)
+			if len(fc.GetFieldDesc()) > 0 {
+				for _, fm := range fc.GetFieldDesc() {
+					fieldDescs = append(fieldDescs, fm)
 				}
 			}
 		}
 		featureColumnsCode = append(featureColumnsCode,
 			fmt.Sprintf("\"%s\": [%s]", target, strings.Join(perTargetFeatureColumnsCode, ",\n")))
 	}
 	isKeras, estimatorStr := IsKerasModel(predStmt.TrainStmt.Estimator)
-	labelFM := predStmt.TrainStmt.Label.GetFieldMeta()[0]
+	labelFM := predStmt.TrainStmt.Label.GetFieldDesc()[0]
 	if labelFM.Name == "" {
 		log.Printf("clustering model, got result table: %s, result column: %s", predStmt.ResultTable, predStmt.ResultColumn)
-		// no label in train SQL means a clustering model, generate a fieldmeta using result table's column
-		labelFM = &ir.FieldMeta{
+		// no label in train SQL means a clustering model, generate a fieldDesc using result table's column
+		labelFM = &ir.FieldDesc{
 			Name:  predStmt.ResultColumn,
 			Shape: []int{1},
 			DType: ir.Int,
@@ -354,7 +353,7 @@ func Pred(predStmt *ir.PredictStmt, session *pb.Session) (string, error) {
 		ResultTable:       predStmt.ResultTable,
 		Estimator:         estimatorStr,
 		IsKerasModel:      isKeras,
-		FieldMetas:        fieldMetas,
+		FieldDescs:        fieldDescs,
 		FeatureColumnCode: fmt.Sprintf("{%s}", strings.Join(featureColumnsCode, ",\n")),
 		Y:                 labelFM,
 		ModelParams:       modelParams,

pkg/sql/codegen/tensorflow/template_pred.go

@@ -22,9 +22,9 @@ type predFiller struct {
 	// below members comes from trainStmt
 	Estimator         string
 	IsKerasModel      bool
-	FieldMetas        []*ir.FieldMeta
+	FieldDescs        []*ir.FieldDesc
 	FeatureColumnCode string
-	Y                 *ir.FieldMeta
+	Y                 *ir.FieldDesc
 	ModelParams       map[string]interface{}
 	Save              string
 	HDFSNameNodeAddr  string
@@ -41,12 +41,12 @@ try:
 except:
     pass
 
-feature_column_names = [{{range .FieldMetas}}
+feature_column_names = [{{range .FieldDescs}}
 "{{.Name}}",
 {{end}}]
 
 feature_metas = dict()
-{{ range $value := .FieldMetas }}
+{{ range $value := .FieldDescs }}
 feature_metas["{{$value.Name}}"] = {
     "feature_name": "{{$value.Name}}",
     "dtype": "{{$value.DType | dtypeToString}}",

pkg/sql/codegen/tensorflow/template_train.go

@@ -21,9 +21,9 @@ type trainFiller struct {
 	ValidationSelect  string
 	Estimator         string
 	IsKerasModel      bool
-	FieldMetas        []*ir.FieldMeta
+	FieldDescs        []*ir.FieldDesc
 	FeatureColumnCode string
-	Y                 *ir.FieldMeta
+	Y                 *ir.FieldDesc
 	ModelParams       map[string]interface{}
 	TrainParams       map[string]interface{}
 	ValidationParams  map[string]interface{}
@@ -40,12 +40,12 @@ try:
 except:
     pass
 
-feature_column_names = [{{range .FieldMetas}}
+feature_column_names = [{{range .FieldDescs}}
 "{{.Name}}",
 {{end}}]
 
 feature_metas = dict()
-{{ range $value := .FieldMetas }}
+{{ range $value := .FieldDescs }}
 feature_metas["{{$value.Name}}"] = {
     "feature_name": "{{$value.Name}}",
     "dtype": "{{$value.DType | dtypeToString}}",

pkg/sql/codegen/xgboost/codegen.go

@@ -74,23 +74,23 @@ func parseAttribute(attrs map[string]interface{}) (map[string]map[string]interfa
 	return params, nil
 }
 
-func getFieldMeta(fcs []ir.FeatureColumn, l ir.FeatureColumn) ([]ir.FieldMeta, ir.FieldMeta, error) {
-	var features []ir.FieldMeta
+func getFieldDesc(fcs []ir.FeatureColumn, l ir.FeatureColumn) ([]ir.FieldDesc, ir.FieldDesc, error) {
+	var features []ir.FieldDesc
 	for _, fc := range fcs {
 		switch c := fc.(type) {
 		case *ir.NumericColumn:
-			features = append(features, *c.FieldMeta)
+			features = append(features, *c.FieldDesc)
 		default:
-			return nil, ir.FieldMeta{}, fmt.Errorf("unsupported feature column type %T on %v", c, c)
+			return nil, ir.FieldDesc{}, fmt.Errorf("unsupported feature column type %T on %v", c, c)
 		}
 	}
 
-	var label ir.FieldMeta
+	var label ir.FieldDesc
 	switch c := l.(type) {
 	case *ir.NumericColumn:
-		label = *c.FieldMeta
+		label = *c.FieldDesc
 	default:
-		return nil, ir.FieldMeta{}, fmt.Errorf("unsupported label column type %T on %v", c, c)
+		return nil, ir.FieldDesc{}, fmt.Errorf("unsupported label column type %T on %v", c, c)
 	}
 
 	return features, label, nil
@@ -111,7 +111,7 @@ func Train(trainStmt *ir.TrainStmt) (string, error) {
 	if len(trainStmt.Features) != 1 {
 		return "", fmt.Errorf("xgboost only support 1 feature column set, received %d", len(trainStmt.Features))
 	}
-	featureFieldMeta, labelFieldMeta, err := getFieldMeta(trainStmt.Features["feature_columns"], trainStmt.Label)
+	featureFieldDesc, labelFieldDesc, err := getFieldDesc(trainStmt.Features["feature_columns"], trainStmt.Label)
 	if err != nil {
 		return "", err
 	}
@@ -123,11 +123,11 @@ func Train(trainStmt *ir.TrainStmt) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	f, err := json.Marshal(featureFieldMeta)
+	f, err := json.Marshal(featureFieldDesc)
 	if err != nil {
 		return "", err
 	}
-	l, err := json.Marshal(labelFieldMeta)
+	l, err := json.Marshal(labelFieldDesc)
 	if err != nil {
 		return "", err
 	}
@@ -137,7 +137,7 @@ func Train(trainStmt *ir.TrainStmt) (string, error) {
 		ValidationSelect: trainStmt.ValidationSelect,
 		ModelParamsJSON:  string(mp),
 		TrainParamsJSON:  string(tp),
-		FieldMetaJSON:    string(f),
+		FieldDescJSON:    string(f),
 		LabelJSON:        string(l)}
 
 	var program bytes.Buffer
@@ -150,15 +150,15 @@ func Train(trainStmt *ir.TrainStmt) (string, error) {
 
 // Pred generates a Python program for predict a xgboost model.
 func Pred(predStmt *ir.PredictStmt, session *pb.Session) (string, error) {
-	featureFieldMeta, labelFieldMeta, err := getFieldMeta(predStmt.TrainStmt.Features["feature_columns"], predStmt.TrainStmt.Label)
+	featureFieldDesc, labelFieldDesc, err := getFieldDesc(predStmt.TrainStmt.Features["feature_columns"], predStmt.TrainStmt.Label)
 	if err != nil {
 		return "", err
 	}
-	f, err := json.Marshal(featureFieldMeta)
+	f, err := json.Marshal(featureFieldDesc)
 	if err != nil {
 		return "", err
 	}
-	l, err := json.Marshal(labelFieldMeta)
+	l, err := json.Marshal(labelFieldDesc)
 	if err != nil {
 		return "", err
 	}

pkg/sql/codegen/xgboost/codegen_analyze.go

@@ -36,7 +36,7 @@ func Analyze(analyzeStmt *ir.AnalyzeStmt) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	xs, y, err := getFieldMeta(analyzeStmt.TrainStmt.Features["feature_columns"], analyzeStmt.TrainStmt.Label)
+	xs, y, err := getFieldDesc(analyzeStmt.TrainStmt.Features["feature_columns"], analyzeStmt.TrainStmt.Label)
 	if err != nil {
 		return "", err
 	}
@@ -49,7 +49,7 @@ func Analyze(analyzeStmt *ir.AnalyzeStmt) (string, error) {
 		DataSource:         analyzeStmt.DataSource,
 		DatasetSQL:         analyzeStmt.Select,
 		ShapSummaryParames: string(jsonSummary),
-		FieldMetaJSON:      string(fm),
+		FieldDescJSON:      string(fm),
 		Label:              y.Name,
 	}
 	var analysis bytes.Buffer