cp to 2.0 'fix a bug in shuffle which cause panic' (#19694)

fix a bug in shuffle which cause panic

Approved by: @ouyuanning, @sukki37, @aunjgr

Author: badboynt1

pkg/sql/plan/shuffle.go

@@ -247,16 +247,16 @@ func maybeSorted(n *plan.Node, builder *QueryBuilder, tag int32) bool {
 	return false
 }
 
-func determinShuffleType(col *plan.ColRef, n *plan.Node, builder *QueryBuilder) {
+func determinShuffleType(col *plan.ColRef, node *plan.Node, builder *QueryBuilder) {
 	// hash by default
-	n.Stats.HashmapStats.ShuffleType = plan.ShuffleType_Hash
+	node.Stats.HashmapStats.ShuffleType = plan.ShuffleType_Hash
 
 	if builder == nil {
 		return
 	}
 	tableDef, ok := builder.tag2Table[col.RelPos]
 	if !ok {
-		child := builder.qry.Nodes[n.Children[0]]
+		child := builder.qry.Nodes[node.Children[0]]
 		if child.NodeType == plan.Node_AGG && child.Stats.HashmapStats.Shuffle && col.RelPos == child.BindingTags[0] {
 			col = child.GroupBy[col.ColPos].GetCol()
 			if col == nil {
@@ -266,38 +266,38 @@ func determinShuffleType(col *plan.ColRef, n *plan.Node, builder *QueryBuilder)
 			if !ok {
 				return
 			}
-			n.Stats.HashmapStats.ShuffleMethod = plan.ShuffleMethod_Reuse
-			n.Stats.HashmapStats.ShuffleType = plan.ShuffleType_Range
-			n.Stats.HashmapStats.HashmapSize = child.Stats.HashmapStats.HashmapSize
-			n.Stats.HashmapStats.ShuffleColMin = child.Stats.HashmapStats.ShuffleColMin
-			n.Stats.HashmapStats.ShuffleColMax = child.Stats.HashmapStats.ShuffleColMax
-			n.Stats.HashmapStats.Ranges = child.Stats.HashmapStats.Ranges
-			n.Stats.HashmapStats.Nullcnt = child.Stats.HashmapStats.Nullcnt
+			node.Stats.HashmapStats.ShuffleMethod = plan.ShuffleMethod_Reuse
+			node.Stats.HashmapStats.ShuffleType = plan.ShuffleType_Range
+			node.Stats.HashmapStats.HashmapSize = child.Stats.HashmapStats.HashmapSize
+			node.Stats.HashmapStats.ShuffleColMin = child.Stats.HashmapStats.ShuffleColMin
+			node.Stats.HashmapStats.ShuffleColMax = child.Stats.HashmapStats.ShuffleColMax
+			node.Stats.HashmapStats.Ranges = child.Stats.HashmapStats.Ranges
+			node.Stats.HashmapStats.Nullcnt = child.Stats.HashmapStats.Nullcnt
 		}
 		return
 	}
 	colName := tableDef.Cols[col.ColPos].Name
 
 	// for shuffle join, if left child is not sorted, the cost will be very high
 	// should use complex shuffle type
-	if n.NodeType == plan.Node_JOIN {
+	if node.NodeType == plan.Node_JOIN {
 		leftSorted := true
 		if GetSortOrder(tableDef, col.ColPos) != 0 {
 			leftSorted = false
 		}
-		if !maybeSorted(builder.qry.Nodes[n.Children[0]], builder, col.RelPos) {
+		if !maybeSorted(builder.qry.Nodes[node.Children[0]], builder, col.RelPos) {
 			leftSorted = false
 		}
 		if !leftSorted {
-			leftCost := builder.qry.Nodes[n.Children[0]].Stats.Outcnt
-			rightCost := builder.qry.Nodes[n.Children[1]].Stats.Outcnt
-			if n.BuildOnLeft {
+			leftCost := builder.qry.Nodes[node.Children[0]].Stats.Outcnt
+			rightCost := builder.qry.Nodes[node.Children[1]].Stats.Outcnt
+			if node.BuildOnLeft {
 				// its better for right join to go shuffle, but can not go complex shuffle
-				if n.BuildOnLeft && leftCost > ShuffleTypeThreshHoldUpperLimit*rightCost {
+				if node.BuildOnLeft && leftCost > ShuffleTypeThreshHoldUpperLimit*rightCost {
 					return
 				}
 			} else if leftCost > ShuffleTypeThreshHoldLowerLimit*rightCost {
-				n.Stats.HashmapStats.ShuffleTypeForMultiCN = plan.ShuffleTypeForMultiCN_Hybrid
+				node.Stats.HashmapStats.ShuffleTypeForMultiCN = plan.ShuffleTypeForMultiCN_Hybrid
 			}
 		}
 	}
@@ -306,16 +306,16 @@ func determinShuffleType(col *plan.ColRef, n *plan.Node, builder *QueryBuilder)
 	if s == nil {
 		return
 	}
-	if n.NodeType == plan.Node_AGG {
+	if node.NodeType == plan.Node_AGG {
 		if shouldUseHashShuffle(s.ShuffleRangeMap[colName]) {
 			return
 		}
 	}
-	n.Stats.HashmapStats.ShuffleType = plan.ShuffleType_Range
-	n.Stats.HashmapStats.ShuffleColMin = int64(s.MinValMap[colName])
-	n.Stats.HashmapStats.ShuffleColMax = int64(s.MaxValMap[colName])
-	n.Stats.HashmapStats.Ranges = shouldUseShuffleRanges(s.ShuffleRangeMap[colName], colName)
-	n.Stats.HashmapStats.Nullcnt = int64(s.NullCntMap[colName])
+	node.Stats.HashmapStats.ShuffleType = plan.ShuffleType_Range
+	node.Stats.HashmapStats.ShuffleColMin = int64(s.MinValMap[colName])
+	node.Stats.HashmapStats.ShuffleColMax = int64(s.MaxValMap[colName])
+	node.Stats.HashmapStats.Ranges = shouldUseShuffleRanges(s.ShuffleRangeMap[colName], colName)
+	node.Stats.HashmapStats.Nullcnt = int64(s.NullCntMap[colName])
 }
 
 // to determine if join need to go shuffle
@@ -378,23 +378,36 @@ func determinShuffleForJoin(n *plan.Node, builder *QueryBuilder) {
 		expr1 = condImpl.F.Args[1]
 	}
 
-	hashCol0, typ := GetHashColumn(expr0)
-	if hashCol0 == nil {
+	leftHashCol, typ := GetHashColumn(expr0)
+	if leftHashCol == nil {
 		return
 	}
-	hashCol1, _ := GetHashColumn(expr1)
-	if hashCol1 == nil {
+	rightHashCol, _ := GetHashColumn(expr1)
+	if rightHashCol == nil {
 		return
 	}
 	//for now ,only support integer and string type
 	switch types.T(typ) {
 	case types.T_int64, types.T_int32, types.T_int16, types.T_uint64, types.T_uint32, types.T_uint16, types.T_varchar, types.T_char, types.T_text:
 		n.Stats.HashmapStats.ShuffleColIdx = int32(idx)
 		n.Stats.HashmapStats.Shuffle = true
-		determinShuffleType(hashCol0, n, builder)
+		determinShuffleType(leftHashCol, n, builder)
+	}
+
+	//recheck shuffle plan
+	if n.Stats.HashmapStats.Shuffle {
 		if n.Stats.HashmapStats.ShuffleType == plan.ShuffleType_Hash && n.Stats.HashmapStats.HashmapSize < threshHoldForHashShuffle {
 			n.Stats.HashmapStats.Shuffle = false
 		}
+		if n.Stats.HashmapStats.ShuffleType == plan.ShuffleType_Range && n.Stats.HashmapStats.Ranges == nil && n.Stats.HashmapStats.ShuffleColMax-n.Stats.HashmapStats.ShuffleColMin < 100000 {
+			n.Stats.HashmapStats.Shuffle = false
+		}
+		if n.Stats.HashmapStats.ShuffleMethod != plan.ShuffleMethod_Reuse {
+			highestNDV := n.OnList[idx].Ndv
+			if highestNDV < ShuffleThreshHoldOfNDV {
+				n.Stats.HashmapStats.Shuffle = false
+			}
+		}
 	}
 }