Skip to content
This repository was archived by the owner on Jan 31, 2024. It is now read-only.

Commit cd5ae14

Browse files
xionghulxionghul
committed
Fix loop split incorrect count and probability
In tree-ssa-loop-split.c, split_loop and split_loop_on_cond does two kind of split. split_loop only works for single loop and insert edge at exit when split, while split_loop_on_cond is not limited to single loop and insert edge at latch when split. Both split behavior should consider loop count and probability update. For split_loop, loop split condition is moved in front of loop1 and loop2; But split_loop_on_cond moves the condition between loop1 and loop2, this patch does: 1) profile count proportion for both original loop and copied loop without dropping down the true branch's count; 2) probability update in the two loops and between the two loops. Regression tested pass. Changes diff for split_loop and split_loop_on_cond cases: 1) diff base/loop-split.c.151t.lsplit patched/loop-split.c.152t.lsplit ... <bb 2> [local count: 118111600]: if (beg_5(D) < end_8(D)) goto <bb 14>; [89.00%] else goto <bb 6>; [11.00%] <bb 14> [local count: 105119324]: if (beg2_6(D) < c_9(D)) - goto <bb 15>; [100.00%] + goto <bb 15>; [33.00%] else - goto <bb 16>; [100.00%] + goto <bb 16>; [67.00%] - <bb 15> [local count: 105119324]: + <bb 15> [local count: 34689377]: _25 = beg_5(D) + 1; _26 = end_8(D) - beg_5(D); _27 = beg2_6(D) + _26; _28 = MIN_EXPR <c_9(D), _27>; - <bb 3> [local count: 955630225]: + <bb 3> [local count: 315357973]: # i_16 = PHI <i_11(8), beg_5(D)(15)> # j_17 = PHI <j_12(8), beg2_6(D)(15)> printf ("a: %d %d\n", i_16, j_17); i_11 = i_16 + 1; j_12 = j_17 + 1; if (j_12 < _28) - goto <bb 8>; [89.00%] + goto <bb 8>; [29.37%] else - goto <bb 17>; [11.00%] + goto <bb 17>; [70.63%] - <bb 8> [local count: 850510901]: + <bb 8> [local count: 280668596]: goto <bb 3>; [100.00%] - <bb 16> [local count: 105119324]: + <bb 16> [local count: 70429947]: # i_22 = PHI <beg_5(D)(14), i_29(17)> # j_23 = PHI <beg2_6(D)(14), j_30(17)> <bb 10> [local count: 955630225]: # i_2 = PHI <i_22(16), i_20(13)> # j_1 = PHI <j_23(16), j_21(13)> i_20 = i_2 + 1; j_21 = j_1 + 1; if (end_8(D) > i_20) - goto <bb 13>; [89.00%] + goto <bb 13>; [59.63%] else - goto <bb 9>; [11.00%] + goto <bb 9>; [40.37%] - <bb 13> [local count: 850510901]: + <bb 13> [local count: 569842305]: goto <bb 10>; [100.00%] <bb 17> [local count: 105119324]: # i_29 = PHI <i_11(3)> # j_30 = PHI <j_12(3)> if (end_8(D) > i_29) goto <bb 16>; [80.00%] else goto <bb 9>; [20.00%] <bb 9> [local count: 105119324]: <bb 6> [local count: 118111600]: return 0; } <bb 2> [local count: 118111600]: - if (beg_5(D) < end_8(D)) + _1 = end_6(D) - beg_7(D); + j_9 = _1 + beg2_8(D); + if (end_6(D) > beg_7(D)) goto <bb 14>; [89.00%] else goto <bb 6>; [11.00%] <bb 14> [local count: 105119324]: - if (beg2_6(D) < c_9(D)) - goto <bb 15>; [100.00%] + if (j_9 >= c_11(D)) + goto <bb 15>; [33.00%] else - goto <bb 16>; [100.00%] + goto <bb 16>; [67.00%] - <bb 15> [local count: 105119324]: - _25 = beg_5(D) + 1; - _26 = end_8(D) - beg_5(D); - _27 = beg2_6(D) + _26; - _28 = MIN_EXPR <c_9(D), _27>; - - <bb 3> [local count: 955630225]: - # i_16 = PHI <i_11(8), beg_5(D)(15)> - # j_17 = PHI <j_12(8), beg2_6(D)(15)> - printf ("a: %d %d\n", i_16, j_17); - i_11 = i_16 + 1; - j_12 = j_17 + 1; - if (j_12 < _28) - goto <bb 8>; [89.00%] + <bb 15> [local count: 34689377]: + _27 = end_6(D) + -1; + _28 = beg_7(D) - end_6(D); + _29 = j_9 + _28; + _30 = _29 + 1; + _31 = MAX_EXPR <c_11(D), _30>; + + <bb 3> [local count: 315357973]: + # i_18 = PHI <i_13(8), end_6(D)(15)> + # j_19 = PHI <j_14(8), j_9(15)> + printf ("a: %d %d\n", i_18, j_19); + i_13 = i_18 + -1; + j_14 = j_19 + -1; + if (j_14 >= _31) + goto <bb 8>; [29.37%] else - goto <bb 17>; [11.00%] + goto <bb 17>; [70.63%] - <bb 8> [local count: 850510901]: + <bb 8> [local count: 280668596]: goto <bb 3>; [100.00%] - <bb 16> [local count: 105119324]: - # i_22 = PHI <beg_5(D)(14), i_29(17)> - # j_23 = PHI <beg2_6(D)(14), j_30(17)> + <bb 16> [local count: 70429947]: + # i_24 = PHI <end_6(D)(14), i_32(17)> + # j_25 = PHI <j_9(14), j_33(17)> <bb 10> [local count: 955630225]: - # i_2 = PHI <i_22(16), i_20(13)> - # j_1 = PHI <j_23(16), j_21(13)> - i_20 = i_2 + 1; - j_21 = j_1 + 1; - if (end_8(D) > i_20) + # i_3 = PHI <i_24(16), i_22(13)> + # j_2 = PHI <j_25(16), j_23(13)> + i_22 = i_3 + -1; + j_23 = j_2 + -1; + if (beg_7(D) < i_22) goto <bb 13>; [89.00%] else goto <bb 9>; [11.00%] - <bb 13> [local count: 850510901]: + <bb 13> [local count: 569842305]: goto <bb 10>; [100.00%] <bb 17> [local count: 105119324]: - # i_29 = PHI <i_11(3)> - # j_30 = PHI <j_12(3)> - if (end_8(D) > i_29) + # i_32 = PHI <i_13(3)> + # j_33 = PHI <j_14(3)> + if (beg_7(D) < i_32) goto <bb 16>; [80.00%] else goto <bb 9>; [20.00%] <bb 9> [local count: 105119324]: <bb 6> [local count: 118111600]: return 0; } 2) diff base/loop-cond-split-1.c.151t.lsplit patched/loop-cond-split-1.c.151t.lsplit: ... <bb 2> [local count: 118111600]: if (n_7(D) > 0) goto <bb 4>; [89.00%] else goto <bb 3>; [11.00%] <bb 3> [local count: 118111600]: return; <bb 4> [local count: 105119324]: pretmp_3 = ga; - <bb 5> [local count: 955630225]: + <bb 5> [local count: 315357973]: # i_13 = PHI <i_10(20), 0(4)> # prephitmp_12 = PHI <prephitmp_5(20), pretmp_3(4)> if (prephitmp_12 != 0) goto <bb 6>; [33.00%] else goto <bb 7>; [67.00%] <bb 6> [local count: 315357972]: _2 = do_something (); ga = _2; - <bb 7> [local count: 955630225]: + <bb 7> [local count: 315357973]: # prephitmp_5 = PHI <prephitmp_12(5), _2(6)> i_10 = inc (i_13); if (n_7(D) > i_10) goto <bb 21>; [89.00%] else goto <bb 11>; [11.00%] <bb 11> [local count: 105119324]: goto <bb 3>; [100.00%] - <bb 21> [local count: 850510901]: + <bb 21> [local count: 280668596]: if (prephitmp_12 != 0) - goto <bb 20>; [100.00%] + goto <bb 20>; [33.00%] else - goto <bb 19>; [INV] + goto <bb 19>; [67.00%] - <bb 20> [local count: 850510901]: + <bb 20> [local count: 280668596]: goto <bb 5>; [100.00%] - <bb 19> [count: 0]: + <bb 19> [local count: 70429947]: # i_23 = PHI <i_10(21)> # prephitmp_25 = PHI <prephitmp_5(21)> - <bb 12> [local count: 955630225]: + <bb 12> [local count: 640272252]: # i_15 = PHI <i_23(19), i_22(16)> # prephitmp_16 = PHI <prephitmp_25(19), prephitmp_16(16)> i_22 = inc (i_15); if (n_7(D) > i_22) goto <bb 16>; [89.00%] else goto <bb 11>; [11.00%] - <bb 16> [local count: 850510901]: + <bb 16> [local count: 569842305]: goto <bb 12>; [100.00%] } gcc/ChangeLog: 2021-12-21 Xionghu Luo <[email protected]> * tree-ssa-loop-split.c (split_loop): Fix incorrect profile_count and probability. (do_split_loop_on_cond): Likewise.
1 parent 46bfe1b commit cd5ae14

File tree

1 file changed

+77
-8
lines changed

1 file changed

+77
-8
lines changed

‎gcc/tree-ssa-loop-split.c

Lines changed: 77 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -575,19 +575,22 @@ split_loop (class loop *loop1)
575575
stmts2);
576576
tree cond = build2 (guard_code, boolean_type_node, guard_init, border);
577577
if (!initial_true)
578-
cond = fold_build1 (TRUTH_NOT_EXPR, boolean_type_node, cond);
578+
cond = fold_build1 (TRUTH_NOT_EXPR, boolean_type_node, cond);
579+
580+
edge true_edge, false_edge;
581+
extract_true_false_edges_from_block (bbs[i], &true_edge, &false_edge);
579582

580583
/* Now version the loop, placing loop2 after loop1 connecting
581584
them, and fix up SSA form for that. */
582585
initialize_original_copy_tables ();
583586
basic_block cond_bb;
584587

585588
class loop *loop2 = loop_version (loop1, cond, &cond_bb,
586-
profile_probability::always (),
587-
profile_probability::always (),
588-
profile_probability::always (),
589-
profile_probability::always (),
590-
true);
589+
true_edge->probability,
590+
true_edge->probability.invert (),
591+
profile_probability::always (),
592+
profile_probability::always (),
593+
true);
591594
gcc_assert (loop2);
592595

593596
edge new_e = connect_loops (loop1, loop2);
@@ -607,6 +610,38 @@ split_loop (class loop *loop1)
607610
tree guard_next = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop1));
608611
patch_loop_exit (loop1, guard_stmt, guard_next, newend, initial_true);
609612

613+
update_ssa (TODO_update_ssa);
614+
615+
/* Proportion first loop's bb counts except those dominated by true
616+
branch to avoid drop 1s down. */
617+
basic_block *bbs1, *bbs2;
618+
bbs1 = get_loop_body (loop1);
619+
unsigned j;
620+
for (j = 0; j < loop1->num_nodes; j++)
621+
if (bbs1[j] == loop1->latch
622+
|| !dominated_by_p (CDI_DOMINATORS, bbs1[j], true_edge->dest))
623+
bbs1[j]->count
624+
= bbs1[j]->count.apply_probability (true_edge->probability);
625+
free (bbs1);
626+
627+
/* Fix first loop's exit probability after scaling. */
628+
edge exit_to_latch1 = single_pred_edge (loop1->latch);
629+
exit_to_latch1->probability = exit_to_latch1->probability.apply_scale (
630+
true_edge->probability.to_reg_br_prob_base (), REG_BR_PROB_BASE);
631+
single_exit (loop1)->probability
632+
= exit_to_latch1->probability.invert ();
633+
634+
/* Proportion second loop's bb counts except those dominated by false
635+
branch to avoid drop 1s down. */
636+
basic_block bbi_copy = get_bb_copy (false_edge->dest);
637+
bbs2 = get_loop_body (loop2);
638+
for (j = 0; j < loop2->num_nodes; j++)
639+
if (bbs2[j] == loop2->latch
640+
|| !dominated_by_p (CDI_DOMINATORS, bbs2[j], bbi_copy))
641+
bbs2[j]->count = bbs2[j]->count.apply_probability (
642+
true_edge->probability.invert ());
643+
free (bbs2);
644+
610645
/* Finally patch out the two copies of the condition to be always
611646
true/false (or opposite). */
612647
gcond *force_true = as_a<gcond *> (last_stmt (bbs[i]));
@@ -1486,8 +1521,8 @@ do_split_loop_on_cond (struct loop *loop1, edge invar_branch)
14861521
initialize_original_copy_tables ();
14871522

14881523
struct loop *loop2 = loop_version (loop1, boolean_true_node, NULL,
1489-
profile_probability::always (),
1490-
profile_probability::never (),
1524+
invar_branch->probability.invert (),
1525+
invar_branch->probability,
14911526
profile_probability::always (),
14921527
profile_probability::always (),
14931528
true);
@@ -1535,6 +1570,40 @@ do_split_loop_on_cond (struct loop *loop1, edge invar_branch)
15351570
between loop1 and loop2. */
15361571
connect_loop_phis (loop1, loop2, to_loop2);
15371572

1573+
update_ssa (TODO_update_ssa);
1574+
1575+
edge true_edge, false_edge, skip_edge1, skip_edge2;
1576+
extract_true_false_edges_from_block (cond_bb, &true_edge, &false_edge);
1577+
1578+
/* Proportion first loop's bb counts except those dominated by true
1579+
branch to avoid drop 1s down. */
1580+
skip_edge1 = true_invar ? false_edge : true_edge;
1581+
skip_edge2 = true_invar ? true_edge : false_edge;
1582+
basic_block *bbs1, *bbs2;
1583+
bbs1 = get_loop_body (loop1);
1584+
unsigned j;
1585+
for (j = 0; j < loop1->num_nodes; j++)
1586+
if (bbs1[j] == loop1->latch
1587+
|| !dominated_by_p (CDI_DOMINATORS, bbs1[j], skip_edge1->dest))
1588+
bbs1[j]->count
1589+
= bbs1[j]->count.apply_probability (skip_edge1->probability);
1590+
free (bbs1);
1591+
1592+
/* Fix first loop's exit probability after scaling. */
1593+
to_loop1->probability = invar_branch->probability.invert ();
1594+
to_loop2->probability = invar_branch->probability;
1595+
1596+
/* Proportion second loop's bb counts except those dominated by false
1597+
branch to avoid drop 1s down. */
1598+
basic_block bbi_copy = get_bb_copy (skip_edge2->dest);
1599+
bbs2 = get_loop_body (loop2);
1600+
for (j = 0; j < loop2->num_nodes; j++)
1601+
if (bbs2[j] == loop2->latch
1602+
|| !dominated_by_p (CDI_DOMINATORS, bbs2[j], bbi_copy))
1603+
bbs2[j]->count
1604+
= bbs2[j]->count.apply_probability (skip_edge2->probability);
1605+
free (bbs2);
1606+
15381607
free_original_copy_tables ();
15391608

15401609
return true;

0 commit comments

Comments
 (0)