Skip to content

Commit 02a015f

Browse files
zhongjuzheIncarnation-p-lee
zhongjuzhe
authored and
Incarnation-p-lee
committed
VECT: Support CALL vectorization for COND_LEN_*
Hi, Richard and Richi. Base on the suggestions from Richard: https://gcc.gnu.org/pipermail/gcc-patches/2023-July/625396.html This patch choose (1) approach that Richard provided, meaning: RVV implements cond_* optabs as expanders. RVV therefore supports both IFN_COND_ADD and IFN_COND_LEN_ADD. No dummy length arguments are needed at the gimple level. Such approach can make codes much cleaner and reasonable. Consider this following case: void foo (float * __restrict a, float * __restrict b, int * __restrict cond, int n) { for (int i = 0; i < n; i++) if (cond[i]) a[i] = b[i] + a[i]; } Output of RISC-V (32-bits) gcc (trunk) (Compiler gcc-mirror#3) <source>:5:21: missed: couldn't vectorize loop <source>:5:21: missed: not vectorized: control flow in loop. ARM SVE: ... mask__27.10_51 = vect__4.9_49 != { 0, ... }; ... vec_mask_and_55 = loop_mask_49 & mask__27.10_51; ... vect__9.17_62 = .COND_ADD (vec_mask_and_55, vect__6.13_56, vect__8.16_60, vect__6.13_56); For RVV, we want IR as follows: ... _68 = .SELECT_VL (ivtmp_66, POLY_INT_CST [4, 4]); ... mask__27.10_51 = vect__4.9_49 != { 0, ... }; ... vect__9.17_60 = .COND_LEN_ADD (mask__27.10_51, vect__6.13_55, vect__8.16_59, vect__6.13_55, _68, 0); ... Both len and mask of COND_LEN_ADD are real not dummy. This patch has been fully tested in RISC-V port with supporting both COND_* and COND_LEN_*. And also, Bootstrap and Regression on X86 passed. OK for trunk? gcc/ChangeLog: * internal-fn.cc (get_len_internal_fn): New function. (DEF_INTERNAL_COND_FN): Ditto. (DEF_INTERNAL_SIGNED_COND_FN): Ditto. * internal-fn.h (get_len_internal_fn): Ditto. * tree-vect-stmts.cc (vectorizable_call): Add CALL auto-vectorization.
1 parent 31ec413 commit 02a015f

File tree

3 files changed

+78
-5
lines changed

3 files changed

+78
-5
lines changed

‎gcc/internal-fn.cc‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4443,6 +4443,30 @@ get_conditional_internal_fn (internal_fn fn)
44434443
}
44444444
}
44454445

4446+
/* If there exists an internal function like IFN that operates on vectors,
4447+
but with additional length and bias parameters, return the internal_fn
4448+
for that function, otherwise return IFN_LAST. */
4449+
internal_fn
4450+
get_len_internal_fn (internal_fn fn)
4451+
{
4452+
switch (fn)
4453+
{
4454+
#undef DEF_INTERNAL_COND_FN
4455+
#undef DEF_INTERNAL_SIGNED_COND_FN
4456+
#define DEF_INTERNAL_COND_FN(NAME, ...) \
4457+
case IFN_COND_##NAME: \
4458+
return IFN_COND_LEN_##NAME;
4459+
#define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4460+
case IFN_COND_##NAME: \
4461+
return IFN_COND_LEN_##NAME;
4462+
#include "internal-fn.def"
4463+
#undef DEF_INTERNAL_COND_FN
4464+
#undef DEF_INTERNAL_SIGNED_COND_FN
4465+
default:
4466+
return IFN_LAST;
4467+
}
4468+
}
4469+
44464470
/* If IFN implements the conditional form of an unconditional internal
44474471
function, return that unconditional function, otherwise return IFN_LAST. */
44484472

‎gcc/internal-fn.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ extern bool set_edom_supported_p (void);
224224

225225
extern internal_fn get_conditional_internal_fn (tree_code);
226226
extern internal_fn get_conditional_internal_fn (internal_fn);
227+
extern internal_fn get_len_internal_fn (internal_fn);
227228
extern internal_fn get_conditional_len_internal_fn (tree_code);
228229
extern tree_code conditional_internal_fn_code (internal_fn);
229230
extern internal_fn get_unconditional_internal_fn (internal_fn);

‎gcc/tree-vect-stmts.cc‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3540,7 +3540,10 @@ vectorizable_call (vec_info *vinfo,
35403540

35413541
int reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
35423542
internal_fn cond_fn = get_conditional_internal_fn (ifn);
3543+
internal_fn cond_len_fn = get_len_internal_fn (ifn);
3544+
int len_opno = internal_fn_len_index (cond_len_fn);
35433545
vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
3546+
vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
35443547
if (!vec_stmt) /* transformation not required. */
35453548
{
35463549
if (slp_node)
@@ -3569,6 +3572,9 @@ vectorizable_call (vec_info *vinfo,
35693572
if (reduc_idx >= 0
35703573
&& (cond_fn == IFN_LAST
35713574
|| !direct_internal_fn_supported_p (cond_fn, vectype_out,
3575+
OPTIMIZE_FOR_SPEED))
3576+
&& (cond_len_fn == IFN_LAST
3577+
|| !direct_internal_fn_supported_p (cond_len_fn, vectype_out,
35723578
OPTIMIZE_FOR_SPEED)))
35733579
{
35743580
if (dump_enabled_p ())
@@ -3586,8 +3592,14 @@ vectorizable_call (vec_info *vinfo,
35863592
tree scalar_mask = NULL_TREE;
35873593
if (mask_opno >= 0)
35883594
scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
3589-
vect_record_loop_mask (loop_vinfo, masks, nvectors,
3590-
vectype_out, scalar_mask);
3595+
if (cond_len_fn != IFN_LAST
3596+
&& direct_internal_fn_supported_p (cond_len_fn, vectype_out,
3597+
OPTIMIZE_FOR_SPEED))
3598+
vect_record_loop_len (loop_vinfo, lens, nvectors, vectype_out,
3599+
1);
3600+
else
3601+
vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype_out,
3602+
scalar_mask);
35913603
}
35923604
}
35933605
return true;
@@ -3603,8 +3615,20 @@ vectorizable_call (vec_info *vinfo,
36033615
vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
36043616

36053617
bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
3618+
bool len_loop_p = loop_vinfo && LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo);
36063619
unsigned int vect_nargs = nargs;
3607-
if (masked_loop_p && reduc_idx >= 0)
3620+
if (len_loop_p)
3621+
{
3622+
if (len_opno >= 0)
3623+
{
3624+
ifn = cond_len_fn;
3625+
/* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3626+
vect_nargs += 2;
3627+
}
3628+
else if (reduc_idx >= 0)
3629+
gcc_unreachable ();
3630+
}
3631+
else if (masked_loop_p && reduc_idx >= 0)
36083632
{
36093633
ifn = cond_fn;
36103634
vect_nargs += 2;
@@ -3671,7 +3695,21 @@ vectorizable_call (vec_info *vinfo,
36713695
}
36723696
else
36733697
{
3674-
if (mask_opno >= 0 && masked_loop_p)
3698+
if (len_opno >= 0 && len_loop_p)
3699+
{
3700+
unsigned int vec_num = vec_oprnds0.length ();
3701+
/* Always true for SLP. */
3702+
gcc_assert (ncopies == 1);
3703+
tree len
3704+
= vect_get_loop_len (loop_vinfo, gsi, lens, vec_num,
3705+
vectype_out, i, 1);
3706+
signed char biasval
3707+
= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
3708+
tree bias = build_int_cst (intQI_type_node, biasval);
3709+
vargs[len_opno] = len;
3710+
vargs[len_opno + 1] = bias;
3711+
}
3712+
else if (mask_opno >= 0 && masked_loop_p)
36753713
{
36763714
unsigned int vec_num = vec_oprnds0.length ();
36773715
/* Always true for SLP. */
@@ -3719,7 +3757,17 @@ vectorizable_call (vec_info *vinfo,
37193757
if (masked_loop_p && reduc_idx >= 0)
37203758
vargs[varg++] = vargs[reduc_idx + 1];
37213759

3722-
if (mask_opno >= 0 && masked_loop_p)
3760+
if (len_opno >= 0 && len_loop_p)
3761+
{
3762+
tree len = vect_get_loop_len (loop_vinfo, gsi, lens, ncopies,
3763+
vectype_out, j, 1);
3764+
signed char biasval
3765+
= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
3766+
tree bias = build_int_cst (intQI_type_node, biasval);
3767+
vargs[len_opno] = len;
3768+
vargs[len_opno + 1] = bias;
3769+
}
3770+
else if (mask_opno >= 0 && masked_loop_p)
37233771
{
37243772
tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies,
37253773
vectype_out, j);

0 commit comments

Comments
 (0)