https://gcc.gnu.org/g:58f804554996fe057e5ca4fc814449640a6e816c

commit r17-6012-g58f804554996fe057e5ca4fc814449640a6e816c
Author: Jakub Jelinek <[email protected]>
Date:   Sat Nov 7 09:42:42 2026 +0100

    c, c++, middle-end, i386: Introduce __builtin_clmulg builtin [PR128111]
    
    The https://wg21.link/P3642R7 paper proposes to add std::clmul and
    std::widening_clmul function templates.  Because many targets have
    instructions to implement some cases of the carry-less product in
    hardware and there are various optimizations only the compiler can do
    and lots of different ways to open code it in the source, I think we
    need a builtin and pattern matching is of limited use.
    Now, clang added __builtin_elementwise_clmul builtin for this, but
    1) we don't have any __builtin_elementwise_* builtins so far and no
       agreement on those
    2) the builtin only provides type-generic scalar (and vector)
       T = clmul (T, T) operation, but many targets actually have
       instructions to perform widening clmul (e.g. x86 64x64->128,
       etc.), and because std::widening_clmul needs the latter, I think
       it is useful to have a builtin that can handle both
    So, the following patch introduces a __builtin_clmulg builtin instead,
    if we ever decide to add support of __builtin_elementwise_* builtins
    we can certainly add easily support for the clang one as well.
    Now, because it is a type-generic builtin, for the widening case
    there is a problem that we sometimes don't actually have wider types.
    For 32-bit targets without _BitInt there is no type wider than
    unsigned long long, for 64-bit targets without _BitInt there is no
    type wider than unsigned __int128, for _BitInt target there is no
    type wider than unsigned _BitInt(__BITINT_MAXWIDTH__).
    So, having a builtin which for 2 arguments of the same type returns
    some other type with precision twice as big would only work sometimes.
    Instead of this, the builtin has 2 or 3 arguments, as if
      template <__unsigned_integer T>
      T __builtin_clmulg (T, T);
    and
      template <__unsigned_integer T>
      T __builtin_clmulg (T, T, T *);
    where the former computes the low __builtin_popcountg ((T) ~(T) 0)
    bits of the result, while the latter returns the same plus stores
    the remaining higher bits of the result to what the third argument
    points to (similarly to e.g. __builtin_mul_overflow, except that
    __builtin_mul_overflow returns just bool, and accepts arbitrary
    set of types, while this one requires all of them to be the same).
    
    Under the hood, this is implemented using two new internal functions,
    IFN_CLMUL which returns just the low bits of the result, and
    IFN_CLMUL_WIDEN which is similar to IFN_MUL_OVERFLOW etc. in that
    it returns COMPLEX_TYPE result with real part being the low half
    and imag part being the high half.
    As for the implementation, various targets have hw instructions
    for some cases of this.
    x86 has pclmulqdq (64x64->128) and vcplmulqdq (vector version of that),
    arm/aarch64 has some vmul*/pmul* which can do all kinds of stuff
    depending on ISA (8x8->8, 8x8->16, 32x32->64, 64x64->128,
    powerpc has vpsum* (8x8->16, 16x16->32, 32x32->64, 64x64->128), including
    vector in some cases,
    riscv has clmul and clmulh (both together can do 32x32->64 resp.
    64x64->128 depending on mode, only clmul can do 32x32 resp. 64x64)
    and vclmul* to handle vectors,
    sparc has xmulx and xmluxhi (64x64->128 together, 64x64 the first),
    s390x has vgfm, vgfma (8x8->16, 16x16->32, 32x32->64, 64x64->128).
    This patch implements just the x86 part, I'll defer to target maintainers
    to do the other arches.
    
    For software implementation fallback, there are various algorithms.
    One is the naive O(n^2),
    T clmul_naive (T x, T y) {
      T r = 0;
      for (int i = 0; i < __builtin_popcountg ((T) ~(T) 0); ++i)
        if ((unsigned) (y >> i) & 1)
          r ^= x << i;
      return r;
    }
    T2 below is unsigned type with twice the precision of T:
    T2 clmul_naive_widen (T x, T y) {
      T2 r = 0;
      for (int i = 0; i < __builtin_popcountg ((T) ~(T) 0); ++i)
        if ((unsigned) (y >> i) & 1)
          r ^= ((T2) x) << i;
      return r;
    }
    another is using Kronecker substitution, depending on maximum number of
    active bits in the operands and whether it is the non-widening or widening
    case one chooses S in [1, 13] such as for N (precision or max number of
    active bits) N <= S * ((1 << S) - 1) (for the widening case) or
    N <= S * (1 << S) (for the non-widening one; this is all about how
    many additions during the multiplication still don't cause carry over
    the given spacing, e.g. for S = 4 0x1111111111111111ULL * 0x11111111ULL
    is 0x8888888887654321ULL, so there is no carry over the 4 bits but one
    already over the 3 bits, 0x1111111111111111ULL * 0x11111111111111ULL
    0xeeedcba987654321ULL so getting closer, and
    0x1111111111111111ULL * 0x1111111111111111ULL being
    0xfedcba987654321ULL, where there is still no carry over from one group
    to another within the least significant 64 bits, but there is one above
    that already; unsigned _BitInt(60) can use safely S = 4 even in the
    widening case, but unsigned _BitInt(64) only in the non-widening one
    and for widening it needs to use S = 5 (where 5 * 5 multiplications
    at least if implemented in hw is still better than 4 * 4 * 2),
    T clmul_kronecker (T x, T y, int S) {
      T mask = std::bit_repeat ((T) 1, S);
      T xa[S], ya[S], r = 0;
      for (int i = 0; i < S; ++i) {
        xa[i] = x & (mask << i); ya[i] = y & (mask << i);
      }
      for (int i = 0; i < S; ++i) {
        T t = 0;
        for (int j = 0; j < S; ++j)
          t ^= xa[j] * ya[(i + S - j) % S];
        t &= (mask << i);
        r |= t;
      }
      return r;
    }
    T2 clmul_kronecker_widen (T x, T y, int S) {
      T maskn = std::bit_repeat ((T) 1, S);
      T2 mask = std::bit_repeat ((T2) 1, S);
      T2 xa[S], ya[S], r = 0;
      for (int i = 0; i < S; ++i) {
        xa[i] = x & (maskn << i); ya[i] = y & (maskn << i);
      }
      for (int i = 0; i < S; ++i) {
        T2 t = 0;
        for (int j = 0; j < S; ++j)
          t ^= xa[j] * ya[(i + S - j) % S];
        t &= (mask << i);
        r |= t;
      }
      return r;
    }
    another one is recursive, e.g. adjusted Karatsuba multiplication,
    TH is a type with half the precision of T:
    T2 clmul_karatsuba_widen (T x, T y) {
      int N = __builtin_popcountg ((T) ~(T) 0);
      TH x0 = x, x1 = x >> (N / 2);
      TH y0 = y, y1 = y >> (N / 2);
      T l = clmul_widen (x0, y0);
      T m = clmul_widen (x0 ^ x1, y0 ^ y1);
      T h = clmul_widen (x1, y1);
      m ^= l ^ h;
      return (((T2) h) << N) | (((T2) m) << (N / 2)) | l;
    }
    and if we have a non-widening clmul, we can do:
    T2 clmul_bitreverse_widen (T x, T y) {
      int N = __builtin_popcountg ((T) ~(T) 0);
      T l = clmul (x, y);
      T h = __builtin_bitreverseg (clmul (__builtin_bitreverseg (x),
                                          __builtin_bitreverseg (y))) >> 1;
      return (((T2) h) << N) | l;
    }
    The following patch uses all of these algorithms depending on circumstances.
    For wi::clmul it uses the naive one, for expansion obviously attempts to
    use optabs first and then chooses among the rest.
    
    For _BitInt lowering, it uses an optimized version of the naive one,
    while theoretically a recursive algorithm could be best, we don't even
    handle normal multiplication that way in libgcc right now.  And from
    my benchmarking on x86_64 (see the PR for details), the optimized version
    of the naive algorithm consistently beat the Kronecker one when there is
    no corresponding hw multiply (while the non-optimized naive was always
    slower than the Kronecker substitution).  And the naive one is smallest
    in the emitted code.
    
    The patch doesn't add any libgcc libcalls right now, I'm not sure if
    we want to do that or not, the problem is that having too many different
    cases in libgcc would be code bloat for programs which don't use this,
    and choosing just one or two could be too slow in case we only need fewer
    bits.  In any case, libgcc libcalls can be added later on, including in
    future GCC relelases.
    
    I admit I haven't figured out how the IFNs work with ranges, the only
    thing I know is that if the x and y operands have some trailing zeros,
    ctz (clmul_widen (x, y)) == ctz (x) + ctz (y), and
    clz (clmul_widen (x, y)) == clz (x) + clz (y) + 1.  Not implemented
    for now, neither in CCP nor in the ranger.
    
    Possible future optimization might be to lower the two IFNs to their
    fallback if optabs are missing, and either let the SLP vectorizer
    vectorize it (it doesn't currently) or vectorize it manually,
    e.g. the Kronecker substitution algorithm could very well perform some
    vector permutations and perform e.g. 4 multiplications at a time, etc.
    The paper also talks about the case that clmul (x, x) can be implemented
    more efficiently.  And perhaps we could eventually pattern match the
    clmul and reversed clmul of reversed args shifted down by 1 as widening
    clmul.
    
    The FE changes are mostly following what is done for __builtin_bswapg
    or __builtin_bitreverseg already, as similarly to those this is
    a type-generic builtin with the return type dependent on the argument
    type(s).
    
    2026-11-07  Jakub Jelinek  <[email protected]>
    
            PR middle-end/128111
            * doc/extend.texi (__builtin_clmulg): Document.
            * builtins.h (fold_build_builtin_clmulg): Declare.
            * fold-const-call.cc (fold_const_call_sss): Handle CFN_CLMUL.
            (fold_const_call): Handle CFN_CLMUL_WIDEN.
            * internal-fn.h (expand_CLMUL, expand_CLMUL_WIDEN): Declare.
            * fold-const.cc (tree_call_nonnegative_p): Handle CFN_CLMUL.
            * wide-int.cc (wi::bit_repeat, wi::clmul): New functions.
            * wide-int.h (wi::bit_repeat, wi::clmul): Declare.
            * builtins.cc (fold_build_builtin_clmulg): New function.
            * internal-fn.def (CLMUL, CLMUL_WIDEN): New internal functions.
            * internal-fn.cc (commutative_binary_fn_p): Handle IFN_CLMUL
            and IFN_CLMUL_WIDEN.
            (expand_clmul, expand_CLMUL, expand_CLMUL_WIDEN): New functions.
            * optabs.def (clmul_optab, clmul_widen_optab): New optabs.
            * gimple-lower-bitint.cc (struct bitint_large_huge): Declare
            lower_clmul method.
            (bitint_large_huge::lower_clmul): New method.
            (bitint_large_huge::lower_call): Handle IFN_CLMUL.
            (bitint_large_huge::lower_stmt): Likewise.
            (stmt_needs_operand_addr): Likewise.
            (build_bitint_stmt_ssa_conflicts): Likewise.
            (gimple_lower_bitint): Lower IFN_CLMUL_WIDEN to 2 IFN_CLMUL
            calls, 3 IFN_BITREVERSE calls, one shift and one COMPLEX_EXPR.
            Handle IFN_CLMUL.
            * match.pd (.CLMUL (X, Y) for [0, 1] Y -> X * Y,
            .CLMUL_WIDEN (X, Y) -> { .CLMUL (X, 0), 0 } if upper bits are
            known to be zeros): New simplifications.
            * genmatch.cc (genmatch_diag_selftests): CFN_CLMUL and 
CFN_CLMUL_WIDEN
            are commutative.
            (commutative_op): Handle CFN_CLMUL and CFN_CLMUL_WIDEN.
            * config/i386/sse.md (clmuldi3, clmul_widendi3): New expanders.
    gcc/c-family/
            * c-common.h (enum rid): Add RID_BUILTIN_CLMULG enumerator.
            * c-common.cc (c_common_reswords): Add __builtin_clmulg entry.
    gcc/c/
            * c-parser.cc (c_parser_postfix_expression): Handle
            RID_BUILTIN_CLMULG.  Use argument locations when reporting
            problems with a single argument.
            * c-decl.cc (names_builtin_p): Handle RID_BUILTIN_CLMULG.
    gcc/cp/
            * parser.cc (cp_parser_postfix_expression): Handle 
RID_BUILTIN_CLMULG.
            * typeck.cc (build_x_clmulg): New function.
            * cp-tree.h (build_x_clmulg): Declare.
            * pt.cc (tsubst_expr): Handle IFN_CLMUL.
            * mangle.cc (write_expression): Likewise.
            * cp-objcp-common.cc (names_builtin_p): Handle RID_BUILTIN_CLMULG.
            * cp-gimplify.cc (cp_gimplify_expr): Handle IFN_CLMUL.
            * constexpr.cc (cxx_eval_internal_function): Likewise.
            (potential_constant_expression_1): Likewise.
    gcc/testsuite/
            * c-c++-common/builtin-clmulg-1.c: New test.
            * c-c++-common/builtin-clmulg-2.c: New test.
            * g++.dg/ext/builtin-clmulg-1.C: New test.
            * gcc.target/i386/pclmul-builtin-clmulg-1.c: New test.
    
    Reviewed-by: Richard Biener <[email protected]>
    Reviewed-by: Jason Merrill <[email protected]>
    Reviewed-by: Marek Polacek <[email protected]>

Diff:
---
 gcc/builtins.cc                                    |  46 ++
 gcc/builtins.h                                     |   1 +
 gcc/c-family/c-common.cc                           |   1 +
 gcc/c-family/c-common.h                            |   2 +-
 gcc/c/c-decl.cc                                    |   1 +
 gcc/c/c-parser.cc                                  | 120 ++++-
 gcc/config/i386/sse.md                             |  43 ++
 gcc/cp/constexpr.cc                                |  44 ++
 gcc/cp/cp-gimplify.cc                              |  18 +
 gcc/cp/cp-objcp-common.cc                          |   1 +
 gcc/cp/cp-tree.h                                   |   3 +
 gcc/cp/mangle.cc                                   |  11 +-
 gcc/cp/parser.cc                                   |   6 +
 gcc/cp/pt.cc                                       |   8 +
 gcc/cp/typeck.cc                                   | 101 ++++
 gcc/doc/extend.texi                                |  23 +
 gcc/fold-const-call.cc                             |  18 +
 gcc/fold-const.cc                                  |   1 +
 gcc/genmatch.cc                                    |   2 +
 gcc/gimple-lower-bitint.cc                         | 291 +++++++++++-
 gcc/internal-fn.cc                                 | 329 +++++++++++++
 gcc/internal-fn.def                                |   2 +
 gcc/internal-fn.h                                  |   2 +
 gcc/match.pd                                       |  19 +
 gcc/optabs.def                                     |   2 +
 gcc/testsuite/c-c++-common/builtin-clmulg-1.c      | 524 +++++++++++++++++++++
 gcc/testsuite/c-c++-common/builtin-clmulg-2.c      |  55 +++
 gcc/testsuite/g++.dg/ext/builtin-clmulg-1.C        |  24 +
 .../gcc.target/i386/pclmul-builtin-clmulg-1.c      |  19 +
 gcc/wide-int.cc                                    |  66 +++
 gcc/wide-int.h                                     |   3 +
 31 files changed, 1767 insertions(+), 19 deletions(-)

diff --git a/gcc/builtins.cc b/gcc/builtins.cc
index 220010902cdd..b640a8776495 100644
--- a/gcc/builtins.cc
+++ b/gcc/builtins.cc
@@ -10732,6 +10732,52 @@ fold_builtin_addc_subc (location_t loc, enum 
built_in_function fcode,
   return build2_loc (loc, COMPOUND_EXPR, type, store, intres);
 }
 
+/* Fold __builtin_clmulg (arg1, arg2) or __builtin_clmulg (arg1, arg2, arg3)
+   into IFN_CLMUL or IFN_CLMUL_WIDEN internal call.  */
+
+tree
+fold_build_builtin_clmulg (location_t loc, tree arg1, tree arg2, tree arg3)
+{
+  tree type = TYPE_MAIN_VARIANT (TREE_TYPE (arg1));
+  if (TREE_CODE (arg1) == INTEGER_CST
+      && !TREE_OVERFLOW (arg1)
+      && TREE_CODE (arg2) == INTEGER_CST
+      && !TREE_OVERFLOW (arg2))
+    {
+      unsigned prec = TYPE_PRECISION (type);
+      wide_int res = wi::clmul (wi::to_wide (arg1), wi::to_wide (arg2),
+                               prec * (arg3 == NULL_TREE ? 1 : 2));
+      if (arg3 == NULL_TREE)
+       return wide_int_to_tree (type, res);
+      wide_int lo = wide_int::from (res, prec, UNSIGNED);
+      wide_int hi = wide_int::from (wi::rshift (res, prec, UNSIGNED),
+                                   prec, UNSIGNED);
+      tree mem_arg3 = build_fold_indirect_ref_loc (loc, arg3);
+      tree store
+       = fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg3,
+                          wide_int_to_tree (type, hi));
+      return build2_loc (loc, COMPOUND_EXPR, type, store,
+                        wide_int_to_tree (type, lo));
+    }
+  if (arg3 == NULL_TREE)
+    return build_call_expr_internal_loc (loc, IFN_CLMUL, type, 2, arg1, arg2);
+  tree ctype = build_complex_type (type);
+  tree call = build_call_expr_internal_loc (loc, IFN_CLMUL_WIDEN, ctype, 2,
+                                           arg1, arg2);
+  /* Force SAVE_EXPR even for calls which satisfy tree_invariant_p_1,
+     as while the call itself is const, the REALPART_EXPR store is
+     certainly not.  And in any case, we want just one call,
+     not multiple and trying to CSE them later.  */
+  TREE_SIDE_EFFECTS (call) = 1;
+  tree tgt = save_expr (call);
+  tree intres = build1_loc (loc, REALPART_EXPR, type, tgt);
+  tree ovfres = build1_loc (loc, IMAGPART_EXPR, type, tgt);
+  tree mem_arg3 = build_fold_indirect_ref_loc (loc, arg3);
+  tree store
+    = fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg3, ovfres);
+  return build2_loc (loc, COMPOUND_EXPR, type, store, intres);
+}
+
 /* Fold a call to __builtin_FILE to a constant string.  */
 
 static inline tree
diff --git a/gcc/builtins.h b/gcc/builtins.h
index 76d2bc73956c..be65b95ffbca 100644
--- a/gcc/builtins.h
+++ b/gcc/builtins.h
@@ -142,6 +142,7 @@ extern tree fold_builtin_expect (location_t, tree, tree, 
tree, tree);
 extern tree fold_builtin_constant_p (tree);
 extern tree fold_build_builtin_bswapg_bitreverseg (location_t, internal_fn,
                                                   tree);
+extern tree fold_build_builtin_clmulg (location_t, tree, tree, tree);
 extern bool avoid_folding_inline_builtin (tree);
 extern tree fold_call_expr (location_t, tree, bool);
 extern tree fold_builtin_call_array (location_t, tree, tree, int, tree *);
diff --git a/gcc/c-family/c-common.cc b/gcc/c-family/c-common.cc
index 6380bec4515d..76adc9a2b36a 100644
--- a/gcc/c-family/c-common.cc
+++ b/gcc/c-family/c-common.cc
@@ -439,6 +439,7 @@ const struct c_common_resword c_common_reswords[] =
   { "__builtin_call_with_static_chain",
     RID_BUILTIN_CALL_WITH_STATIC_CHAIN, D_CONLY },
   { "__builtin_choose_expr", RID_CHOOSE_EXPR, D_CONLY },
+  { "__builtin_clmulg", RID_BUILTIN_CLMULG, 0 },
   { "__builtin_complex", RID_BUILTIN_COMPLEX, D_CONLY },
   { "__builtin_convertvector", RID_BUILTIN_CONVERTVECTOR, 0 },
   { "__builtin_counted_by_ref", RID_BUILTIN_COUNTED_BY_REF, D_CONLY },
diff --git a/gcc/c-family/c-common.h b/gcc/c-family/c-common.h
index 197db508c163..9e00fe877ad7 100644
--- a/gcc/c-family/c-common.h
+++ b/gcc/c-family/c-common.h
@@ -112,7 +112,7 @@ enum rid
   RID_BUILTIN_SHUFFLEVECTOR,   RID_BUILTIN_CONVERTVECTOR,  RID_BUILTIN_TGMATH,
   RID_BUILTIN_HAS_ATTRIBUTE,   RID_BUILTIN_ASSOC_BARRIER,  RID_BUILTIN_STDC,
   RID_BUILTIN_COUNTED_BY_REF,  RID_BUILTIN_BSWAPG,
-  RID_BUILTIN_BITREVERSEG,
+  RID_BUILTIN_BITREVERSEG,     RID_BUILTIN_CLMULG,
   RID_DFLOAT32, RID_DFLOAT64, RID_DFLOAT128, RID_DFLOAT64X,
 
   /* TS 18661-3 keywords, in the same sequence as the TI_* values.  */
diff --git a/gcc/c/c-decl.cc b/gcc/c/c-decl.cc
index ada5a5043af4..6529969aace3 100644
--- a/gcc/c/c-decl.cc
+++ b/gcc/c/c-decl.cc
@@ -11936,6 +11936,7 @@ names_builtin_p (const char *name)
     case RID_BUILTIN_ASSOC_BARRIER:
     case RID_BUILTIN_BITREVERSEG:
     case RID_BUILTIN_BSWAPG:
+    case RID_BUILTIN_CLMULG:
     case RID_BUILTIN_CONVERTVECTOR:
     case RID_BUILTIN_HAS_ATTRIBUTE:
     case RID_BUILTIN_SHUFFLE:
diff --git a/gcc/c/c-parser.cc b/gcc/c/c-parser.cc
index 519959e7290c..028ff4943220 100644
--- a/gcc/c/c-parser.cc
+++ b/gcc/c/c-parser.cc
@@ -13556,15 +13556,29 @@ c_parser_postfix_expression (c_parser *parser)
          }
        case RID_BUILTIN_BSWAPG:
        case RID_BUILTIN_BITREVERSEG:
+       case RID_BUILTIN_CLMULG:
          {
            vec<c_expr_t, va_gc> *cexpr_list;
            location_t close_paren_loc;
-           internal_fn ifn
-             = (c_parser_peek_token (parser)->keyword == RID_BUILTIN_BSWAPG
-                ? IFN_BSWAP : IFN_BITREVERSE);
-           const char *name
-             = (ifn == IFN_BSWAP ? "__builtin_bswapg"
-                : "__builtin_bitreverseg");
+           internal_fn ifn;
+           const char *name;
+           switch (c_parser_peek_token (parser)->keyword)
+             {
+             case RID_BUILTIN_BSWAPG:
+               ifn = IFN_BSWAP;
+               name = "__builtin_bswapg";
+               break;
+             case RID_BUILTIN_BITREVERSEG:
+               ifn = IFN_BITREVERSE;
+               name = "__builtin_bitreverseg";
+               break;
+             case RID_BUILTIN_CLMULG:
+               ifn = IFN_CLMUL;
+               name = "__builtin_clmulg";
+               break;
+             default:
+               gcc_unreachable ();
+             }
 
            c_parser_consume_token (parser);
            if (!c_parser_get_builtin_args (parser, name, &cexpr_list, false,
@@ -13574,7 +13588,9 @@ c_parser_postfix_expression (c_parser *parser)
                break;
              }
 
-           if (vec_safe_length (cexpr_list) != 1)
+           const unsigned nargs = vec_safe_length (cexpr_list);
+           if (nargs < (ifn == IFN_CLMUL ? 2 : 1)
+               || nargs > (ifn == IFN_CLMUL ? 3 : 1))
              {
                error_at (loc, "wrong number of arguments to %qs", name);
                expr.set_error ();
@@ -13592,41 +13608,119 @@ c_parser_postfix_expression (c_parser *parser)
            tree type = TYPE_MAIN_VARIANT (TREE_TYPE (arg));
            if (!INTEGRAL_TYPE_P (type))
              {
-               error_at (loc, "%qs operand not an integral type", name);
+               error_at (arg_p->get_location (),
+                         "%qs operand not an integral type", name);
                expr.set_error ();
                break;
              }
            if (TREE_CODE (type) == ENUMERAL_TYPE)
              {
-               error_at (loc, "argument %u in call to function "
+               error_at (arg_p->get_location (),
+                         "argument %u in call to function "
                          "%qs has enumerated type", 1, name);
                expr.set_error ();
                break;
              }
            if (TREE_CODE (type) == BOOLEAN_TYPE)
              {
-               error_at (loc, "argument %u in call to function "
+               error_at (arg_p->get_location (),
+                         "argument %u in call to function "
                          "%qs has boolean type", 1, name);
                expr.set_error ();
                break;
              }
            if (!TYPE_UNSIGNED (type))
              {
-               error_at (loc, "argument 1 in call to function "
+               error_at (arg_p->get_location (),
+                         "argument 1 in call to function "
                          "%qs has signed type", name);
                expr.set_error ();
                break;
              }
            if (ifn == IFN_BSWAP && (TYPE_PRECISION (type) % 8) != 0)
              {
-               error_at (loc, "precision %d of argument 1 to function "
+               error_at (arg_p->get_location (),
+                         "precision %d of argument 1 to function "
                          "%qs is not a multiple of 8",
                          TYPE_PRECISION (type), name);
                expr.set_error ();
                break;
              }
            arg = c_fully_fold (arg, false, NULL);
-           expr.value = fold_build_builtin_bswapg_bitreverseg (loc, ifn, arg);
+           if (ifn == IFN_CLMUL)
+             {
+               arg_p = &(*cexpr_list)[1];
+               *arg_p = convert_lvalue_to_rvalue (loc, *arg_p, true, true);
+               tree arg2 = arg_p->value;
+               if (error_operand_p (arg2))
+                 {
+                   expr.set_error ();
+                   break;
+                 }
+               if (!comptypes_same_p (type,
+                                      TYPE_MAIN_VARIANT (TREE_TYPE (arg2))))
+                 {
+                   error_at (loc, "arguments 1 and 2 to function %qs "
+                                  "have different types %qT and %qT",
+                             name, type,
+                             TYPE_MAIN_VARIANT (TREE_TYPE (arg2)));
+                   expr.set_error ();
+                   break;
+                 }
+               arg2 = c_fully_fold (arg2, false, NULL);
+               tree arg3 = NULL_TREE;
+               if (nargs == 3)
+                 {
+                   arg_p = &(*cexpr_list)[2];
+                   *arg_p = convert_lvalue_to_rvalue (loc, *arg_p,
+                                                      true, true);
+                   arg3 = arg_p->value;
+                   if (error_operand_p (arg3))
+                     {
+                       expr.set_error ();
+                       break;
+                     }
+                   if (!POINTER_TYPE_P (TREE_TYPE (arg3)))
+                     {
+                       error_at (arg_p->get_location (),
+                                 "argument 3 to function %qs does not have "
+                                 "pointer type", name);
+                       expr.set_error ();
+                       break;
+                     }
+                   tree htype = TREE_TYPE (TREE_TYPE (arg3));
+                   if (!comptypes_same_p (type, TYPE_MAIN_VARIANT (htype)))
+                     {
+                       error_at (loc, "arguments 1 and 3 to function %qs "
+                                      "have incompatible types %qT and %qT",
+                                 name, type, TREE_TYPE (arg3));
+                       expr.set_error ();
+                       break;
+                     }
+                   if (TYPE_READONLY (htype))
+                     {
+                       error_at (arg_p->get_location (),
+                                 "argument 3 to function %qs has pointer to "
+                                 "%qs type %qT",
+                                 name, "const", TREE_TYPE (arg3));
+                       expr.set_error ();
+                       break;
+                     }
+                   if (TYPE_ATOMIC (htype))
+                     {
+                       error_at (arg_p->get_location (),
+                                 "argument 3 to function %qs "
+                                 "has pointer to %qs type %qT",
+                                 name, "_Atomic", TREE_TYPE (arg3));
+                       expr.set_error ();
+                       break;
+                     }
+                 }
+               expr.value = fold_build_builtin_clmulg (loc, arg, arg2, arg3);
+             }
+           else
+             expr.value = fold_build_builtin_bswapg_bitreverseg (loc, ifn,
+                                                                 arg);
            set_c_expr_source_range (&expr, loc, close_paren_loc);
            break;
          }
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index fda267fc477f..c4d408943ec7 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -28879,6 +28879,49 @@
    (set_attr "c86_decode" "*,vector,vector")
    (set_attr "mode" "TI")])
 
+(define_expand "clmuldi3"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:DI 1 "register_operand")
+   (match_operand:DI 2 "register_operand")]
+  "TARGET_PCLMUL"
+{
+  rtx x = gen_reg_rtx (V2DImode);
+  rtx y = gen_reg_rtx (V2DImode);
+  rtx z = gen_reg_rtx (V2DImode);
+  emit_move_insn (x, gen_rtx_VEC_CONCAT (V2DImode, operands[1], const0_rtx));
+  emit_move_insn (y, gen_rtx_VEC_CONCAT (V2DImode, operands[2], const0_rtx));
+  emit_insn (gen_pclmulqdq (z, x, y, const0_rtx));
+  rtvec tmp = rtvec_alloc (1);
+  RTVEC_ELT (tmp, 0) = const0_rtx;
+  rtx selector = gen_rtx_PARALLEL (VOIDmode, tmp);
+  emit_move_insn (operands[0], gen_rtx_VEC_SELECT (DImode, z, selector));
+  DONE;
+})
+
+(define_expand "clmul_widendi3"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:DI 1 "register_operand")
+   (match_operand:DI 2 "register_operand")
+   (match_operand:DI 3 "register_operand")]
+  "TARGET_PCLMUL"
+{
+  rtx x = gen_reg_rtx (V2DImode);
+  rtx y = gen_reg_rtx (V2DImode);
+  rtx z = gen_reg_rtx (V2DImode);
+  emit_move_insn (x, gen_rtx_VEC_CONCAT (V2DImode, operands[2], const0_rtx));
+  emit_move_insn (y, gen_rtx_VEC_CONCAT (V2DImode, operands[3], const0_rtx));
+  emit_insn (gen_pclmulqdq (z, x, y, const0_rtx));
+  rtvec tmp = rtvec_alloc (1);
+  RTVEC_ELT (tmp, 0) = const0_rtx;
+  rtx selector = gen_rtx_PARALLEL (VOIDmode, tmp);
+  emit_move_insn (operands[0], gen_rtx_VEC_SELECT (DImode, z, selector));
+  tmp = rtvec_alloc (1);
+  RTVEC_ELT (tmp, 0) = const1_rtx;
+  selector = gen_rtx_PARALLEL (VOIDmode, tmp);
+  emit_move_insn (operands[1], gen_rtx_VEC_SELECT (DImode, z, selector));
+  DONE;
+})
+
 (define_expand "avx_vzeroall"
   [(match_par_dup 0 [(const_int 0)])]
   "TARGET_AVX"
diff --git a/gcc/cp/constexpr.cc b/gcc/cp/constexpr.cc
index ec4969843b72..32e9fb7673fa 100644
--- a/gcc/cp/constexpr.cc
+++ b/gcc/cp/constexpr.cc
@@ -3738,6 +3738,49 @@ cxx_eval_internal_function (const constexpr_ctx *ctx, 
tree t,
                                                      arg);
       }
 
+    case IFN_CLMUL:
+      {
+       tree args[3] = {};
+       for (int i = 0; i < call_expr_nargs (t); ++i)
+         {
+           args[i] = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, i),
+                                                   vc_prvalue, non_constant_p,
+                                                   overflow_p, jump_target);
+           if (*jump_target)
+             return NULL_TREE;
+           if (*non_constant_p)
+             return t;
+         }
+       location_t loc = cp_expr_loc_or_input_loc (t);
+       if (TREE_CODE (args[0]) != INTEGER_CST
+           || TREE_CODE (args[1]) != INTEGER_CST)
+         {
+           if (!ctx->quiet)
+             error_at (loc, "call to internal function %qE", t);
+           *non_constant_p = true;
+           return t;
+         }
+       unsigned int prec = TYPE_PRECISION (TREE_TYPE (args[0]));
+       wide_int res = wi::clmul (wi::to_wide (args[0]),
+                                 wi::to_wide (args[1]),
+                                 prec * (args[2] ? 2 : 1));
+       if (args[2] == NULL_TREE)
+         return wide_int_to_tree (TREE_TYPE (t), res);
+       wide_int lo = wide_int::from (res, prec, UNSIGNED);
+       wide_int hi = wide_int::from (wi::rshift (res, prec, UNSIGNED),
+                                     prec, UNSIGNED);
+       tree mod = build2_loc (loc, MODIFY_EXPR, void_type_node,
+                              build_fold_indirect_ref_loc (loc, args[2]),
+                              wide_int_to_tree (TREE_TYPE (t), hi));
+       cxx_eval_constant_expression (ctx, mod, vc_prvalue, non_constant_p,
+                                     overflow_p, jump_target);
+       if (*jump_target)
+         return NULL_TREE;
+       if (*non_constant_p)
+         return t;
+       return wide_int_to_tree (TREE_TYPE (t), lo);
+      }
+
     case IFN_VEC_CONVERT:
       {
        tree arg = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 0),
@@ -12680,6 +12723,7 @@ potential_constant_expression_1 (tree t, bool 
want_rval, bool strict, bool now,
                case IFN_VEC_CONVERT:
                case IFN_BSWAP:
                case IFN_BITREVERSE:
+               case IFN_CLMUL:
                  bail = false;
                  break;
 
diff --git a/gcc/cp/cp-gimplify.cc b/gcc/cp/cp-gimplify.cc
index 76dc9aeb1813..247ac4eb86ea 100644
--- a/gcc/cp/cp-gimplify.cc
+++ b/gcc/cp/cp-gimplify.cc
@@ -898,6 +898,24 @@ cp_gimplify_expr (tree *expr_p, gimple_seq *pre_p, 
gimple_seq *post_p)
                return ret;
              }
            break;
+         case IFN_CLMUL:
+           if (ret == GS_OK)
+             {
+               location_t loc = EXPR_LOCATION (*expr_p);
+               tree arg1 = CALL_EXPR_ARG (*expr_p, 0);
+               tree arg2 = CALL_EXPR_ARG (*expr_p, 1);
+               tree arg3 = NULL_TREE;
+               if (call_expr_nargs (*expr_p) == 3)
+                 arg3 = CALL_EXPR_ARG (*expr_p, 2);
+               tree r = fold_build_builtin_clmulg (loc, arg1, arg2, arg3);
+               if (TREE_CODE (r) == CALL_EXPR
+                   && !CALL_EXPR_FN (r)
+                   && CALL_EXPR_IFN (r) == IFN_CLMUL)
+                 break;
+               *expr_p = r;
+               return ret;
+             }
+           break;
          default:
            break;
          }
diff --git a/gcc/cp/cp-objcp-common.cc b/gcc/cp/cp-objcp-common.cc
index 0cdf35adaa77..19ee211e248f 100644
--- a/gcc/cp/cp-objcp-common.cc
+++ b/gcc/cp/cp-objcp-common.cc
@@ -574,6 +574,7 @@ names_builtin_p (const char *name)
     case RID_ADDRESSOF:
     case RID_BUILTIN_BITREVERSEG:
     case RID_BUILTIN_BSWAPG:
+    case RID_BUILTIN_CLMULG:
     case RID_BUILTIN_CONVERTVECTOR:
     case RID_BUILTIN_HAS_ATTRIBUTE:
     case RID_BUILTIN_SHUFFLE:
diff --git a/gcc/cp/cp-tree.h b/gcc/cp/cp-tree.h
index f87d14006404..27a1a3cc90cc 100644
--- a/gcc/cp/cp-tree.h
+++ b/gcc/cp/cp-tree.h
@@ -9068,6 +9068,9 @@ extern tree build_x_shufflevector               
(location_t,
 extern tree build_x_bswapg_bitreverseg         (location_t, internal_fn,
                                                 vec<tree, va_gc> *,
                                                 tsubst_flags_t);
+extern tree build_x_clmulg                     (location_t,
+                                                vec<tree, va_gc> *,
+                                                tsubst_flags_t);
 #define cxx_sizeof(T)  cxx_sizeof_or_alignof_type (input_location, T, 
SIZEOF_EXPR, false, true)
 extern tree build_simple_component_ref         (tree, tree);
 extern tree build_ptrmemfunc_access_expr       (tree, tree);
diff --git a/gcc/cp/mangle.cc b/gcc/cp/mangle.cc
index db194b8b3977..0ce322fb52ba 100644
--- a/gcc/cp/mangle.cc
+++ b/gcc/cp/mangle.cc
@@ -4110,16 +4110,19 @@ write_expression (tree expr)
          else
            switch (CALL_EXPR_IFN (expr))
              {
-             /* Calls to __builtin_bswapg/__builtin_bitreverseg are parsed
-                as IFN_BSWAP/IFN_BITREVERSE calls because we don't have
-                those builtins but instead keywords.  Pretend they are
-                builtin function template.  */
+             /* Calls to 
__builtin_bswapg/__builtin_bitreverseg/__builtin_clmulg
+                are parsed as IFN_BSWAP/IFN_BITREVERSE/IFN_CLMUL calls because
+                we don't have those builtins but instead keywords.  Pretend
+                they are builtin function template.  */
              case IFN_BSWAP:
                write_source_name (get_identifier ("__builtin_bswapg"));
                break;
              case IFN_BITREVERSE:
                write_source_name (get_identifier ("__builtin_bitreverseg"));
                break;
+             case IFN_CLMUL:
+               write_source_name (get_identifier ("__builtin_clmulg"));
+               break;
              default:
                gcc_unreachable ();
              }
diff --git a/gcc/cp/parser.cc b/gcc/cp/parser.cc
index 28eb38cd7803..349ffeedbcd0 100644
--- a/gcc/cp/parser.cc
+++ b/gcc/cp/parser.cc
@@ -8729,6 +8729,7 @@ cp_parser_postfix_expression (cp_parser *parser, bool 
address_p, bool cast_p,
     case RID_BUILTIN_OPERATOR_DELETE:
     case RID_BUILTIN_BSWAPG:
     case RID_BUILTIN_BITREVERSEG:
+    case RID_BUILTIN_CLMULG:
       {
        vec<tree, va_gc> *vec;
 
@@ -8860,6 +8861,11 @@ cp_parser_postfix_expression (cp_parser *parser, bool 
address_p, bool cast_p,
                                            tf_warning_or_error);
            break;
 
+         case RID_BUILTIN_CLMULG:
+           postfix_expression
+             = build_x_clmulg (loc, vec, tf_warning_or_error);
+           break;
+
          default:
            gcc_unreachable ();
          }
diff --git a/gcc/cp/pt.cc b/gcc/cp/pt.cc
index e6a68bf1d374..b77039ced03e 100644
--- a/gcc/cp/pt.cc
+++ b/gcc/cp/pt.cc
@@ -22732,6 +22732,14 @@ tsubst_expr (tree t, tree args, tsubst_flags_t 
complain, tree in_decl)
                break;
              }
 
+           case IFN_CLMUL:
+             {
+               ret = build_x_clmulg (input_location, call_args, complain);
+               if (ret != error_mark_node)
+                 RETURN (ret);
+               break;
+             }
+
            case IFN_ASSUME:
              gcc_assert (nargs == 1);
              if (vec_safe_length (call_args) != 1)
diff --git a/gcc/cp/typeck.cc b/gcc/cp/typeck.cc
index 0b200804fe99..41f731150fdd 100644
--- a/gcc/cp/typeck.cc
+++ b/gcc/cp/typeck.cc
@@ -7180,6 +7180,107 @@ build_x_bswapg_bitreverseg (location_t loc, internal_fn 
ifn,
     }
   return exp;
 }
+
+/* Build __builtin_clmulg (ARG1, ARG2) or
+   __builtin_clmulg (ARG1, ARG2, ARG3).  */
+tree
+build_x_clmulg (location_t loc, vec<tree, va_gc> *args,
+               tsubst_flags_t complain)
+{
+  const char *name = "__builtin_clmulg";
+  if (args->length () < 2 || args->length () > 3)
+    {
+      if (complain & tf_error)
+       error_at (loc, "wrong number of arguments to %qs", name);
+      return error_mark_node;
+    }
+  tree arg1 = (*args)[0];
+  tree arg2 = (*args)[1];
+  tree arg3 = args->length () == 3 ? (*args)[2] : NULL_TREE;
+  if (error_operand_p (arg1)
+      || error_operand_p (arg2)
+      || error_operand_p (arg3))
+    return error_mark_node;
+  if (type_dependent_expression_p (arg1)
+      || type_dependent_expression_p (arg2)
+      || type_dependent_expression_p (arg3))
+    {
+      tree exp = build_min_nt_call_vec (NULL, args);
+      CALL_EXPR_IFN (exp) = IFN_CLMUL;
+      return exp;
+    }
+  tree type = TYPE_MAIN_VARIANT (TREE_TYPE (arg1));
+  if (!INTEGRAL_TYPE_P (type))
+    {
+      if (complain & tf_error)
+       error_at (loc, "%qs operand not an integral type", name);
+      return error_mark_node;
+    }
+  if (TREE_CODE (type) == ENUMERAL_TYPE)
+    {
+      if (complain & tf_error)
+       error_at (loc, "argument %u in call to function "
+                      "%qs has enumerated type", 1, name);
+      return error_mark_node;
+    }
+  if (TREE_CODE (type) == BOOLEAN_TYPE)
+    {
+      if (complain & tf_error)
+       error_at (loc, "argument %u in call to function "
+                      "%qs has boolean type", 1, name);
+      return error_mark_node;
+    }
+  if (!TYPE_UNSIGNED (type))
+    {
+      if (complain & tf_error)
+       error_at (loc, "argument 1 in call to function "
+                      "%qs has signed type", name);
+      return error_mark_node;
+    }
+  if (!same_type_p (type, TYPE_MAIN_VARIANT (TREE_TYPE (arg2))))
+    {
+      if (complain & tf_error)
+       error_at (loc, "arguments 1 and 2 to function %qs "
+                      "have different types %qT and %qT",
+                 name, type, TYPE_MAIN_VARIANT (TREE_TYPE (arg2)));
+      return error_mark_node;
+    }
+  if (arg3)
+    {
+      if (!POINTER_TYPE_P (TREE_TYPE (arg3)))
+       {
+         if (complain & tf_error)
+           error_at (loc, "argument 3 to function %qs does not have "
+                          "pointer type", name);
+         return error_mark_node;
+       }
+      tree htype = TREE_TYPE (TREE_TYPE (arg3));
+      if (!same_type_p (type, TYPE_MAIN_VARIANT (htype)))
+       {
+         if (complain & tf_error)
+           error_at (loc, "arguments 1 and 3 to function %qs "
+                          "have incompatible types %qT and %qT",
+                     name, type, TREE_TYPE (arg3));
+         return error_mark_node;
+       }
+      if (TYPE_READONLY (htype))
+       {
+         if (complain & tf_error)
+           error_at (loc, "argument 3 to function %qs "
+                          "has pointer to %qs type %qT",
+                     name, "const", TREE_TYPE (arg3));
+         return error_mark_node;
+       }
+    }
+  tree exp = build_call_expr_internal_loc (loc, IFN_CLMUL, type,
+                                          arg3 ? 3 : 2, arg1, arg2, arg3);
+  if (processing_template_decl)
+    {
+      exp = build_min_non_dep_call_vec (exp, NULL, args);
+      CALL_EXPR_IFN (exp) = IFN_CLMUL;
+    }
+  return exp;
+}
 
 /* Return a tree for the sum or difference (RESULTCODE says which)
    of pointer PTROP and integer INTOP.  */
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index aef35e7e37c4..efcaae0737d7 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -18101,6 +18101,29 @@ recognized in such contexts.
 
 @enddefbuiltin
 
+@defbuiltin{@var{type} __builtin_clmulg (@var{type} @var{a}, @var{type} 
@var{b})}
+@defbuiltinx{@var{type} __builtin_clmulg (@var{type} @var{a}, @var{type} 
@var{b}, @var{type} *@var{c})}
+
+This type-generic built-in function returns carry-less product of the
+first two arguments.  @var{type} may be any unsigned integral type.
+The function returns the lower half of the full result, which has
+twice the precision of @var{type}.  If three arguments are specified,
+the higher half of the full result is stored into the object the argument
+points to.
+
+@smallexample
+  unsigned int a = 0x10111111U, b = 0x22002222U;
+  unsigned int c = __builtin_clmulg (a, b);
+  unsigned int d, e = __builtin_clmulg (a, b, &d);
+@end smallexample
+
+If @code{int} is 32-bit, both @var{c} and @var{e} will be set to the least 
significant
+32 bits of the full 64-bit carry-less product result 
@code{0x222022020000202ULL},
+i.e. @code{0x20000202U}, while @var{d} will be set to the most significant 32 
bits
+of the full result, i.e. @code{0x2220220U}.
+
+@enddefbuiltin
+
 @defbuiltin{@var{type} __builtin_speculation_safe_value (@var{type} @var{val}, 
@var{type} @var{failval})}
 
 This built-in function can be used to help mitigate against unsafe
diff --git a/gcc/fold-const-call.cc b/gcc/fold-const-call.cc
index a4bf526d0bf8..f7d6193ac2d0 100644
--- a/gcc/fold-const-call.cc
+++ b/gcc/fold-const-call.cc
@@ -1804,6 +1804,10 @@ fold_const_call_sss (wide_int *result, combined_fn fn,
        return true;
       }
 
+    case CFN_CLMUL:
+      *result = wi::clmul (arg0, arg1, precision);
+      return true;
+
     default:
       return false;
     }
@@ -2092,6 +2096,20 @@ fold_const_call (combined_fn fn, tree type, tree arg0, 
tree arg1)
        }
       return NULL_TREE;
 
+    case CFN_CLMUL_WIDEN:
+      if (integer_cst_p (arg0) && integer_cst_p (arg1))
+       {
+         unsigned int prec = TYPE_PRECISION (TREE_TYPE (arg0));
+         wide_int res = wi::clmul (wi::to_wide (arg0), wi::to_wide (arg1),
+                                   prec * 2);
+         wide_int lo = wide_int::from (res, prec, UNSIGNED);
+         wide_int hi = wide_int::from (wi::rshift (res, prec, UNSIGNED),
+                                       prec, UNSIGNED);
+         return build_complex (type, wide_int_to_tree (TREE_TYPE (type), lo),
+                               wide_int_to_tree (TREE_TYPE (type), hi));
+       }
+      return NULL_TREE;
+
     default:
       return fold_const_call_1 (fn, type, arg0, arg1);
     }
diff --git a/gcc/fold-const.cc b/gcc/fold-const.cc
index a37026e1be76..4bb80084b8b0 100644
--- a/gcc/fold-const.cc
+++ b/gcc/fold-const.cc
@@ -14924,6 +14924,7 @@ tree_call_nonnegative_p (tree type, combined_fn fn, 
tree arg0, tree arg1,
     CASE_CFN_CLRSB:
     CASE_CFN_BSWAP:
     CASE_CFN_BITREVERSE:
+    case CFN_CLMUL:
       /* Always true.  */
       return true;
 
diff --git a/gcc/genmatch.cc b/gcc/genmatch.cc
index 7d3c56570f22..77a06ecedd3b 100644
--- a/gcc/genmatch.cc
+++ b/gcc/genmatch.cc
@@ -1393,6 +1393,8 @@ commutative_op (id_base *id, bool 
compares_are_commutative = false)
       case CFN_FNMS:
       case CFN_ADD_OVERFLOW:
       case CFN_MUL_OVERFLOW:
+      case CFN_CLMUL:
+      case CFN_CLMUL_WIDEN:
       case CFN_SAT_ADD:
        return 0;
 
diff --git a/gcc/gimple-lower-bitint.cc b/gcc/gimple-lower-bitint.cc
index f871f68ba1e6..4b5163c0b9fd 100644
--- a/gcc/gimple-lower-bitint.cc
+++ b/gcc/gimple-lower-bitint.cc
@@ -478,6 +478,7 @@ struct bitint_large_huge
   void lower_complexexpr_stmt (gimple *);
   void lower_bit_query (gimple *);
   void lower_bswap_bitreverse (tree, gimple *);
+  void lower_clmul (tree, gimple *);
   void lower_call (tree, gimple *);
   void lower_asm (gimple *);
   void lower_stmt (gimple *);
@@ -6305,6 +6306,236 @@ bitint_large_huge::lower_bswap_bitreverse (tree obj, 
gimple *stmt)
     }
 }
 
+/* Lower a .CLMUL call with two large/huge _BitInt arguments.  */
+
+void
+bitint_large_huge::lower_clmul (tree obj, gimple *stmt)
+{
+  tree arg1 = gimple_call_arg (stmt, 0);
+  tree arg2 = gimple_call_arg (stmt, 1);
+  tree lhs = gimple_call_lhs (stmt);
+  gimple *g;
+
+  if (!lhs)
+    {
+      gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
+      gsi_remove (&gsi, true);
+      return;
+    }
+  if (wi::popcount (get_nonzero_bits (arg1))
+      < wi::popcount (get_nonzero_bits (arg2)))
+    std::swap (arg1, arg2);
+  tree type = TREE_TYPE (arg1);
+  gcc_assert (BITINT_TYPE_P (type));
+  bitint_prec_kind kind = bitint_precision_kind (type);
+  gcc_assert (kind >= bitint_prec_large);
+  if (!obj)
+    {
+      int part = var_to_partition (m_map, lhs);
+      gcc_assert (m_vars[part] != NULL_TREE);
+      obj = m_vars[part];
+    }
+  /* For .CLMUL, we can use handle_operand with non-INTEGER_CST
+     idx even to access the most significant partial limb.  */
+  m_var_msb = true;
+  unsigned prec = TYPE_PRECISION (type);
+  /* Lower
+       dst = .CLMUL (src1, src2);
+     as
+       size_t p = prec / limb_prec - (prec % limb_prec == 0);
+       dst = {};
+       for (idx = 0; idx <= p; ++idx)
+        {
+          limb_type t = src2[idx];
+          size_t i = 0, j = 0;
+          do
+            {
+              if (t & 1)
+                {
+                  size_t idx3 = idx + j;
+                  size_t k = -i & (limb_prec - 1);
+                  r[idx] ^= (src1[0] << i);
+                  for (idx2 = idx + 1; idx2 <= p; ++idx2)
+                    r[idx2] ^= (src1[idx2 - idx] << i)
+                               | (src1[idx2 - idx3] >> k);
+                }
+              t >>= 1;
+              ++i;
+              j = 1;
+            }
+          while (t);
+        }  */
+  tree p = build_int_cst (sizetype,
+                         prec / limb_prec - (prec % limb_prec == 0));
+  tree zero = build_zero_cst (TREE_TYPE (obj));
+  g = gimple_build_assign (obj, zero);
+  insert_before (g);
+  tree idx_next;
+  tree idx = create_loop (bitint_big_endian ? p : size_zero_node, &idx_next);
+  m_data_cnt = 0;
+  tree t_init = handle_operand (arg2, idx);
+  m_first = false;
+  tree i_next;
+  tree i = create_loop (size_zero_node, &i_next);
+  tree j = make_ssa_name (sizetype);
+  gphi *phi = create_phi_node (j, m_bb);
+  add_phi_arg (phi, size_zero_node, find_edge (m_preheader_bb, m_bb),
+              UNKNOWN_LOCATION);
+  add_phi_arg (phi, size_one_node, find_edge (m_bb, m_bb), UNKNOWN_LOCATION);
+  tree t = make_ssa_name (m_limb_type);
+  phi = create_phi_node (t, m_bb);
+  add_phi_arg (phi, t_init, find_edge (m_preheader_bb, m_bb),
+              UNKNOWN_LOCATION);
+  tree t_next = make_ssa_name (m_limb_type);
+  add_phi_arg (phi, t_next, find_edge (m_bb, m_bb), UNKNOWN_LOCATION);
+  tree t1 = make_ssa_name (m_limb_type);
+  g = gimple_build_assign (t1, BIT_AND_EXPR, t, build_one_cst (m_limb_type));
+  insert_before (g);
+  g = gimple_build_cond (NE_EXPR, t1, build_zero_cst (m_limb_type),
+                        NULL_TREE, NULL_TREE);
+  edge edge_true, edge_false;
+  if_then (g, profile_probability::even (), edge_true, edge_false);
+  tree idx3 = make_ssa_name (sizetype), idx4 = idx;
+  g = gimple_build_assign (idx3, bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
+                          idx, j);
+  insert_before (g);
+  if (bitint_big_endian)
+    {
+      g = gimple_build_assign (make_ssa_name (sizetype), MINUS_EXPR, p, idx3);
+      insert_before (g);
+      idx3 = gimple_assign_lhs (g);
+      idx4 = make_ssa_name (sizetype);
+      g = gimple_build_assign (idx4, MINUS_EXPR, p, idx);
+      insert_before (g);
+    }
+  g = gimple_build_assign (make_ssa_name (sizetype), NEGATE_EXPR, i);
+  insert_before (g);
+  tree k = make_ssa_name (sizetype);
+  g = gimple_build_assign (k, BIT_AND_EXPR, gimple_assign_lhs (g),
+                          size_int (limb_prec - 1));
+  insert_before (g);
+  unsigned data_cnt = m_data_cnt;
+  m_first = true;
+  tree s1 = handle_operand (arg1, bitint_big_endian ? p : size_zero_node);
+  m_first = false;
+  m_data_cnt = data_cnt;
+  g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR, s1, i);
+  insert_before (g);
+  s1 = gimple_assign_lhs (g);
+  tree r = limb_access (TREE_TYPE (lhs), obj, idx, false);
+  g = gimple_build_assign (make_ssa_name (m_limb_type), r);
+  insert_before (g);
+  g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_XOR_EXPR,
+                          gimple_assign_lhs (g), s1);
+  insert_before (g);
+  r = limb_access (TREE_TYPE (lhs), obj, idx, true);
+  g = gimple_build_assign (r, gimple_assign_lhs (g));
+  insert_before (g);
+  tree idxp1 = make_ssa_name (sizetype);
+  g = gimple_build_assign (idxp1, PLUS_EXPR, idx,
+                          bitint_big_endian ? size_int (-1) : size_one_node);
+  insert_before (g);
+  if (bitint_big_endian)
+    g = gimple_build_cond (NE_EXPR, idxp1, size_int (-1),
+                          NULL_TREE, NULL_TREE);
+  else
+    g = gimple_build_cond (LE_EXPR, idxp1, p, NULL_TREE, NULL_TREE);
+  edge edge_true2, edge_false2;
+  if_then (g, profile_probability::likely (), edge_true2, edge_false2);
+  tree idx2_next;
+  tree idx2 = create_loop (idxp1, &idx2_next);
+  tree idx2midx = make_ssa_name (sizetype);
+  g = gimple_build_assign (idx2midx, MINUS_EXPR, idx2, idx4);
+  insert_before (g);
+  tree idx2midx3 = make_ssa_name (sizetype);
+  g = gimple_build_assign (idx2midx3, MINUS_EXPR, idx2, idx3);
+  insert_before (g);
+  m_data_cnt = data_cnt;
+  tree s2 = handle_operand (arg1, idx2midx);
+  g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
+                          s2, i);
+  insert_before (g);
+  s2 = gimple_assign_lhs (g);
+  m_data_cnt = data_cnt;
+  tree s3 = handle_operand (arg1, idx2midx3);
+  g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
+                          s3, k);
+  insert_before (g);
+  g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
+                          s2, gimple_assign_lhs (g));
+  insert_before (g);
+  s2 = gimple_assign_lhs (g);
+  r = limb_access (TREE_TYPE (lhs), obj, idx2, false);
+  g = gimple_build_assign (make_ssa_name (m_limb_type), r);
+  insert_before (g);
+  g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_XOR_EXPR,
+                          gimple_assign_lhs (g), s2);
+  insert_before (g);
+  r = limb_access (TREE_TYPE (lhs), obj, idx2, true);
+  g = gimple_build_assign (r, gimple_assign_lhs (g));
+  insert_before (g);
+  g = gimple_build_assign (idx2_next, PLUS_EXPR, idx2,
+                          bitint_big_endian ? size_int (-1) : size_one_node);
+  insert_before (g);
+  if (bitint_big_endian)
+    g = gimple_build_cond (NE_EXPR, idx2_next, size_int (-1),
+                          NULL_TREE, NULL_TREE);
+  else
+    g = gimple_build_cond (LE_EXPR, idx2_next, p, NULL_TREE, NULL_TREE);
+  insert_before (g);
+  m_gsi = gsi_after_labels (edge_false->dest);
+  g = gimple_build_assign (t_next, RSHIFT_EXPR, t, integer_one_node);
+  insert_before (g);
+  g = gimple_build_assign (i_next, PLUS_EXPR, i, size_one_node);
+  insert_before (g);
+  g = gimple_build_cond (NE_EXPR, t_next, build_zero_cst (m_limb_type),
+                        NULL_TREE, NULL_TREE);
+  insert_before (g);
+  if ((EDGE_SUCC (edge_false->dest, 0)->flags & EDGE_FALSE_VALUE) != 0)
+    m_gsi = gsi_after_labels (EDGE_SUCC (edge_false->dest, 0)->dest);
+  else
+    m_gsi = gsi_after_labels (EDGE_SUCC (edge_false->dest, 1)->dest);
+  g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
+                          bitint_big_endian ? size_int (-1) : size_one_node);
+  insert_before (g);
+  if (bitint_big_endian)
+    g = gimple_build_cond (NE_EXPR, idx_next, size_int (-1),
+                          NULL_TREE, NULL_TREE);
+  else
+    g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
+  insert_before (g);
+  m_gsi = gsi_for_stmt (stmt);
+  if (bitint_extended && prec % limb_prec != 0)
+    {
+      /* The most significant limb has been updated either in the
+        loop or in the if after it.  To simplify the code, just
+        read it back from memory and extend.  */
+      idx = bitint_big_endian ? size_zero_node : p;
+      tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
+      tree type = limb_access_type (TREE_TYPE (lhs), idx);
+      tree v = make_ssa_name (m_limb_type);
+      g = gimple_build_assign (v, l);
+      insert_before (g);
+      v = add_cast (type, v);
+      l = limb_access (TREE_TYPE (lhs), obj, idx, true);
+      v = add_cast (m_limb_type, v);
+      g = gimple_build_assign (l, v);
+      insert_before (g);
+    }
+  if (bitint_extended == bitint_ext_full
+      && abi_limb_prec > limb_prec
+      && (CEIL (prec, abi_limb_prec) * abi_limb_prec
+         > CEIL (prec, limb_prec) * limb_prec))
+    {
+      tree p2 = build_int_cst (sizetype,
+                              CEIL (prec, abi_limb_prec)
+                              * abi_limb_prec / limb_prec - 1);
+      tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
+      g = gimple_build_assign (l, build_zero_cst (m_limb_type));
+      insert_before (g);
+    }
+}
+
 /* Lower a call statement with one or more large/huge _BitInt
    arguments or large/huge _BitInt return value.  */
 
@@ -6338,6 +6569,9 @@ bitint_large_huge::lower_call (tree obj, gimple *stmt)
       case IFN_BITREVERSE:
        lower_bswap_bitreverse (obj, stmt);
        return;
+      case IFN_CLMUL:
+       lower_clmul (obj, stmt);
+       return;
       default:
        break;
       }
@@ -6654,6 +6888,7 @@ bitint_large_huge::lower_stmt (gimple *stmt)
              {
              case IFN_BSWAP:
              case IFN_BITREVERSE:
+             case IFN_CLMUL:
                lower_call (lhs, g);
                goto handled;
              default:
@@ -6820,10 +7055,11 @@ stmt_needs_operand_addr (gimple *stmt)
       case IFN_MUL_OVERFLOW:
       case IFN_UBSAN_CHECK_MUL:
         return true;
-      /* These two actually don't take address, but reshuffle
+      /* These three actually don't take address, but reshuffle
         all bytes or bits, so need similar treatment.  */
       case IFN_BSWAP:
       case IFN_BITREVERSE:
+      case IFN_CLMUL:
        return true;
       default:
        break;
@@ -7058,6 +7294,7 @@ build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track 
*live,
       case IFN_UBSAN_CHECK_MUL:
       case IFN_BSWAP:
       case IFN_BITREVERSE:
+      case IFN_CLMUL:
        lhs = gimple_call_lhs (stmt);
        if (lhs)
          muldiv_p = true;
@@ -7598,6 +7835,55 @@ gimple_lower_bitint (void)
                add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION);
                break;
              }
+         else if (gimple_call_internal_p (stmt, IFN_CLMUL_WIDEN))
+           {
+             /* For now, lower this to two IFN_CLMUL and three
+                IFN_BITREVERSE calls.  Could be implemented in
+                lower_clmul too, just needs special care on padding
+                bits and split the shifting into up to two disjunct
+                results, or treat result as 2*prec array and adjust
+                afterwards.  */
+             first_large_huge = 0;
+             location_t loc = gimple_location (stmt);
+             tree arg0 = gimple_call_arg (stmt, 0);
+             tree arg1 = gimple_call_arg (stmt, 1);
+             tree type = TREE_TYPE (arg0);
+             gsi = gsi_for_stmt (stmt);
+             gimple *g = gimple_build_call_internal (IFN_CLMUL, 2,
+                                                     arg0, arg1);
+             tree real = make_ssa_name (type);
+             gimple_call_set_lhs (g, real);
+             gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+             gimple_set_location (g, loc);
+             g = gimple_build_call_internal (IFN_BITREVERSE, 1, arg0);
+             tree rarg0 = make_ssa_name (type);
+             gimple_call_set_lhs (g, rarg0);
+             gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+             gimple_set_location (g, loc);
+             g = gimple_build_call_internal (IFN_BITREVERSE, 1, arg1);
+             tree rarg1 = make_ssa_name (type);
+             gimple_call_set_lhs (g, rarg1);
+             gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+             gimple_set_location (g, loc);
+             g = gimple_build_call_internal (IFN_CLMUL, 2, rarg0, rarg1);
+             tree rimag = make_ssa_name (type);
+             gimple_call_set_lhs (g, rimag);
+             gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+             gimple_set_location (g, loc);
+             g = gimple_build_call_internal (IFN_BITREVERSE, 1, rimag);
+             tree imag = make_ssa_name (type);
+             gimple_call_set_lhs (g, imag);
+             gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+             gimple_set_location (g, loc);
+             g = gimple_build_assign (make_ssa_name (type), RSHIFT_EXPR, imag,
+                                      build_one_cst (unsigned_type_node));
+             gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+             gimple_set_location (g, loc);
+             g = gimple_build_assign (gimple_call_lhs (stmt), COMPLEX_EXPR,
+                                      real, gimple_assign_lhs (g));
+             gimple_set_location (g, loc);
+             gsi_replace (&gsi, g, true);
+           }
          else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
                   && is_gimple_call (stmt)
                   && gimple_call_internal_p (stmt))
@@ -7611,6 +7897,7 @@ gimple_lower_bitint (void)
              case IFN_UBSAN_CHECK_MUL:
              case IFN_BSWAP:
              case IFN_BITREVERSE:
+             case IFN_CLMUL:
                for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
                  {
                    location_t loc = gimple_location (stmt);
@@ -7948,6 +8235,7 @@ gimple_lower_bitint (void)
                  {
                  case IFN_BSWAP:
                  case IFN_BITREVERSE:
+                 case IFN_CLMUL:
                    if (gimple_store_p (use_stmt)
                        && is_gimple_assign (use_stmt)
                        && !gimple_has_volatile_ops (use_stmt)
@@ -8069,6 +8357,7 @@ gimple_lower_bitint (void)
                      case IFN_POPCOUNT:
                      case IFN_BSWAP:
                      case IFN_BITREVERSE:
+                     case IFN_CLMUL:
                        continue;
                      default:
                        break;
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 40fe58dc4f60..5fc6c35780c3 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -4653,6 +4653,8 @@ commutative_binary_fn_p (internal_fn fn)
     case IFN_UBSAN_CHECK_MUL:
     case IFN_ADD_OVERFLOW:
     case IFN_MUL_OVERFLOW:
+    case IFN_CLMUL:
+    case IFN_CLMUL_WIDEN:
     case IFN_SAT_ADD:
     case IFN_SAT_MUL:
     case IFN_VEC_WIDEN_PLUS:
@@ -6020,3 +6022,330 @@ expand_BITREVERSE (internal_fn fn, gcall *stmt)
   if (expand_bitquery (fn, stmt))
     expand_unary_optab_fn (fn, stmt, bitreverse_optab);
 }
+
+/* Expand CLMUL (OP0, OP1) where operands are in OP_MODE and have
+   minimum unsigned precision PREC0 and PREC1.  Compute result in MODE.  */
+
+static rtx
+expand_clmul (machine_mode mode, machine_mode op_mode, rtx op0, rtx op1,
+             unsigned int prec0, unsigned int prec1)
+{
+  unsigned int op_prec = GET_MODE_PRECISION (as_a <scalar_int_mode> (op_mode));
+  unsigned int prec = GET_MODE_PRECISION (as_a <scalar_int_mode> (mode));
+  machine_mode cmode;
+  optab_methods olw = OPTAB_LIB_WIDEN;
+  if (MIN (prec0, prec1) == 1)
+    {
+      /* If one operand has [0, 1] range, we can do just
+        multiplication and if both are [0, 1], this is equivalent to AND.  */
+      rtx res = expand_binop (op_mode, prec0 == prec1 ? and_optab : smul_optab,
+                             op0, op1, NULL_RTX, 1, olw);
+      return convert_modes (mode, op_mode, res, 1);
+    }
+  FOR_EACH_MODE_IN_CLASS (cmode, MODE_INT)
+    {
+      unsigned int this_prec
+       = GET_MODE_PRECISION (as_a <scalar_int_mode> (cmode));
+      if (prec0 > this_prec || prec1 > this_prec)
+       continue;
+      if (optab_handler (clmul_optab, cmode) != CODE_FOR_nothing
+         && (prec0 + prec1 - 1 <= this_prec || prec <= this_prec))
+       {
+         rtx_insn *start = get_last_insn ();
+         rtx op0c = convert_modes (cmode, op_mode, op0, 1);
+         rtx op1c = convert_modes (cmode, op_mode, op1, 1);
+         if (rtx res = expand_binop (cmode, clmul_optab, op0c, op1c,
+                                     NULL_RTX, 1, OPTAB_DIRECT))
+           return convert_modes (mode, cmode, res, 1);
+         delete_insns_since (start);
+       }
+      if (optab_handler (clmul_widen_optab, cmode) != CODE_FOR_nothing)
+       {
+         rtx_insn *start = get_last_insn ();
+         rtx op0c = convert_modes (cmode, op_mode, op0, 1);
+         rtx op1c = convert_modes (cmode, op_mode, op1, 1);
+         rtx reslo = gen_reg_rtx (cmode);
+         rtx reshi = gen_reg_rtx (cmode);
+         class expand_operand ops[4];
+         create_output_operand (&ops[0], reslo, cmode);
+         create_output_operand (&ops[1], reshi, cmode);
+         create_input_operand (&ops[2], op0c, cmode);
+         create_input_operand (&ops[3], op1c, cmode);
+         if (maybe_expand_insn (optab_handler (clmul_widen_optab, cmode),
+                                4, ops))
+           {
+             if (prec0 + prec1 - 1 <= this_prec || prec <= this_prec)
+               return convert_modes (mode, cmode, reslo, 1);
+             machine_mode wmode;
+             if (!GET_MODE_2XWIDER_MODE (cmode).exists (&wmode))
+               wmode = mode;
+             reslo = convert_modes (wmode, cmode, reslo, 1);
+             reshi = convert_modes (wmode, cmode, reshi, 1);
+             reshi = expand_shift (LSHIFT_EXPR, wmode, reshi, this_prec,
+                                   NULL_RTX, 1);
+             rtx res = expand_binop (wmode, ior_optab, reslo, reshi,
+                                     NULL_RTX, 1, olw);
+             return convert_modes (mode, wmode, res, 1);
+           }
+         delete_insns_since (start);
+       }
+    }
+  /* Fallback using Kronecker substitution.
+     Depending on the number of active bits in the operand with smaller
+     number of those and the result precision we choose smallest spacing
+     S in [2, 13], such that MIN (prec0, prec1) <= S * ((1 << S) - 1),
+     or prec <= S * (1 << S) or prec0 + prec1 - 1 <= S * (1 << S).
+     The algorithm then uses S * S multiplications, 3 * S bitwise ANDs,
+     S * (S - 1) XORs and S - 1 IORs.
+     For e.g. S = 4 this is:
+     uint64_t x0 = x & 0x1111111111111111ULL, y0 = y & 0x1111111111111111ULL;
+     uint64_t x1 = x & 0x2222222222222222ULL, y1 = y & 0x2222222222222222ULL;
+     uint64_t x2 = x & 0x4444444444444444ULL, y2 = y & 0x4444444444444444ULL;
+     uint64_t x3 = x & 0x8888888888888888ULL, y3 = y & 0x8888888888888888ULL;
+     uint64_t z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
+     uint64_t z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
+     uint64_t z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
+     uint64_t z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
+     z0 &= 0x1111111111111111ULL;
+     z1 &= 0x2222222222222222ULL;
+     z2 &= 0x4444444444444444ULL;
+     z3 &= 0x8888888888888888ULL;
+     return (z0 | z1) | (z2 | z3);  */
+  unsigned int s;
+  for (s = 2; s <= 13; ++s)
+    if (MIN (prec0, prec1) <= s * ((1 << s) - 1)
+       || MIN (prec, prec0 + prec1 - 1) <= s * (1 << s))
+      break;
+  /* S = 13 should handle up to _BitInt maximum 65535 (106483 bits).  */
+  gcc_assert (s <= 13);
+  rtx mask[13], x[13], y[13], t[13], r;
+  bool do_widen_mult = false;
+  if (mode != op_mode
+      && (convert_optab_handler (umul_widen_optab, mode, op_mode)
+         != CODE_FOR_nothing))
+    do_widen_mult = true;
+  for (unsigned int i = 0; i < s; ++i)
+    {
+      mask[i]
+       = immed_wide_int_const (wi::bit_repeat (wide_int::from (1 << i, s,
+                                                               UNSIGNED),
+                                               op_prec), op_mode);
+      if (CONST_INT_P (op0) && CONST_INT_P (mask[i]))
+       x[i] = gen_int_mode (INTVAL (op0) & INTVAL (mask[i]), op_mode);
+      else
+       x[i] = expand_binop (op_mode, and_optab, op0, mask[i], NULL_RTX, 1,
+                            olw);
+      if (mode != op_mode && !do_widen_mult)
+       x[i] = convert_modes (mode, op_mode, x[i], 1);
+      if (CONST_INT_P (op1) && CONST_INT_P (mask[i]))
+       y[i] = gen_int_mode (INTVAL (op1) & INTVAL (mask[i]), op_mode);
+      else
+       y[i] = expand_binop (op_mode, and_optab, op1, mask[i], NULL_RTX, 1,
+                            olw);
+      if (mode != op_mode && !do_widen_mult)
+       y[i] = convert_modes (mode, op_mode, y[i], 1);
+    }
+  for (unsigned int i = 0; i < s; ++i)
+    {
+      for (unsigned int j = 0; j < s; ++j)
+       {
+         unsigned int k = (i + s - j) % s;
+         if (do_widen_mult)
+           t[j] = expand_widening_mult (mode, x[j], y[k], NULL_RTX, 1,
+                                        umul_widen_optab);
+         else
+           t[j] = expand_binop (mode, smul_optab, x[j], y[k], NULL_RTX, 1,
+                                olw);
+         if ((j & 1) == 1)
+           {
+             t[j - 1] = expand_binop (mode, xor_optab, t[j - 1], t[j],
+                                      NULL_RTX, 1, olw);
+             t[j] = NULL_RTX;
+           }
+         if ((j & 3) == 3)
+           {
+             t[j - 3] = expand_binop (mode, xor_optab, t[j - 3], t[j - 1],
+                                      NULL_RTX, 1, olw);
+             t[j - 1] = NULL_RTX;
+           }
+       }
+      for (unsigned int j = 1; j < s; ++j)
+       if (t[j])
+         t[0] = expand_binop (mode, xor_optab, t[0], t[j], NULL_RTX, 1, olw);
+      if (mode != op_mode)
+       mask[i]
+         = immed_wide_int_const (wi::bit_repeat (wide_int::from (1 << i, s,
+                                                                 UNSIGNED),
+                                                 prec), mode);
+      t[0] = expand_binop (mode, and_optab, t[0], mask[i], NULL_RTX, 1, olw);
+      if (i == 0)
+       r = t[0];
+      else
+       r = expand_binop (mode, ior_optab, r, t[0], NULL_RTX, 1, olw);
+    }
+  return r;
+}
+
+void
+expand_CLMUL (internal_fn, gcall *stmt)
+{
+  tree lhs = gimple_call_lhs (stmt);
+  tree arg0 = gimple_call_arg (stmt, 0);
+  tree arg1 = gimple_call_arg (stmt, 1);
+  scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
+  unsigned int prec0 = get_min_precision (arg0, UNSIGNED);
+  unsigned int prec1 = get_min_precision (arg1, UNSIGNED);
+  rtx op0 = expand_normal (arg0);
+  rtx op1 = expand_normal (arg1);
+  rtx res = expand_clmul (mode, mode, op0, op1, prec0, prec1);
+  if (TYPE_PRECISION (TREE_TYPE (arg0)) < GET_MODE_PRECISION (mode))
+    {
+      rtx mask
+       = immed_wide_int_const (wi::mask (TYPE_PRECISION (TREE_TYPE (arg0)),
+                                         false,
+                                         GET_MODE_PRECISION (mode)), mode);
+      res = expand_and (mode, res, mask, NULL_RTX);
+    }
+  rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  emit_move_insn (target, res);
+}
+
+void
+expand_CLMUL_WIDEN (internal_fn, gcall *stmt)
+{
+  tree lhs = gimple_call_lhs (stmt);
+  tree arg0 = gimple_call_arg (stmt, 0);
+  tree arg1 = gimple_call_arg (stmt, 1);
+  scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
+  unsigned int prec0 = get_min_precision (arg0, UNSIGNED);
+  unsigned int prec1 = get_min_precision (arg1, UNSIGNED);
+  unsigned int prec = GET_MODE_PRECISION (mode);
+  rtx op0 = expand_normal (arg0);
+  rtx op1 = expand_normal (arg1);
+  rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+  rtx bitint_mask = NULL_RTX;
+  if (TYPE_PRECISION (TREE_TYPE (arg0)) != prec)
+    bitint_mask
+      = immed_wide_int_const (wi::mask (TYPE_PRECISION (TREE_TYPE (arg0)),
+                                       false, prec), mode);
+  if (prec0 + prec1 - 1 <= prec)
+    {
+      rtx res = expand_clmul (mode, mode, op0, op1, prec0, prec1);
+      if (bitint_mask)
+       res = expand_and (mode, res, bitint_mask, NULL_RTX);
+      write_complex_part (target, res, false, true);
+      write_complex_part (target, const0_rtx, true, false);
+      return;
+    }
+  if (optab_handler (clmul_widen_optab, mode) != CODE_FOR_nothing
+      && TYPE_PRECISION (TREE_TYPE (arg0)) == prec)
+    {
+      rtx_insn *start = get_last_insn ();
+      rtx reslo = gen_reg_rtx (mode);
+      rtx reshi = gen_reg_rtx (mode);
+      class expand_operand ops[4];
+      create_output_operand (&ops[0], reslo, mode);
+      create_output_operand (&ops[1], reshi, mode);
+      create_input_operand (&ops[2], op0, mode);
+      create_input_operand (&ops[3], op1, mode);
+      if (maybe_expand_insn (optab_handler (clmul_widen_optab, mode),
+                            4, ops))
+       {
+         write_complex_part (target, reslo, false, true);
+         write_complex_part (target, reshi, true, false);
+         return;
+       }
+      delete_insns_since (start);
+    }
+  scalar_int_mode wmode;
+  if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
+      && targetm.scalar_mode_supported_p (wmode))
+    {
+      rtx res = expand_clmul (wmode, mode, op0, op1, prec0, prec1);
+      rtx reslo = convert_modes (mode, wmode, res, 1);
+      if (bitint_mask)
+       reslo = expand_and (mode, reslo, bitint_mask, NULL_RTX);
+      write_complex_part (target, reslo, false, true);
+      rtx reshi = expand_shift (RSHIFT_EXPR, wmode, res,
+                               TYPE_PRECISION (TREE_TYPE (arg0)),
+                               NULL_RTX, 1);
+      reshi = convert_modes (mode, wmode, reshi, 1);
+      write_complex_part (target, reshi, true, false);
+      return;
+    }
+  /* If we have native bitreverse in mode, fallback using:
+     uint64_t l = clmul (x, y);
+     uint64_t h = clmul (__builtin_bitreverseg (x),
+                        __builtin_bitreverseg (y));
+     return { l, __builtin_bitreverseg (h) >> 1 };  */
+  if (optab_handler (bitreverse_optab, mode) != CODE_FOR_nothing
+      || (prec == 2 * UNITS_PER_WORD
+         && optab_handler (bitreverse_optab, word_mode) != CODE_FOR_nothing))
+    {
+      rtx reslo = expand_clmul (mode, mode, op0, op1, prec0, prec1);
+      if (bitint_mask)
+       reslo = expand_and (mode, reslo, bitint_mask, NULL_RTX);
+      write_complex_part (target, reslo, false, true);
+      rtx revx = expand_unop (mode, bitreverse_optab, op0, NULL_RTX, 1);
+      rtx revy = expand_unop (mode, bitreverse_optab, op1, NULL_RTX, 1);
+      rtx reshi = expand_clmul (mode, mode, revx, revy, prec, prec);
+      reshi = expand_unop (mode, bitreverse_optab, reshi, NULL_RTX, 1);
+      if (bitint_mask)
+       reshi = expand_and (mode, reshi, bitint_mask, NULL_RTX);
+      reshi = expand_shift (RSHIFT_EXPR, mode, reshi, 1, NULL_RTX, 1);
+      write_complex_part (target, reshi, true, false);
+      return;
+    }
+  /* Fallback to modified Karatsuba multiplication step
+     using 3x expand_clmul in mode.
+     For 64x64 -> 128bit CLMUL this would be:
+     uint64_t x0 = (uint32_t) x, x1 = (x >> 32);
+     uint64_t y0 = (uint32_t) y, y1 = (y >> 32);
+     uint64_t l = clmul (x0, y0);
+     uint64_t m = clmul (x0 ^ x1, y0 ^ y1);
+     uint64_t h = clmul (x1, y1);
+     m ^= l ^ h;
+     return { l ^ (m << 32), h ^ (m >> 32) };  */
+  unsigned int hprec = prec / 2;
+  rtx mask = immed_wide_int_const (wi::mask (hprec, false, prec), mode);
+  optab_methods olw = OPTAB_LIB_WIDEN;
+  rtx x0 = expand_binop (mode, and_optab, op0, mask, NULL_RTX, 1, olw);
+  rtx x1 = expand_shift (RSHIFT_EXPR, mode, op0, hprec, NULL_RTX, 1);
+  rtx y0 = expand_binop (mode, and_optab, op1, mask, NULL_RTX, 1, olw);
+  rtx y1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec, NULL_RTX, 1);
+  rtx l = expand_clmul (mode, mode, x0, y0, hprec, hprec);
+  rtx t0 = expand_binop (mode, xor_optab, x0, x1, NULL_RTX, 1, olw);
+  rtx t1 = expand_binop (mode, xor_optab, y0, y1, NULL_RTX, 1, olw);
+  rtx m = expand_clmul (mode, mode, t0, t1, hprec, hprec);
+  rtx h;
+  if (prec0 <= hprec || prec1 <= hprec)
+    h = const0_rtx;
+  else
+    h = expand_clmul (mode, mode, x1, y1, MIN (hprec, prec0 - hprec),
+                     MIN (hprec, prec1 - hprec));
+  t0 = expand_binop (mode, xor_optab, l, h, NULL_RTX, 1, olw);
+  m = expand_binop (mode, xor_optab, m, t0, NULL_RTX, 1, olw);
+  t0 = expand_shift (LSHIFT_EXPR, mode, m, hprec, NULL_RTX, 1);
+  t1 = expand_shift (RSHIFT_EXPR, mode, m, hprec, NULL_RTX, 1);
+  rtx reslo = expand_binop (mode, xor_optab, l, t0, NULL_RTX, 1, olw);
+  if (bitint_mask)
+    {
+      rtx reslom = expand_and (mode, reslo, bitint_mask, NULL_RTX);
+      write_complex_part (target, reslom, false, true);
+      rtx reshi = expand_binop (mode, xor_optab, h, t1, NULL_RTX, 1, olw);
+      reshi = expand_shift (LSHIFT_EXPR, mode, reshi,
+                           prec - TYPE_PRECISION (TREE_TYPE (arg0)),
+                           NULL_RTX, 1);
+      reslo = expand_shift (RSHIFT_EXPR, mode, reslo,
+                           TYPE_PRECISION (TREE_TYPE (arg0)), NULL_RTX, 1);
+      reshi = expand_binop (mode, ior_optab, reshi, reslo, NULL_RTX, 1, olw);
+      reshi = expand_and (mode, reshi, bitint_mask, NULL_RTX);
+      write_complex_part (target, reshi, true, false);
+    }
+  else
+    {
+      write_complex_part (target, reslo, false, true);
+      rtx reshi = expand_binop (mode, xor_optab, h, t1, NULL_RTX, 1, olw);
+      write_complex_part (target, reshi, true, false);
+    }
+}
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 7feb236d8026..35f6daaddc67 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -549,6 +549,8 @@ DEF_INTERNAL_FN (TSAN_FUNC_EXIT, ECF_NOVOPS | ECF_LEAF | 
ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (VA_ARG, ECF_NOTHROW | ECF_LEAF, NULL)
 DEF_INTERNAL_FN (VEC_CONVERT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
 DEF_INTERNAL_FN (RAWMEMCHR, ECF_PURE | ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (CLMUL, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (CLMUL_WIDEN, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
 
 /* An unduplicable, uncombinable function.  Generally used to preserve
    a CFG property in the face of jump threading, tail merging or
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index 30930f388b06..debd84b8ff0e 100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -288,6 +288,8 @@ extern void expand_PARITY (internal_fn, gcall *);
 extern void expand_POPCOUNT (internal_fn, gcall *);
 extern void expand_BSWAP (internal_fn, gcall *);
 extern void expand_BITREVERSE (internal_fn, gcall *);
+extern void expand_CLMUL (internal_fn, gcall *);
+extern void expand_CLMUL_WIDEN (internal_fn, gcall *);
 
 extern bool vectorized_internal_fn_supported_p (internal_fn, tree);
 
diff --git a/gcc/match.pd b/gcc/match.pd
index 3e206704fc6c..c9f5abd9f9a3 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -14000,3 +14000,22 @@ and,
            && wi::le_p (wi::to_wide (@1), bound, TYPE_SIGN (type))))
     @3)))))
 #endif
+
+(simplify
+ (IFN_CLMUL:c @0 zero_one_valued_p@1)
+ (mult @0 @1))
+
+#if GIMPLE
+(simplify
+ (IFN_CLMUL_WIDEN @0 @1)
+ (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) && TYPE_UNSIGNED (TREE_TYPE (@0)))
+  (with { unsigned int min_prec0 = TYPE_PRECISION (TREE_TYPE (@0));
+         unsigned int min_prec1 = min_prec0;
+         int_range_max vr0, vr1;
+         if (gimple_match_range_of_expr (vr0, @0) && !vr0.undefined_p ())
+           min_prec0 = wi::min_precision (vr0.upper_bound (), UNSIGNED);
+         if (gimple_match_range_of_expr (vr1, @1) && !vr1.undefined_p ())
+           min_prec1 = wi::min_precision (vr1.upper_bound (), UNSIGNED); }
+   (if (min_prec0 + min_prec1 - 1 <= TYPE_PRECISION (TREE_TYPE (@0)))
+    (complex (IFN_CLMUL @0 @1) { build_zero_cst (TREE_TYPE (@0)); })))))
+#endif
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 8a5b37558b10..243ef810f4be 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -577,3 +577,5 @@ OPTAB_D (mask_len_strided_store_optab, 
"mask_len_strided_store_$a")
 OPTAB_D (andn_optab, "andn$a3")
 OPTAB_D (iorn_optab, "iorn$a3")
 OPTAB_D (aop_optab, "aop$a3")
+OPTAB_D (clmul_optab, "clmul$a3")
+OPTAB_D (clmul_widen_optab, "clmul_widen$a3")
diff --git a/gcc/testsuite/c-c++-common/builtin-clmulg-1.c 
b/gcc/testsuite/c-c++-common/builtin-clmulg-1.c
new file mode 100644
index 000000000000..af1825cd3e69
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/builtin-clmulg-1.c
@@ -0,0 +1,524 @@
+/* Test __builtin_clmulg.  */
+/* { dg-do run { target { c || c++11 } } } */
+/* { dg-options "-O2" } */
+
+#if !__has_builtin(__builtin_clmulg)
+#error __builtin_clmulg unsupported
+#endif
+
+#if __CHAR_BIT__ == 8
+static_assert (__builtin_clmulg ((unsigned char) 0x7e, (unsigned char) 0x9c) 
== 0xe8, "");
+#if __SIZEOF_SHORT__ == 2
+static_assert (__builtin_clmulg ((unsigned short) 0xc8aa, (unsigned short) 
0x7616) == 0xa55c, "");
+#endif
+#if __SIZEOF_INT__ == 4
+static_assert (__builtin_clmulg (0x9f330680U, 0x4e157eadU) == 0xb23fb880U, "");
+static_assert (__builtin_clmulg (0x11111111U, 0x11111111U) == 0x1010101U, "");
+static_assert (__builtin_clmulg (~0U, ~0U) == 0x55555555U, "");
+#endif
+#if __SIZEOF_LONG__ == 4
+static_assert (__builtin_clmulg (0xcd63e2c1UL, 0xab176a37UL) == 0x2f053277UL, 
"");
+#elif __SIZEOF_LONG__ == 8
+static_assert (__builtin_clmulg (0x43e13a77bec666afUL, 0x39c165e2ed3f98f7UL) 
== 0xe8465657c87e7a1dUL, "");
+#endif
+#if __SIZEOF_LONG_LONG__ == 8
+static_assert (__builtin_clmulg (0xe38ebed3ef3cc8deULL, 0x75224424100f8096ULL) 
== 0xfaf0ddab7a75024ULL, "");
+static_assert (__builtin_clmulg (0x11111111ULL, 0x11111111ULL) == 
0x101010101010101ULL, "");
+static_assert (__builtin_clmulg (0x1111111111111111ULL, 0x1111111111111111ULL) 
== 0x101010101010101ULL, "");
+static_assert (__builtin_clmulg (~0ULL, ~0ULL) == 0x5555555555555555ULL, "");
+#endif
+#if __SIZEOF_INT128__ == 16
+static_assert (__builtin_clmulg ((((unsigned __int128) 0x983a442a106b8a83ULL) 
<< 64)
+                                | 0x1866ccfa2e2b382cULL,
+                                (((unsigned __int128) 0xe9abf4d856d535c3ULL) 
<< 64)
+                                | 0x58708f117a52c14cULL)
+                                == ((((unsigned __int128) 
0xd3ae65b0bac566b2ULL) << 64)
+                                    | 0xdd6b33f9bb8606d0ULL), "");
+#endif
+#endif
+
+#if __BITINT_MAXWIDTH__ >= 856
+static_assert (__builtin_clmulg ((unsigned _BitInt(24)) 0xea754cuwb,
+                                (unsigned _BitInt(24)) 0x7cfcb9uwb) == 
0xd946cuwb, "");
+static_assert (__builtin_clmulg ((unsigned _BitInt(40)) 0xb35cea0986uwb,
+                                (unsigned _BitInt(40)) 0xec8cec0cffuwb) == 
0xcf45a35282uwb, "");
+static_assert (__builtin_clmulg ((unsigned _BitInt(128)) 
0x40a6e26b30f2ff0e2b099e7a71429d54uwb,
+                                (unsigned _BitInt(128)) 
0xddd32528056500d0b79218aa8b55046cuwb)
+              == 0xa11e12e273ecbf7d7f0c6f8409db3070uwb, "");
+static_assert (__builtin_clmulg ((unsigned _BitInt(129)) 
0x01a057b67e5807da3e87d86e34ead6d446uwb,
+                                (unsigned _BitInt(129)) 
0x00ebfe8616a19a74e093c39217b1f599c5uwb)
+              == 0x7aa05ee8e53876da9b60fbe653da1deuwb, "");
+static_assert (__builtin_clmulg ((unsigned _BitInt(136)) 
0xe8654afad921062835d589e9de710c59e2uwb,
+                                (unsigned _BitInt(136)) 
0xec973d421e8e06fda1fc1ec83be74e7ac5uwb)
+              == 0xcaba6f2d4948b5e794137742285f7d83eauwb, "");
+static_assert (__builtin_clmulg ((unsigned _BitInt(832)) 
0x3bf0e6fcd0acb795c68d436f4038d14f1eeadd2447157b04d15ad94ec0638ffb5375f72321afb8e73cfb577d3328cc5213a9765abff15e904c37de0c9b6d08eee2ff1204aecaebebc64268f96b344b7edec2d89db3362dff6e0b0c097814f75a13095ec2d44aad9auwb,
+                                (unsigned _BitInt(832)) 
0x527f6addf01c1827f833ac6222d6d1c2b114862834006297b3a44b8c445a2b97da9574cab18cf1a9c09d0be273dca424f12b4c252baebdde52086b9763962e3d2ba207dd2ef886ee9592d1086e752c5fa07885cc2742aa794a1510adab3eead7e0f1b40fe93afd7euwb)
+              == 
0xed1958f12923520d74fc66cea0973580825a5e563806308c9740731d7873df8d1842a1e78f7c2d957aa122477042739466df964d826d3ae2b77b0d153ba77375b4d53f85667ee9e99bc6e4f20e15ede19ea949c1c3783205003ac0dca600daa9cfc51777640ccfecuwb,
 "");
+static_assert (__builtin_clmulg ((unsigned _BitInt(856)) 
0x7118148ab2f165085830aa8b252e2d1600b3264181b7b611c72bce4e05cb0276e316019607669e609748ebbc7618d276ccf8b84db06e5f77992dc59ef8c715dcdd1672e57c1045135830d0ce49a245159bfd624b6bc1c204ef88a2e74fb7c32dcd35124a45575d9d872d6cuwb,
+                                (unsigned _BitInt(856)) 
0x04c996a2400c474c71d73fb208c9f256676d0beebc3dd80e1dfce8d3cb7613d03faa7280b6b9cc27900bd998d4cbee3b38fa2af537020355feeb28ca613b9aa1e50c219bc5edc255f89bedcc67db079fd531940d33976232838afce4c59685aba2a646679309bc8ba4a957uwb)
+              == 
0xa969732632882314035c08bb15c392653af107f06b90b4e63580f503c7be7163bb379407420d9b72f0147d9a2c4008ed148b88a6c4304c923ec6e148484572893b3474829536e0a44a0400d10764cb7538ef2c24cd7d18374bbead68028cca6cc3433c6ce6f679de41c3c4uwb,
 "");
+#endif
+
+[[gnu::noipa]] unsigned char
+clmul8 (unsigned char x, unsigned char y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned char
+clmul8w (unsigned char x, unsigned char y, unsigned char *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned char
+clmul8b (bool x, bool y)
+{
+  return __builtin_clmulg ((unsigned char) x, (unsigned char) y);
+}
+
+[[gnu::noipa]] unsigned char
+clmul8m (unsigned char x, unsigned char y)
+{
+  return __builtin_clmulg (x, (unsigned char) (y & 0x3f));
+}
+
+[[gnu::noipa]] unsigned short
+clmul16 (unsigned short x, unsigned short y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned short
+clmul16w (unsigned short x, unsigned short y, unsigned short *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned int
+clmul32 (unsigned int x, unsigned int y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned int
+clmul32w (unsigned int x, unsigned int y, unsigned int *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned int
+clmul32m (unsigned int x, unsigned int y)
+{
+  return __builtin_clmulg (x & 0x1fffffU, y);
+}
+
+[[gnu::noipa]] unsigned int
+clmul32m2 (unsigned int x, unsigned int y)
+{
+  return __builtin_clmulg (x & 0x1fU, y & 0xf);
+}
+
+[[gnu::noipa]] unsigned int
+clmul32m3 (unsigned int x, unsigned int y)
+{
+  return __builtin_clmulg (x & 0x7fffU, y & 0x3ffU);
+}
+
+[[gnu::noipa]] unsigned long
+clmull (unsigned long x, unsigned long y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned long
+clmullw (unsigned long x, unsigned long y, unsigned long *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned long long
+clmul64 (unsigned long long x, unsigned long long y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned long long
+clmul64w (unsigned long long x, unsigned long long y, unsigned long long *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned long long
+clmul64wm (unsigned long long x, unsigned long long y, unsigned long long *z)
+{
+  return __builtin_clmulg (x & 0x3fffffffULL, y & 0x3fffffffULL, z);
+}
+
+#if __CHAR_BIT__ == 8 && __SIZEOF_INT128__ == 16
+[[gnu::noipa]] unsigned __int128
+clmul128 (unsigned __int128 x, unsigned __int128 y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned __int128
+clmul128w (unsigned __int128 x, unsigned __int128 y, unsigned __int128 *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+#endif
+
+#if __BITINT_MAXWIDTH__ >= 856
+[[gnu::noipa]] unsigned _BitInt(1)
+clmul1 (unsigned _BitInt(1) x, unsigned _BitInt(1) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(1)
+clmul1w (unsigned _BitInt(1) x, unsigned _BitInt(1) y, unsigned _BitInt(1) *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned _BitInt(24)
+clmul24 (unsigned _BitInt(24) x, unsigned _BitInt(24) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(24)
+clmul24w (unsigned _BitInt(24) x, unsigned _BitInt(24) y, unsigned _BitInt(24) 
*z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned _BitInt(40)
+clmul40 (unsigned _BitInt(40) x, unsigned _BitInt(40) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(40)
+clmul40w (unsigned _BitInt(40) x, unsigned _BitInt(40) y, unsigned _BitInt(40) 
*z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned _BitInt(128)
+clmul128b (unsigned _BitInt(128) x, unsigned _BitInt(128) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(128)
+clmul128bw (unsigned _BitInt(128) x, unsigned _BitInt(128) y, unsigned 
_BitInt(128) *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned _BitInt(129)
+clmul129 (unsigned _BitInt(129) x, unsigned _BitInt(129) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(129)
+clmul129w (unsigned _BitInt(129) x, unsigned _BitInt(129) y, unsigned 
_BitInt(129) *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned _BitInt(136)
+clmul136 (unsigned _BitInt(136) x, unsigned _BitInt(136) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(136)
+clmul136w (unsigned _BitInt(136) x, unsigned _BitInt(136) y, unsigned 
_BitInt(136) *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned _BitInt(832)
+clmul832 (unsigned _BitInt(832) x, unsigned _BitInt(832) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(832)
+clmul832w (unsigned _BitInt(832) x, unsigned _BitInt(832) y, unsigned 
_BitInt(832) *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+
+[[gnu::noipa]] unsigned _BitInt(856)
+clmul856 (unsigned _BitInt(856) x, unsigned _BitInt(856) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(856)
+clmul856w (unsigned _BitInt(856) x, unsigned _BitInt(856) y, unsigned 
_BitInt(856) *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+#endif
+
+#if __BITINT_MAXWIDTH__ >= 65535
+[[gnu::noipa]] unsigned _BitInt(65535)
+clmul65535 (unsigned _BitInt(65535) x, unsigned _BitInt(65535) y)
+{
+  return __builtin_clmulg (x, y);
+}
+
+[[gnu::noipa]] unsigned _BitInt(65535)
+clmul65535w (unsigned _BitInt(65535) x, unsigned _BitInt(65535) y, unsigned 
_BitInt(65535) *z)
+{
+  return __builtin_clmulg (x, y, z);
+}
+#endif
+
+int
+main ()
+{
+#if __CHAR_BIT__ == 8
+  if (clmul8 ((unsigned char) 0x7e, (unsigned char) 0x9c) != 0xe8)
+    __builtin_abort ();
+  if (clmul8b (false, false) != 0
+      || clmul8b (false, true) != 0
+      || clmul8b (true, false) != 0
+      || clmul8b (true, true) != 1)
+    __builtin_abort ();
+  if (clmul8m ((unsigned char) 0x7e, (unsigned char) 0x9c)
+      != __builtin_clmulg ((unsigned char) 0x7e, (unsigned char) (0x9c & 
0x3f)))
+    __builtin_abort ();
+  if (clmul8m ((unsigned char) 0xff, (unsigned char) 0xff)
+      != __builtin_clmulg ((unsigned char) 0xff, (unsigned char) (0xff & 
0x3f)))
+    __builtin_abort ();
+  {
+    unsigned char hi;
+    if (clmul8w ((unsigned char) 0x7e, (unsigned char) 0x9c, &hi) != 0xe8
+       || hi != 0x3a)
+      __builtin_abort ();
+  }
+#if __SIZEOF_SHORT__ == 2
+  if (clmul16 ((unsigned short) 0xc8aa, (unsigned short) 0x7616) != 0xa55c)
+    __builtin_abort ();
+  {
+    unsigned short hi;
+    if (clmul16w ((unsigned short) 0xc8aa, (unsigned short) 0x7616, &hi) != 
0xa55c
+       || hi != 0x2508)
+      __builtin_abort ();
+  }
+#endif
+#if __SIZEOF_INT__ == 4
+  if (clmul32 (0x9f330680U, 0x4e157eadU) != 0xb23fb880U)
+    __builtin_abort ();
+  if (clmul32 (0x11111111U, 0x11111111U) != 0x1010101U)
+    __builtin_abort ();
+  if (clmul32 (~0U, ~0U) != 0x55555555U)
+    __builtin_abort ();
+  if (clmul32m (0x9f330680U, 0x4e157eadU)
+      != __builtin_clmulg (0x9f330680U & 0x1fffffU, 0x4e157eadU))
+    __builtin_abort ();
+  if (clmul32m2 (0x9f330680U, 0x4e157eadU)
+      != __builtin_clmulg (0x9f330680U & 0x1fU, 0x4e157eadU & 0xfU))
+    __builtin_abort ();
+  if (clmul32m3 (0x9f330680U, 0x4e157eadU)
+      != __builtin_clmulg (0x9f330680U & 0x7fffU, 0x4e157eadU & 0x3ffU))
+    __builtin_abort ();
+  if (clmul32m (~0U, ~0U) != __builtin_clmulg (0x1fffffU, ~0U))
+    __builtin_abort ();
+  if (clmul32m2 (~0U, ~0U) != __builtin_clmulg (0x1fU, 0xfU))
+    __builtin_abort ();
+  if (clmul32m3 (~0U, ~0U) != __builtin_clmulg (0x7fffU, 0x3ffU))
+    __builtin_abort ();
+  {
+    unsigned int hi;
+    if (clmul32w (0x9f330680U, 0x4e157eadU, &hi) != 0xb23fb880U
+       || hi != 0x207cd974)
+      __builtin_abort ();
+    if (clmul32w (0x11111111U, 0x11111111U, &hi) != 0x1010101U
+       || hi != 0x1010101U)
+      __builtin_abort ();
+    if (clmul32w (~0U, ~0U, &hi) != 0x55555555U
+       || hi != 0x55555555U)
+      __builtin_abort ();
+  }
+#endif
+#if __SIZEOF_LONG__ == 4
+  if (clmull (0xcd63e2c1UL, 0xab176a37UL) != 0x2f053277UL)
+    __builtin_abort ();
+  {
+    unsigned long hi;
+    if (clmullw (0xcd63e2c1UL, 0xab176a37UL, &hi) != 0x2f053277UL
+       || hi != 0x782faf0eUL)
+      __builtin_abort ();
+  }
+#elif __SIZEOF_LONG__ == 8
+  if (clmull (0x43e13a77bec666afUL, 0x39c165e2ed3f98f7UL) != 
0xe8465657c87e7a1dUL)
+    __builtin_abort ();
+  {
+    unsigned long hi;
+    if (clmullw (0x43e13a77bec666afUL, 0x39c165e2ed3f98f7UL, &hi) != 
0xe8465657c87e7a1dUL
+       || hi != 0xe2f8ec31f25371bUL)
+      __builtin_abort ();
+  }
+#endif
+#if __SIZEOF_LONG_LONG__ == 8
+  if (clmul64 (0xe38ebed3ef3cc8deULL, 0x75224424100f8096ULL) != 
0xfaf0ddab7a75024ULL)
+    __builtin_abort ();
+  if (clmul64 (0x11111111ULL, 0x11111111ULL) != 0x101010101010101ULL)
+    __builtin_abort ();
+  if (clmul64 (0x1111111111111111ULL, 0x1111111111111111ULL) != 
0x101010101010101ULL)
+    __builtin_abort ();
+  if (clmul64 (~0ULL, ~0ULL) != 0x5555555555555555ULL)
+    __builtin_abort ();
+  {
+    unsigned long long hi;
+    if (clmul64w (0xe38ebed3ef3cc8deULL, 0x75224424100f8096ULL, &hi) != 
0xfaf0ddab7a75024ULL
+       || hi != 0x29daaa7ee7cd2860ULL)
+      __builtin_abort ();
+    if (clmul64w (0x11111111ULL, 0x11111111ULL, &hi) != 0x101010101010101ULL
+       || hi != 0ULL)
+      __builtin_abort ();
+    if (clmul64w (0x1111111111111111ULL, 0x1111111111111111ULL, &hi) != 
0x101010101010101ULL
+       || hi != 0x101010101010101ULL)
+      __builtin_abort ();
+    if (clmul64w (~0ULL, ~0ULL, &hi) != 0x5555555555555555ULL
+       || hi != 0x5555555555555555ULL)
+      __builtin_abort ();
+    if (clmul64wm (0xe38ebed3ef3cc8deULL, 0x75224424100f8096ULL, &hi)
+       != __builtin_clmulg (0xe38ebed3ef3cc8deULL & 0x3fffffffULL,
+                            0x75224424100f8096ULL & 0x3fffffffULL)
+       || hi != 0ULL)
+      __builtin_abort ();
+    if (clmul64wm (~0ULL, ~0ULL, &hi) != __builtin_clmulg (0x3fffffffULL, 
0x3fffffffULL)
+       || hi != 0ULL)
+      __builtin_abort ();
+  }
+#endif
+#if __SIZEOF_INT128__ == 16
+  if (clmul128 ((((unsigned __int128) 0x983a442a106b8a83ULL) << 64)
+               | 0x1866ccfa2e2b382cULL,
+               (((unsigned __int128) 0xe9abf4d856d535c3ULL) << 64)
+               | 0x58708f117a52c14cULL)
+      != ((((unsigned __int128) 0xd3ae65b0bac566b2ULL) << 64)
+         | 0xdd6b33f9bb8606d0ULL))
+    __builtin_abort ();
+  {
+    unsigned __int128 hi;
+    if (clmul128w ((((unsigned __int128) 0x983a442a106b8a83ULL) << 64)
+                  | 0x1866ccfa2e2b382cULL,
+                  (((unsigned __int128) 0xe9abf4d856d535c3ULL) << 64)
+                  | 0x58708f117a52c14cULL, &hi)
+       != ((((unsigned __int128) 0xd3ae65b0bac566b2ULL) << 64)
+           | 0xdd6b33f9bb8606d0ULL)
+       || hi != ((((unsigned __int128) 0x7d17129902648920ULL) << 64)
+                 | 0xfdb34e3b7c058047ULL))
+    __builtin_abort ();
+  }
+#endif
+#endif
+#if __BITINT_MAXWIDTH__ >= 856
+  if (clmul1 (0, 0) != 0
+      || clmul1 (0, 1) != 0
+      || clmul1 (1, 0) != 0
+      || clmul1 (1, 1) != 1)
+    __builtin_abort ();
+  if (clmul24 (0xea754cuwb, 0x7cfcb9uwb) != 0xd946cuwb)
+    __builtin_abort ();
+  if (clmul40 (0xb35cea0986uwb, 0xec8cec0cffuwb) != 0xcf45a35282uwb)
+    __builtin_abort ();
+  if (clmul128b (0x40a6e26b30f2ff0e2b099e7a71429d54uwb, 
0xddd32528056500d0b79218aa8b55046cuwb)
+      != 0xa11e12e273ecbf7d7f0c6f8409db3070uwb)
+    __builtin_abort ();
+  if (clmul129 (0x01a057b67e5807da3e87d86e34ead6d446uwb, 
0x00ebfe8616a19a74e093c39217b1f599c5uwb)
+      != 0x7aa05ee8e53876da9b60fbe653da1deuwb)
+    __builtin_abort ();
+  if (clmul136 (0xe8654afad921062835d589e9de710c59e2uwb, 
0xec973d421e8e06fda1fc1ec83be74e7ac5uwb)
+      != 0xcaba6f2d4948b5e794137742285f7d83eauwb)
+    __builtin_abort ();
+  if (clmul832 
(0x3bf0e6fcd0acb795c68d436f4038d14f1eeadd2447157b04d15ad94ec0638ffb5375f72321afb8e73cfb577d3328cc5213a9765abff15e904c37de0c9b6d08eee2ff1204aecaebebc64268f96b344b7edec2d89db3362dff6e0b0c097814f75a13095ec2d44aad9auwb,
+               
0x527f6addf01c1827f833ac6222d6d1c2b114862834006297b3a44b8c445a2b97da9574cab18cf1a9c09d0be273dca424f12b4c252baebdde52086b9763962e3d2ba207dd2ef886ee9592d1086e752c5fa07885cc2742aa794a1510adab3eead7e0f1b40fe93afd7euwb)
+      != 
0xed1958f12923520d74fc66cea0973580825a5e563806308c9740731d7873df8d1842a1e78f7c2d957aa122477042739466df964d826d3ae2b77b0d153ba77375b4d53f85667ee9e99bc6e4f20e15ede19ea949c1c3783205003ac0dca600daa9cfc51777640ccfecuwb)
+    __builtin_abort ();
+  if (clmul856 
(0x7118148ab2f165085830aa8b252e2d1600b3264181b7b611c72bce4e05cb0276e316019607669e609748ebbc7618d276ccf8b84db06e5f77992dc59ef8c715dcdd1672e57c1045135830d0ce49a245159bfd624b6bc1c204ef88a2e74fb7c32dcd35124a45575d9d872d6cuwb,
+               
0x04c996a2400c474c71d73fb208c9f256676d0beebc3dd80e1dfce8d3cb7613d03faa7280b6b9cc27900bd998d4cbee3b38fa2af537020355feeb28ca613b9aa1e50c219bc5edc255f89bedcc67db079fd531940d33976232838afce4c59685aba2a646679309bc8ba4a957uwb)
+      != 
0xa969732632882314035c08bb15c392653af107f06b90b4e63580f503c7be7163bb379407420d9b72f0147d9a2c4008ed148b88a6c4304c923ec6e148484572893b3474829536e0a44a0400d10764cb7538ef2c24cd7d18374bbead68028cca6cc3433c6ce6f679de41c3c4uwb)
+    __builtin_abort ();
+#endif
+#if __BITINT_MAXWIDTH__ >= 856 * 2
+  {
+    unsigned _BitInt(1) hi;
+    if (clmul1w (0, 0, &hi) != 0
+       || hi != 0
+       || clmul1w (0, 1, &hi) != 0
+       || hi != 0
+       || clmul1w (1, 0, &hi) != 0
+       || hi != 0
+       || clmul1w (1, 1, &hi) != 1
+       || hi != 0)
+      __builtin_abort ();
+  }
+  {
+    unsigned _BitInt(24) hi;
+    if (clmul24w (0xea754cuwb, 0x7cfcb9uwb, &hi) != 0xd946cuwb
+       || hi != __builtin_clmulg ((unsigned _BitInt(24 * 2)) 0xea754cuwb,
+                                  (unsigned _BitInt(24 * 2)) 0x7cfcb9uwb) >> 
24)
+      __builtin_abort ();
+  }
+  {
+    unsigned _BitInt(40) hi;
+    if (clmul40w (0xb35cea0986uwb, 0xec8cec0cffuwb, &hi) != 0xcf45a35282uwb
+       || hi != __builtin_clmulg ((unsigned _BitInt(40 * 2)) 0xb35cea0986uwb,
+                                  (unsigned _BitInt(40 * 2)) 0xec8cec0cffuwb) 
>> 40)
+      __builtin_abort ();
+  }
+  {
+    unsigned _BitInt(128) hi;
+    if (clmul128bw (0x40a6e26b30f2ff0e2b099e7a71429d54uwb, 
0xddd32528056500d0b79218aa8b55046cuwb, &hi)
+       != 0xa11e12e273ecbf7d7f0c6f8409db3070uwb
+       || hi != __builtin_clmulg ((unsigned _BitInt(128 * 2)) 
0x40a6e26b30f2ff0e2b099e7a71429d54uwb,
+                                  (unsigned _BitInt(128 * 2)) 
0xddd32528056500d0b79218aa8b55046cuwb) >> 128)
+      __builtin_abort ();
+  }
+  {
+    unsigned _BitInt(129) hi;
+    if (clmul129w (0x01a057b67e5807da3e87d86e34ead6d446uwb, 
0x00ebfe8616a19a74e093c39217b1f599c5uwb, &hi)
+       != 0x7aa05ee8e53876da9b60fbe653da1deuwb
+       || hi != __builtin_clmulg ((unsigned _BitInt(129 * 2)) 
0x01a057b67e5807da3e87d86e34ead6d446uwb,
+                                  (unsigned _BitInt(129 * 2)) 
0x00ebfe8616a19a74e093c39217b1f599c5uwb) >> 129)
+      __builtin_abort ();
+  }
+  {
+    unsigned _BitInt(136) hi;
+    if (clmul136w (0xe8654afad921062835d589e9de710c59e2uwb, 
0xec973d421e8e06fda1fc1ec83be74e7ac5uwb, &hi)
+       != 0xcaba6f2d4948b5e794137742285f7d83eauwb
+       || hi != __builtin_clmulg ((unsigned _BitInt(136 * 2)) 
0xe8654afad921062835d589e9de710c59e2uwb,
+                                  (unsigned _BitInt(136 * 2)) 
0xec973d421e8e06fda1fc1ec83be74e7ac5uwb) >> 136)
+      __builtin_abort ();
+  }
+  {
+    unsigned _BitInt(832) hi;
+    if (clmul832w 
(0x3bf0e6fcd0acb795c68d436f4038d14f1eeadd2447157b04d15ad94ec0638ffb5375f72321afb8e73cfb577d3328cc5213a9765abff15e904c37de0c9b6d08eee2ff1204aecaebebc64268f96b344b7edec2d89db3362dff6e0b0c097814f75a13095ec2d44aad9auwb,
+                  
0x527f6addf01c1827f833ac6222d6d1c2b114862834006297b3a44b8c445a2b97da9574cab18cf1a9c09d0be273dca424f12b4c252baebdde52086b9763962e3d2ba207dd2ef886ee9592d1086e752c5fa07885cc2742aa794a1510adab3eead7e0f1b40fe93afd7euwb,
 &hi)
+       != 
0xed1958f12923520d74fc66cea0973580825a5e563806308c9740731d7873df8d1842a1e78f7c2d957aa122477042739466df964d826d3ae2b77b0d153ba77375b4d53f85667ee9e99bc6e4f20e15ede19ea949c1c3783205003ac0dca600daa9cfc51777640ccfecuwb
+       || hi != __builtin_clmulg ((unsigned _BitInt(832 * 2)) 
0x3bf0e6fcd0acb795c68d436f4038d14f1eeadd2447157b04d15ad94ec0638ffb5375f72321afb8e73cfb577d3328cc5213a9765abff15e904c37de0c9b6d08eee2ff1204aecaebebc64268f96b344b7edec2d89db3362dff6e0b0c097814f75a13095ec2d44aad9auwb,
+                                  (unsigned _BitInt(832 * 2)) 
0x527f6addf01c1827f833ac6222d6d1c2b114862834006297b3a44b8c445a2b97da9574cab18cf1a9c09d0be273dca424f12b4c252baebdde52086b9763962e3d2ba207dd2ef886ee9592d1086e752c5fa07885cc2742aa794a1510adab3eead7e0f1b40fe93afd7euwb)
 >> 832)
+      __builtin_abort ();
+  }
+  {
+    unsigned _BitInt(856) hi;
+    if (clmul856w 
(0x7118148ab2f165085830aa8b252e2d1600b3264181b7b611c72bce4e05cb0276e316019607669e609748ebbc7618d276ccf8b84db06e5f77992dc59ef8c715dcdd1672e57c1045135830d0ce49a245159bfd624b6bc1c204ef88a2e74fb7c32dcd35124a45575d9d872d6cuwb,
+                  
0x04c996a2400c474c71d73fb208c9f256676d0beebc3dd80e1dfce8d3cb7613d03faa7280b6b9cc27900bd998d4cbee3b38fa2af537020355feeb28ca613b9aa1e50c219bc5edc255f89bedcc67db079fd531940d33976232838afce4c59685aba2a646679309bc8ba4a957uwb,
 &hi)
+       != 
0xa969732632882314035c08bb15c392653af107f06b90b4e63580f503c7be7163bb379407420d9b72f0147d9a2c4008ed148b88a6c4304c923ec6e148484572893b3474829536e0a44a0400d10764cb7538ef2c24cd7d18374bbead68028cca6cc3433c6ce6f679de41c3c4uwb
+       || hi != __builtin_clmulg ((unsigned _BitInt(856 * 2)) 
0x7118148ab2f165085830aa8b252e2d1600b3264181b7b611c72bce4e05cb0276e316019607669e609748ebbc7618d276ccf8b84db06e5f77992dc59ef8c715dcdd1672e57c1045135830d0ce49a245159bfd624b6bc1c204ef88a2e74fb7c32dcd35124a45575d9d872d6cuwb,
+                                  (unsigned _BitInt(856 * 2)) 
0x04c996a2400c474c71d73fb208c9f256676d0beebc3dd80e1dfce8d3cb7613d03faa7280b6b9cc27900bd998d4cbee3b38fa2af537020355feeb28ca613b9aa1e50c219bc5edc255f89bedcc67db079fd531940d33976232838afce4c59685aba2a646679309bc8ba4a957uwb)
 >> 856)
+      __builtin_abort ();
+  }
+#endif
+}
diff --git a/gcc/testsuite/c-c++-common/builtin-clmulg-2.c 
b/gcc/testsuite/c-c++-common/builtin-clmulg-2.c
new file mode 100644
index 000000000000..6c451b42a7cd
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/builtin-clmulg-2.c
@@ -0,0 +1,55 @@
+/* Test __builtin_clmulg.  */
+/* { dg-do compile } */
+/* { dg-options "" } */
+
+int
+main ()
+{
+  unsigned int a;
+  __builtin_clmulg ();                 /* { dg-error "wrong number of 
arguments to '__builtin_clmulg'" } */
+  __builtin_clmulg (1U);               /* { dg-error "wrong number of 
arguments to '__builtin_clmulg'" } */
+  __builtin_clmulg (1U, 1U, &a, 1U);   /* { dg-error "wrong number of 
arguments to '__builtin_clmulg'" } */
+  __builtin_clmulg (1.0, 1.0);         /* { dg-error "'__builtin_clmulg' 
operand not an integral type" } */
+  __builtin_clmulg (1.0f, 1.0f);       /* { dg-error "'__builtin_clmulg' 
operand not an integral type" } */
+  __builtin_clmulg (1.0L, 1.0L);       /* { dg-error "'__builtin_clmulg' 
operand not an integral type" } */
+  struct S { int a; };
+  __builtin_clmulg ((struct S) { 42 }, (struct S) { 43 }); /* { dg-error 
"'__builtin_clmulg' operand not an integral type" } */
+  enum E { E0, E1 } e;
+  __builtin_clmulg (E1, E1);           /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has signed type" "" { target c } } */
+                                       /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has enumerated type" "" { target c++ } .-1 } */
+  __builtin_clmulg (e, e);             /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has enumerated type" } */
+  __builtin_clmulg ((enum E) 1, (enum E) 1); /* { dg-error "argument 1 in call 
to function '__builtin_clmulg' has enumerated type" } */
+#if !defined(__cplusplus) || __cplusplus >=  201103L
+  enum F : int { F0, F1 } f;
+  __builtin_clmulg (F1, F1);           /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has enumerated type" "" { target { c || c++11 } } } 
*/
+  __builtin_clmulg (f, f);             /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has enumerated type" "" { target { c || c++11 } } } 
*/
+  __builtin_clmulg ((enum F) 1, (enum F) 1); /* { dg-error "argument 1 in call 
to function '__builtin_clmulg' has enumerated type" "" { target { c || c++11 } 
} } */
+#endif
+  __builtin_clmulg (false, true);      /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has boolean type" } */
+  __builtin_clmulg (1, 1);             /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has signed type" } */
+  __builtin_clmulg ((char) 1, (char) 1);/* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has signed type" "" { target { ! unsigned_char } } 
} */
+  __builtin_clmulg (1UL, 1ULL);                /* { dg-error "arguments 1 and 
2 to function '__builtin_clmulg' have different types 'long unsigned int' and 
'long long unsigned int'" } */
+  __builtin_clmulg (1U, 1.0);          /* { dg-error "arguments 1 and 2 to 
function '__builtin_clmulg' have different types 'unsigned int' and 'double'" } 
*/
+  __builtin_clmulg ((unsigned char) 1, (char) 1); /* { dg-error "arguments 1 
and 2 to function '__builtin_clmulg' have different types 'unsigned char' and 
'char'" } */
+  __builtin_clmulg ((unsigned char) 1, true); /* { dg-error "arguments 1 and 2 
to function '__builtin_clmulg' have different types 'unsigned char' and 'bool'" 
} */
+  __builtin_clmulg (1UL, 1UL, 1UL);    /* { dg-error "argument 3 to function 
'__builtin_clmulg' does not have pointer type" } */
+  __builtin_clmulg (1UL, 1UL, &a);     /* { dg-error "arguments 1 and 3 to 
function '__builtin_clmulg' have incompatible types 'long unsigned int' and 
'unsigned int ?\\\*'" } */
+  const unsigned int b = 42;
+  __builtin_clmulg (1U, 1U, &b);       /* { dg-error "argument 3 to function 
'__builtin_clmulg' has pointer to 'const' type 'const unsigned int ?\\\*'" } */
+  bool c = false;
+  int d = 1;
+  char g = ' ';
+  unsigned long h = 42;
+  unsigned char i = 1;
+  __builtin_clmulg (false, true, &c);  /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has boolean type" } */
+  __builtin_clmulg (1, 1, &d);         /* { dg-error "argument 1 in call to 
function '__builtin_clmulg' has signed type" } */
+  __builtin_clmulg ((char) 1, (char) 1, &g); /* { dg-error "argument 1 in call 
to function '__builtin_clmulg' has signed type" "" { target { ! unsigned_char } 
} } */
+  __builtin_clmulg (1UL, 1ULL, &h);    /* { dg-error "arguments 1 and 2 to 
function '__builtin_clmulg' have different types 'long unsigned int' and 'long 
long unsigned int'" } */
+  __builtin_clmulg (1U, 1.0, &a);      /* { dg-error "arguments 1 and 2 to 
function '__builtin_clmulg' have different types 'unsigned int' and 'double'" } 
*/
+  __builtin_clmulg ((unsigned char) 1, (char) 1, &i); /* { dg-error "arguments 
1 and 2 to function '__builtin_clmulg' have different types 'unsigned char' and 
'char'" } */
+  __builtin_clmulg ((unsigned char) 1, true, &i); /* { dg-error "arguments 1 
and 2 to function '__builtin_clmulg' have different types 'unsigned char' and 
'bool'" } */
+#ifndef __cplusplus
+  _Atomic unsigned int j = 42;
+  __builtin_clmulg (1U, 2U, &j);       /* { dg-error "argument 3 to function 
'__builtin_clmulg' has pointer to '_Atomic' type '_Atomic unsigned int \\\*'" 
"" { target c } } */
+#endif
+}
diff --git a/gcc/testsuite/g++.dg/ext/builtin-clmulg-1.C 
b/gcc/testsuite/g++.dg/ext/builtin-clmulg-1.C
new file mode 100644
index 000000000000..e073fd368e8a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/builtin-clmulg-1.C
@@ -0,0 +1,24 @@
+// { dg-do compile { target c++14 } }
+
+constexpr unsigned long long
+foo (unsigned int x, unsigned int y)
+{
+  unsigned int a = 0;
+  unsigned int b = __builtin_clmulg (x, y, &a);
+  return (((unsigned long long) a) << ((__SIZEOF_LONG_LONG__ * __CHAR_BIT__) / 
2)) | b;
+}
+
+constexpr unsigned long long
+bar (unsigned int x, unsigned int y)
+{
+  return __builtin_clmulg ((unsigned long long) x, (unsigned long long) y);
+}
+
+#if __SIZEOF_INT__ == 4 && __SIZEOF_LONG_LONG__ == 8 && __CHAR_BIT__ == 8
+static_assert (foo (0x10111111U, 0x22002222U) == 0x222022020000202ULL, "");
+static_assert (bar (0x10111111U, 0x22002222U) == 0x222022020000202ULL, "");
+#endif
+static_assert (foo (0, 0) == 0, "");
+static_assert (bar (0, 0) == 0, "");
+static_assert (foo (6, 4) == 0x18, "");
+static_assert (bar (6, 4) == 0x18, "");
diff --git a/gcc/testsuite/gcc.target/i386/pclmul-builtin-clmulg-1.c 
b/gcc/testsuite/gcc.target/i386/pclmul-builtin-clmulg-1.c
new file mode 100644
index 000000000000..b41f0997d1e5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pclmul-builtin-clmulg-1.c
@@ -0,0 +1,19 @@
+/* { dg-do run } */
+/* { dg-require-effective-target pclmul } */
+/* { dg-options "-O2 -mpclmul" } */
+
+#ifndef CHECK_H
+#define CHECK_H "pclmul-check.h"
+#endif
+
+#include CHECK_H
+
+#define main do_main
+
+#include "../../c-c++-common/builtin-clmulg-1.c"
+
+static void
+pclmul_test (void)
+{
+  do_main ();
+}
diff --git a/gcc/wide-int.cc b/gcc/wide-int.cc
index 668b2e4ea7db..bf979c160031 100644
--- a/gcc/wide-int.cc
+++ b/gcc/wide-int.cc
@@ -2400,6 +2400,72 @@ wi::mod_inv (const wide_int &a, const wide_int &b)
   return x1;
 }
 
+/* Return as many repetition of X.get_precision () bits from X
+   as fit into PREC.  */
+wide_int
+wi::bit_repeat (const wide_int_ref &x, unsigned int prec)
+{
+  unsigned int xprec = x.get_precision ();
+  if (xprec >= prec)
+    return wide_int::from (x, prec, UNSIGNED);
+  if (xprec <= HOST_BITS_PER_WIDE_INT && (HOST_BITS_PER_WIDE_INT % xprec) == 0)
+    {
+      unsigned HOST_WIDE_INT val = x.to_uhwi (xprec);
+      while (xprec < HOST_BITS_PER_WIDE_INT)
+       {
+         val |= val << xprec;
+         xprec <<= 1;
+       }
+      wide_int ret = wide_int::create (prec);
+      HOST_WIDE_INT *v = ret.write_val (0);
+      unsigned int i = 0;
+      for (unsigned int p = 0; p < prec; p += HOST_BITS_PER_WIDE_INT)
+       v[i++] = val;
+      ret.set_len (i, false);
+      return ret;
+    }
+  wide_int ret = wide_int::from (x, prec, UNSIGNED), b;
+  while (xprec < prec)
+    {
+      ret |= (ret << xprec);
+      xprec <<= 1;
+    }
+  return ret;
+}
+
+/* Compute carry-less product of X and Y with precision PREC.  */
+
+wide_int
+wi::clmul (const wide_int_ref &x, const wide_int_ref &y, unsigned int prec)
+{
+  wide_int a;
+  const wide_int_ref *b;
+  if (wi::popcount (x) >= wi::popcount (y))
+    {
+      a = wide_int::from (x, prec, UNSIGNED);
+      b = &y;
+    }
+  else
+    {
+      a = wide_int::from (y, prec, UNSIGNED);
+      b = &x;
+    }
+  unsigned int bprec = b->get_precision ();
+  wide_int r = wi::zero (prec);
+  unsigned int j = 0, k = 0;
+  for (unsigned int i = 0; i < bprec; ++i)
+    {
+      if (((((unsigned HOST_WIDE_INT) b->elt (k)) >> j) & 1) != 0)
+       r ^= a << i;
+      if (++j == HOST_BITS_PER_WIDE_INT)
+       {
+         j = 0;
+         ++k;
+       }
+    }
+  return r;
+}
+
 /*
  * Private utilities.
  */
diff --git a/gcc/wide-int.h b/gcc/wide-int.h
index 6c39ce176985..59b2796ed95e 100644
--- a/gcc/wide-int.h
+++ b/gcc/wide-int.h
@@ -3982,6 +3982,9 @@ namespace wi
                             bool, unsigned int);
   unsigned int from_array (HOST_WIDE_INT *, const HOST_WIDE_INT *,
                           unsigned int, unsigned int, bool);
+
+  wide_int bit_repeat (const wide_int_ref &, unsigned int);
+  wide_int clmul (const wide_int_ref &, const wide_int_ref &, unsigned int);
 }
 
 /* Return a PRECISION-bit integer in which the low WIDTH bits are set

Reply via email to