casaroli commented on code in PR #21491:
URL: https://github.com/apache/nuttx/pull/21491#discussion_r4497806638
##########
include/stdint.h:
##########
@@ -185,6 +185,11 @@ typedef _int64_t int64_t;
typedef _uint64_t uint64_t;
#endif
+#ifdef __INT128_DEFINED
+typedef _int128_t int128_t;
Review Comment:
`int128_t` and `uint128_t` in `<stdint.h>` break every program that defines
its own `uint128_t` as another type. Python's HACL `krml/types.h` defines it as
a struct and now fails with "conflicting types for 'uint128_t'": apps #3985 has
to patch Python only for this. glibc, musl and newlib do not define these
names, so portable code uses them freely.
Fix: keep the type private: use `_uint128_t` under `__INT128_DEFINED` in
`math32.h` and drop the two typedefs here (then apps #3985 and the
`crypto/ecc.c` change are not needed).
##########
include/nuttx/lib/math32.h:
##########
@@ -195,160 +203,175 @@ extern "C"
/* Default C implementation for umul64_const()
*
- * Prototype: uint64_t umul64_const(uint64_t retval, uint64_t m,
- * uint64_t n, bool bias);
- * Semantic: retval = ((bias ? m : 0) + m * n) >> 64
+ * Prototype: uint64_t umul64_const(uint64_t m, uint64_t n, int bias);
+ * Semantic: return ((bias ? m : 0) + m * n) >> 64
*
* The product is a 128-bit value, scaled down to 64 bits.
* Assuming constant propagation to optimize away unused conditional code.
* Architectures may provide their own optimized assembly implementation.
*/
-# ifdef up_umul64_const
-# define umul64_const(res, m, n, bias) \
- (res) = up_umul64_const(m, n, bias)
-# else
-# define umul64_const(res, m, n, bias) \
- do \
- { \
- uint32_t __m_lo = (m) & 0xffffffff; \
- uint32_t __m_hi = (m) >> 32; \
- uint32_t __n_lo = (n) & 0xffffffff; \
- uint32_t __n_hi = (n) >> 32; \
- uint32_t __res_lo; \
- uint32_t __res_hi; \
- uint32_t __tmp; \
- \
- if (!(bias)) \
- { \
- (res) = ((uint64_t)__m_lo * __n_lo) >> 32; \
- } \
- else if (!((m) & ((1ULL << 63) | (1ULL << 31)))) \
- { \
- (res) = ((m) + (uint64_t)__m_lo * __n_lo) >> 32; \
- } \
- else \
- { \
- (res) = (m) + (uint64_t)__m_lo * __n_lo; \
- __res_lo = (res) >> 32; \
- __res_hi = (__res_lo < __m_hi); \
- (res) = __res_lo | ((uint64_t)__res_hi << 32); \
- } \
- \
- if (!((m) & ((1ULL << 63) | (1ULL << 31)))) \
- { \
- (res) += (uint64_t)__m_lo * __n_hi; \
- (res) += (uint64_t)__m_hi * __n_lo; \
- (res) >>= 32; \
- } \
- else \
- { \
- (res) += (uint64_t)__m_lo * __n_hi; \
- __tmp = (res) >> 32; \
- (res) += (uint64_t)__m_hi * __n_lo; \
- __res_lo = (res) >> 32; \
- __res_hi = (__res_lo < __tmp); \
- (res) = __res_lo | ((uint64_t)__res_hi << 32); \
- } \
- \
- (res) += (uint64_t)__m_hi * __n_hi; \
- } \
- while (0)
-# endif
+/* Due to compilation issues, use inline functions instead of macros. */
-# define div64_const32(n, b) \
- do \
- { \
- uint64_t ___res; \
- uint64_t ___x; \
- uint64_t ___t; \
- uint64_t ___m; \
- uint64_t ___n = (n); \
- uint32_t ___p; \
- uint32_t ___bias; \
- uint32_t ___b = (b); \
- ___p = 1 << LOG2_FLOOR(___b); \
- ___m = (~0ULL / ___b) * ___p; \
- ___m += (((~0ULL % ___b + 1) * ___p) + ___b - 1) / ___b; \
- ___x = ~0ULL / ___b * ___b - 1; \
- ___res = ((___m & 0xffffffff) * (___x & 0xffffffff)) >> 32; \
- ___t = ___res += (___m & 0xffffffff) * (___x >> 32); \
- ___res += (___x & 0xffffffff) * (___m >> 32); \
- ___t = (___res < ___t) ? (1ULL << 32) : 0; \
- ___res = (___res >> 32) + ___t; \
- ___res += (___m >> 32) * (___x >> 32); \
- ___res /= ___p; \
- if (~0ULL % (___b / (___b & -___b)) == 0) \
- { \
- ___n /= (___b & -___b); \
- ___m = ~0ULL / (___b / (___b & -___b)); \
- ___p = 1; \
- ___bias = 1; \
- } \
- else if (___res != ___x / ___b) \
- { \
- ___bias = 1; \
- ___m = (~0ULL / ___b) * ___p; \
- ___m += ((~0ULL % ___b + 1) * ___p) / ___b; \
- } \
- else \
- { \
- uint32_t ___bits = -(___m & -___m); \
- ___bits |= ___m >> 32; \
- ___bits = (~___bits) << 1; \
- if (!___bits) \
- { \
- ___p /= (___m & -___m); \
- ___m /= (___m & -___m); \
- } \
- else \
- { \
- ___p >>= LOG2_FLOOR(___bits); \
- ___m >>= LOG2_FLOOR(___bits); \
- } \
- ___bias = 0; \
- } \
- umul64_const(___res, ___m, ___n, ___bias); \
- \
- ___res /= ___p; \
- (n) = ___res; \
- } \
- while (0)
-
-#if defined(CONFIG_HAVE_EXPRESSION_STATEMENT)
-# define div64_const(n, base) \
- ({ \
- uint64_t __n = (n); \
- uint32_t __base = (base); \
- if (IS_POWER_OF_2(__base)) \
- { \
- (__n) >>= LOG2_FLOOR(__base); \
- } \
- else if (UINTPTR_MAX == UINT32_MAX) \
- { \
- div64_const32(__n, __base); \
- } \
- else \
- { \
- __n /= __base; \
- } \
- __n; \
- })
-
-# define div_const(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const(n, base) : ((n) /
(base)))
-# define div_const_roundup(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const((n) + (base) - 1,
base) : \
- (((n) + (base) - 1) / (base)))
-# define div_const_roundnearest(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const((n) + ((base) / 2),
base) : \
- (((n) + ((base) / 2)) / (base)))
+#ifdef up_umul64_const
+# define umul64_const(m, n, bias) up_umul64_const(m, n, bias)
#else
-# define div_const(n, base) ((n) / (base))
-# define div_const_roundup(n, base) (((n) + (base) - 1) / (base))
-# define div_const_roundnearest(n, base) (((n) + ((base) / 2)) / (base))
+static inline_function uint64_t umul64_const(uint64_t m, uint64_t n,
+ int bias)
+{
+ uint32_t m_lo = (m) & 0xffffffff;
+ uint32_t m_hi = (m) >> 32;
+ uint32_t n_lo = (n) & 0xffffffff;
+ uint32_t n_hi = (n) >> 32;
+ uint32_t res_lo;
+ uint32_t res_hi;
+ uint32_t tmp;
+ uint64_t res;
+
+ if (!(bias))
+ {
+ res = ((uint64_t)m_lo * n_lo) >> 32;
+ }
+ else if (!((m) & ((1ULL << 63) | (1ULL << 31))))
+ {
+ res = ((m) + (uint64_t)m_lo * n_lo) >> 32;
+ }
+ else
+ {
+ res = (m) + (uint64_t)m_lo * n_lo;
+ res_lo = res >> 32;
+ res_hi = (res_lo < m_hi);
+ res = res_lo | ((uint64_t)res_hi << 32);
+ }
+
+ if (!((m) & ((1ULL << 63) | (1ULL << 31))))
+ {
+ res += (uint64_t)m_lo * n_hi;
+ res += (uint64_t)m_hi * n_lo;
+ res >>= 32;
+ }
+ else
+ {
+ res += (uint64_t)m_lo * n_hi;
+ tmp = res >> 32;
+ res += (uint64_t)m_hi * n_lo;
+ res_lo = res >> 32;
+ res_hi = (res_lo < tmp);
+ res = res_lo | ((uint64_t)res_hi << 32);
+ }
+
+ res += (uint64_t)m_hi * n_hi;
+
+ return res;
+}
#endif
+static inline_function uint64_t div64_const32(uint64_t n, uint32_t b)
+{
+ uint64_t res;
+ uint64_t x;
+ uint64_t t;
+ uint64_t m;
+ uint32_t p;
+ uint32_t bias;
+
+ DEBUGASSERT(b > 1);
+
+ p = 1u << LOG2_FLOOR(b);
+ m = (~0ULL / b) * p;
+ m += (((~0ULL % b + 1u) * p) + b - 1u) / b;
+ x = ~0ULL / b * b - 1u;
+ res = ((m & 0xffffffffu) * (x & 0xffffffffu)) >> 32;
+ t = res += (m & 0xffffffffu) * (x >> 32);
+ res += (x & 0xffffffffu) * (m >> 32);
+ t = (res < t) ? (1ULL << 32) : 0u;
+ res = (res >> 32) + t;
+ res += (m >> 32) * (x >> 32);
+ res /= p;
+
+ if (~0ULL % (b / (b & -b)) == 0u)
+ {
+ n /= (b & -b);
+ m = ~0ULL / (b / (b & -b));
+ p = 1u;
+ bias = 1u;
+ }
+ else if (res != x / b)
+ {
+ bias = 1u;
+ m = (~0ULL / b) * p;
+ m += ((~0ULL % b + 1u) * p) / b;
+ }
+ else
+ {
+ uint32_t bits = -(m & -m);
+ bits |= m >> 32;
+ bits = (~bits) << 1;
+ if (!bits)
+ {
+ p /= (m & -m);
+ m /= (m & -m);
+ }
+ else
+ {
+ p >>= LOG2_FLOOR(bits);
+ m >>= LOG2_FLOOR(bits);
+ }
+
+ bias = 0u;
+ }
+
+ res = umul64_const(m, n, bias);
+ res /= p;
+
+ return res;
+}
+
+#ifdef CONFIG_DEBUG_NOOPT
+
+/* Without optimization the magic number setup is not folded at compile
+ * time but runs on every call (it even contains several divisions),
+ * which is far slower than a plain division.
+ */
+
+# define div64_const(n, base) ((n) / (base))
+#else
+
+/* The divisor is expected to be a compile-time constant so that the
+ * magic multiplier setup below is folded away; for runtime-invariant
+ * divisors use the invdiv API instead.
+ */
+
+static inline_function uint64_t div64_const(uint64_t n, uint64_t base)
+{
+ DEBUGASSERT(base > 0);
+
+ if (IS_POWER_OF_2(base))
+ {
+ n >>= LOG2_FLOOR(base);
+ }
+ else if (base <= UINT32_MAX)
+ {
+ n = div64_const32(n, (uint32_t)base);
+ }
+ else
+ {
+ n /= base;
+ }
+
+ return n;
+}
+#endif
+
+#define div_const(n, base) \
Review Comment:
`div_const()` now always has the type `uint64_t`, because the other operand
of `?:` is `div64_const()`. On master it is the plain `(n) / (base)` everywhere
(`CONFIG_HAVE_EXPRESSION_STATEMENT` is defined nowhere), so `MSEC2TICK()` of a
`uint32_t` was `unsigned long` on 32-bit targets. This is the CI failure:
`esp_ble_adapter.c:1025: format '%lu' expects argument of type 'long unsigned
int', but argument 3 has type 'uint64_t'` (also `esp32_ble_adapter.c:1124`,
`esp32s3_ble_adapter.c:1003`).
Fix: cast the result to the type of `(n) / (base)` (`__typeof__` in the GNU
section), and use the plain `((n) / (base))` where `__typeof__` is not
available.
##########
include/nuttx/lib/math32.h:
##########
@@ -195,160 +203,175 @@ extern "C"
/* Default C implementation for umul64_const()
*
- * Prototype: uint64_t umul64_const(uint64_t retval, uint64_t m,
- * uint64_t n, bool bias);
- * Semantic: retval = ((bias ? m : 0) + m * n) >> 64
+ * Prototype: uint64_t umul64_const(uint64_t m, uint64_t n, int bias);
+ * Semantic: return ((bias ? m : 0) + m * n) >> 64
*
* The product is a 128-bit value, scaled down to 64 bits.
* Assuming constant propagation to optimize away unused conditional code.
* Architectures may provide their own optimized assembly implementation.
*/
-# ifdef up_umul64_const
-# define umul64_const(res, m, n, bias) \
- (res) = up_umul64_const(m, n, bias)
-# else
-# define umul64_const(res, m, n, bias) \
- do \
- { \
- uint32_t __m_lo = (m) & 0xffffffff; \
- uint32_t __m_hi = (m) >> 32; \
- uint32_t __n_lo = (n) & 0xffffffff; \
- uint32_t __n_hi = (n) >> 32; \
- uint32_t __res_lo; \
- uint32_t __res_hi; \
- uint32_t __tmp; \
- \
- if (!(bias)) \
- { \
- (res) = ((uint64_t)__m_lo * __n_lo) >> 32; \
- } \
- else if (!((m) & ((1ULL << 63) | (1ULL << 31)))) \
- { \
- (res) = ((m) + (uint64_t)__m_lo * __n_lo) >> 32; \
- } \
- else \
- { \
- (res) = (m) + (uint64_t)__m_lo * __n_lo; \
- __res_lo = (res) >> 32; \
- __res_hi = (__res_lo < __m_hi); \
- (res) = __res_lo | ((uint64_t)__res_hi << 32); \
- } \
- \
- if (!((m) & ((1ULL << 63) | (1ULL << 31)))) \
- { \
- (res) += (uint64_t)__m_lo * __n_hi; \
- (res) += (uint64_t)__m_hi * __n_lo; \
- (res) >>= 32; \
- } \
- else \
- { \
- (res) += (uint64_t)__m_lo * __n_hi; \
- __tmp = (res) >> 32; \
- (res) += (uint64_t)__m_hi * __n_lo; \
- __res_lo = (res) >> 32; \
- __res_hi = (__res_lo < __tmp); \
- (res) = __res_lo | ((uint64_t)__res_hi << 32); \
- } \
- \
- (res) += (uint64_t)__m_hi * __n_hi; \
- } \
- while (0)
-# endif
+/* Due to compilation issues, use inline functions instead of macros. */
-# define div64_const32(n, b) \
- do \
- { \
- uint64_t ___res; \
- uint64_t ___x; \
- uint64_t ___t; \
- uint64_t ___m; \
- uint64_t ___n = (n); \
- uint32_t ___p; \
- uint32_t ___bias; \
- uint32_t ___b = (b); \
- ___p = 1 << LOG2_FLOOR(___b); \
- ___m = (~0ULL / ___b) * ___p; \
- ___m += (((~0ULL % ___b + 1) * ___p) + ___b - 1) / ___b; \
- ___x = ~0ULL / ___b * ___b - 1; \
- ___res = ((___m & 0xffffffff) * (___x & 0xffffffff)) >> 32; \
- ___t = ___res += (___m & 0xffffffff) * (___x >> 32); \
- ___res += (___x & 0xffffffff) * (___m >> 32); \
- ___t = (___res < ___t) ? (1ULL << 32) : 0; \
- ___res = (___res >> 32) + ___t; \
- ___res += (___m >> 32) * (___x >> 32); \
- ___res /= ___p; \
- if (~0ULL % (___b / (___b & -___b)) == 0) \
- { \
- ___n /= (___b & -___b); \
- ___m = ~0ULL / (___b / (___b & -___b)); \
- ___p = 1; \
- ___bias = 1; \
- } \
- else if (___res != ___x / ___b) \
- { \
- ___bias = 1; \
- ___m = (~0ULL / ___b) * ___p; \
- ___m += ((~0ULL % ___b + 1) * ___p) / ___b; \
- } \
- else \
- { \
- uint32_t ___bits = -(___m & -___m); \
- ___bits |= ___m >> 32; \
- ___bits = (~___bits) << 1; \
- if (!___bits) \
- { \
- ___p /= (___m & -___m); \
- ___m /= (___m & -___m); \
- } \
- else \
- { \
- ___p >>= LOG2_FLOOR(___bits); \
- ___m >>= LOG2_FLOOR(___bits); \
- } \
- ___bias = 0; \
- } \
- umul64_const(___res, ___m, ___n, ___bias); \
- \
- ___res /= ___p; \
- (n) = ___res; \
- } \
- while (0)
-
-#if defined(CONFIG_HAVE_EXPRESSION_STATEMENT)
-# define div64_const(n, base) \
- ({ \
- uint64_t __n = (n); \
- uint32_t __base = (base); \
- if (IS_POWER_OF_2(__base)) \
- { \
- (__n) >>= LOG2_FLOOR(__base); \
- } \
- else if (UINTPTR_MAX == UINT32_MAX) \
- { \
- div64_const32(__n, __base); \
- } \
- else \
- { \
- __n /= __base; \
- } \
- __n; \
- })
-
-# define div_const(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const(n, base) : ((n) /
(base)))
-# define div_const_roundup(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const((n) + (base) - 1,
base) : \
- (((n) + (base) - 1) / (base)))
-# define div_const_roundnearest(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const((n) + ((base) / 2),
base) : \
- (((n) + ((base) / 2)) / (base)))
+#ifdef up_umul64_const
+# define umul64_const(m, n, bias) up_umul64_const(m, n, bias)
#else
-# define div_const(n, base) ((n) / (base))
-# define div_const_roundup(n, base) (((n) + (base) - 1) / (base))
-# define div_const_roundnearest(n, base) (((n) + ((base) / 2)) / (base))
+static inline_function uint64_t umul64_const(uint64_t m, uint64_t n,
+ int bias)
+{
+ uint32_t m_lo = (m) & 0xffffffff;
+ uint32_t m_hi = (m) >> 32;
+ uint32_t n_lo = (n) & 0xffffffff;
+ uint32_t n_hi = (n) >> 32;
+ uint32_t res_lo;
+ uint32_t res_hi;
+ uint32_t tmp;
+ uint64_t res;
+
+ if (!(bias))
+ {
+ res = ((uint64_t)m_lo * n_lo) >> 32;
+ }
+ else if (!((m) & ((1ULL << 63) | (1ULL << 31))))
+ {
+ res = ((m) + (uint64_t)m_lo * n_lo) >> 32;
+ }
+ else
+ {
+ res = (m) + (uint64_t)m_lo * n_lo;
+ res_lo = res >> 32;
+ res_hi = (res_lo < m_hi);
+ res = res_lo | ((uint64_t)res_hi << 32);
+ }
+
+ if (!((m) & ((1ULL << 63) | (1ULL << 31))))
+ {
+ res += (uint64_t)m_lo * n_hi;
+ res += (uint64_t)m_hi * n_lo;
+ res >>= 32;
+ }
+ else
+ {
+ res += (uint64_t)m_lo * n_hi;
+ tmp = res >> 32;
+ res += (uint64_t)m_hi * n_lo;
+ res_lo = res >> 32;
+ res_hi = (res_lo < tmp);
+ res = res_lo | ((uint64_t)res_hi << 32);
+ }
+
+ res += (uint64_t)m_hi * n_hi;
+
+ return res;
+}
#endif
+static inline_function uint64_t div64_const32(uint64_t n, uint32_t b)
+{
+ uint64_t res;
+ uint64_t x;
+ uint64_t t;
+ uint64_t m;
+ uint32_t p;
+ uint32_t bias;
+
+ DEBUGASSERT(b > 1);
+
+ p = 1u << LOG2_FLOOR(b);
+ m = (~0ULL / b) * p;
+ m += (((~0ULL % b + 1u) * p) + b - 1u) / b;
+ x = ~0ULL / b * b - 1u;
+ res = ((m & 0xffffffffu) * (x & 0xffffffffu)) >> 32;
+ t = res += (m & 0xffffffffu) * (x >> 32);
+ res += (x & 0xffffffffu) * (m >> 32);
+ t = (res < t) ? (1ULL << 32) : 0u;
+ res = (res >> 32) + t;
+ res += (m >> 32) * (x >> 32);
+ res /= p;
+
+ if (~0ULL % (b / (b & -b)) == 0u)
+ {
+ n /= (b & -b);
+ m = ~0ULL / (b / (b & -b));
+ p = 1u;
+ bias = 1u;
+ }
+ else if (res != x / b)
+ {
+ bias = 1u;
+ m = (~0ULL / b) * p;
+ m += ((~0ULL % b + 1u) * p) / b;
+ }
+ else
+ {
+ uint32_t bits = -(m & -m);
+ bits |= m >> 32;
+ bits = (~bits) << 1;
+ if (!bits)
+ {
+ p /= (m & -m);
+ m /= (m & -m);
+ }
+ else
+ {
+ p >>= LOG2_FLOOR(bits);
+ m >>= LOG2_FLOOR(bits);
+ }
+
+ bias = 0u;
+ }
+
+ res = umul64_const(m, n, bias);
+ res /= p;
+
+ return res;
+}
+
+#ifdef CONFIG_DEBUG_NOOPT
+
+/* Without optimization the magic number setup is not folded at compile
+ * time but runs on every call (it even contains several divisions),
+ * which is far slower than a plain division.
+ */
+
+# define div64_const(n, base) ((n) / (base))
+#else
+
+/* The divisor is expected to be a compile-time constant so that the
+ * magic multiplier setup below is folded away; for runtime-invariant
+ * divisors use the invdiv API instead.
+ */
+
+static inline_function uint64_t div64_const(uint64_t n, uint64_t base)
+{
+ DEBUGASSERT(base > 0);
+
+ if (IS_POWER_OF_2(base))
+ {
+ n >>= LOG2_FLOOR(base);
+ }
+ else if (base <= UINT32_MAX)
Review Comment:
On 64-bit targets this replaces the compiler's own division by a constant
with `div64_const32()`. With aarch64 GCC 14 -O2, `div_const(n, 1000)` for a
`uint64_t n` is 8 instructions with one `umulh` on master and 26 instructions
with four `umaddl` with this PR (rv64gc: 30 -> 108 bytes).
Fix: use the plain `n / base` when `UINTPTR_MAX > UINT32_MAX`, as the old
`div64_const()` did.
##########
include/nuttx/lib/math32.h:
##########
@@ -195,160 +203,175 @@ extern "C"
/* Default C implementation for umul64_const()
*
- * Prototype: uint64_t umul64_const(uint64_t retval, uint64_t m,
- * uint64_t n, bool bias);
- * Semantic: retval = ((bias ? m : 0) + m * n) >> 64
+ * Prototype: uint64_t umul64_const(uint64_t m, uint64_t n, int bias);
+ * Semantic: return ((bias ? m : 0) + m * n) >> 64
*
* The product is a 128-bit value, scaled down to 64 bits.
* Assuming constant propagation to optimize away unused conditional code.
* Architectures may provide their own optimized assembly implementation.
*/
-# ifdef up_umul64_const
-# define umul64_const(res, m, n, bias) \
- (res) = up_umul64_const(m, n, bias)
-# else
-# define umul64_const(res, m, n, bias) \
- do \
- { \
- uint32_t __m_lo = (m) & 0xffffffff; \
- uint32_t __m_hi = (m) >> 32; \
- uint32_t __n_lo = (n) & 0xffffffff; \
- uint32_t __n_hi = (n) >> 32; \
- uint32_t __res_lo; \
- uint32_t __res_hi; \
- uint32_t __tmp; \
- \
- if (!(bias)) \
- { \
- (res) = ((uint64_t)__m_lo * __n_lo) >> 32; \
- } \
- else if (!((m) & ((1ULL << 63) | (1ULL << 31)))) \
- { \
- (res) = ((m) + (uint64_t)__m_lo * __n_lo) >> 32; \
- } \
- else \
- { \
- (res) = (m) + (uint64_t)__m_lo * __n_lo; \
- __res_lo = (res) >> 32; \
- __res_hi = (__res_lo < __m_hi); \
- (res) = __res_lo | ((uint64_t)__res_hi << 32); \
- } \
- \
- if (!((m) & ((1ULL << 63) | (1ULL << 31)))) \
- { \
- (res) += (uint64_t)__m_lo * __n_hi; \
- (res) += (uint64_t)__m_hi * __n_lo; \
- (res) >>= 32; \
- } \
- else \
- { \
- (res) += (uint64_t)__m_lo * __n_hi; \
- __tmp = (res) >> 32; \
- (res) += (uint64_t)__m_hi * __n_lo; \
- __res_lo = (res) >> 32; \
- __res_hi = (__res_lo < __tmp); \
- (res) = __res_lo | ((uint64_t)__res_hi << 32); \
- } \
- \
- (res) += (uint64_t)__m_hi * __n_hi; \
- } \
- while (0)
-# endif
+/* Due to compilation issues, use inline functions instead of macros. */
-# define div64_const32(n, b) \
- do \
- { \
- uint64_t ___res; \
- uint64_t ___x; \
- uint64_t ___t; \
- uint64_t ___m; \
- uint64_t ___n = (n); \
- uint32_t ___p; \
- uint32_t ___bias; \
- uint32_t ___b = (b); \
- ___p = 1 << LOG2_FLOOR(___b); \
- ___m = (~0ULL / ___b) * ___p; \
- ___m += (((~0ULL % ___b + 1) * ___p) + ___b - 1) / ___b; \
- ___x = ~0ULL / ___b * ___b - 1; \
- ___res = ((___m & 0xffffffff) * (___x & 0xffffffff)) >> 32; \
- ___t = ___res += (___m & 0xffffffff) * (___x >> 32); \
- ___res += (___x & 0xffffffff) * (___m >> 32); \
- ___t = (___res < ___t) ? (1ULL << 32) : 0; \
- ___res = (___res >> 32) + ___t; \
- ___res += (___m >> 32) * (___x >> 32); \
- ___res /= ___p; \
- if (~0ULL % (___b / (___b & -___b)) == 0) \
- { \
- ___n /= (___b & -___b); \
- ___m = ~0ULL / (___b / (___b & -___b)); \
- ___p = 1; \
- ___bias = 1; \
- } \
- else if (___res != ___x / ___b) \
- { \
- ___bias = 1; \
- ___m = (~0ULL / ___b) * ___p; \
- ___m += ((~0ULL % ___b + 1) * ___p) / ___b; \
- } \
- else \
- { \
- uint32_t ___bits = -(___m & -___m); \
- ___bits |= ___m >> 32; \
- ___bits = (~___bits) << 1; \
- if (!___bits) \
- { \
- ___p /= (___m & -___m); \
- ___m /= (___m & -___m); \
- } \
- else \
- { \
- ___p >>= LOG2_FLOOR(___bits); \
- ___m >>= LOG2_FLOOR(___bits); \
- } \
- ___bias = 0; \
- } \
- umul64_const(___res, ___m, ___n, ___bias); \
- \
- ___res /= ___p; \
- (n) = ___res; \
- } \
- while (0)
-
-#if defined(CONFIG_HAVE_EXPRESSION_STATEMENT)
-# define div64_const(n, base) \
- ({ \
- uint64_t __n = (n); \
- uint32_t __base = (base); \
- if (IS_POWER_OF_2(__base)) \
- { \
- (__n) >>= LOG2_FLOOR(__base); \
- } \
- else if (UINTPTR_MAX == UINT32_MAX) \
- { \
- div64_const32(__n, __base); \
- } \
- else \
- { \
- __n /= __base; \
- } \
- __n; \
- })
-
-# define div_const(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const(n, base) : ((n) /
(base)))
-# define div_const_roundup(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const((n) + (base) - 1,
base) : \
- (((n) + (base) - 1) / (base)))
-# define div_const_roundnearest(n, base) \
- ((sizeof(typeof(n)) == sizeof(uint64_t)) ? div64_const((n) + ((base) / 2),
base) : \
- (((n) + ((base) / 2)) / (base)))
+#ifdef up_umul64_const
+# define umul64_const(m, n, bias) up_umul64_const(m, n, bias)
#else
-# define div_const(n, base) ((n) / (base))
-# define div_const_roundup(n, base) (((n) + (base) - 1) / (base))
-# define div_const_roundnearest(n, base) (((n) + ((base) / 2)) / (base))
+static inline_function uint64_t umul64_const(uint64_t m, uint64_t n,
+ int bias)
+{
+ uint32_t m_lo = (m) & 0xffffffff;
+ uint32_t m_hi = (m) >> 32;
+ uint32_t n_lo = (n) & 0xffffffff;
+ uint32_t n_hi = (n) >> 32;
+ uint32_t res_lo;
+ uint32_t res_hi;
+ uint32_t tmp;
+ uint64_t res;
+
+ if (!(bias))
+ {
+ res = ((uint64_t)m_lo * n_lo) >> 32;
+ }
+ else if (!((m) & ((1ULL << 63) | (1ULL << 31))))
+ {
+ res = ((m) + (uint64_t)m_lo * n_lo) >> 32;
+ }
+ else
+ {
+ res = (m) + (uint64_t)m_lo * n_lo;
+ res_lo = res >> 32;
+ res_hi = (res_lo < m_hi);
+ res = res_lo | ((uint64_t)res_hi << 32);
+ }
+
+ if (!((m) & ((1ULL << 63) | (1ULL << 31))))
+ {
+ res += (uint64_t)m_lo * n_hi;
+ res += (uint64_t)m_hi * n_lo;
+ res >>= 32;
+ }
+ else
+ {
+ res += (uint64_t)m_lo * n_hi;
+ tmp = res >> 32;
+ res += (uint64_t)m_hi * n_lo;
+ res_lo = res >> 32;
+ res_hi = (res_lo < tmp);
+ res = res_lo | ((uint64_t)res_hi << 32);
+ }
+
+ res += (uint64_t)m_hi * n_hi;
+
+ return res;
+}
#endif
+static inline_function uint64_t div64_const32(uint64_t n, uint32_t b)
+{
+ uint64_t res;
+ uint64_t x;
+ uint64_t t;
+ uint64_t m;
+ uint32_t p;
+ uint32_t bias;
+
+ DEBUGASSERT(b > 1);
+
+ p = 1u << LOG2_FLOOR(b);
+ m = (~0ULL / b) * p;
+ m += (((~0ULL % b + 1u) * p) + b - 1u) / b;
+ x = ~0ULL / b * b - 1u;
+ res = ((m & 0xffffffffu) * (x & 0xffffffffu)) >> 32;
+ t = res += (m & 0xffffffffu) * (x >> 32);
+ res += (x & 0xffffffffu) * (m >> 32);
+ t = (res < t) ? (1ULL << 32) : 0u;
+ res = (res >> 32) + t;
+ res += (m >> 32) * (x >> 32);
+ res /= p;
+
+ if (~0ULL % (b / (b & -b)) == 0u)
+ {
+ n /= (b & -b);
+ m = ~0ULL / (b / (b & -b));
+ p = 1u;
+ bias = 1u;
+ }
+ else if (res != x / b)
+ {
+ bias = 1u;
+ m = (~0ULL / b) * p;
+ m += ((~0ULL % b + 1u) * p) / b;
+ }
+ else
+ {
+ uint32_t bits = -(m & -m);
+ bits |= m >> 32;
+ bits = (~bits) << 1;
+ if (!bits)
+ {
+ p /= (m & -m);
+ m /= (m & -m);
+ }
+ else
+ {
+ p >>= LOG2_FLOOR(bits);
+ m >>= LOG2_FLOOR(bits);
+ }
+
+ bias = 0u;
+ }
+
+ res = umul64_const(m, n, bias);
+ res /= p;
+
+ return res;
+}
+
+#ifdef CONFIG_DEBUG_NOOPT
+
+/* Without optimization the magic number setup is not folded at compile
+ * time but runs on every call (it even contains several divisions),
+ * which is far slower than a plain division.
+ */
+
+# define div64_const(n, base) ((n) / (base))
+#else
+
+/* The divisor is expected to be a compile-time constant so that the
+ * magic multiplier setup below is folded away; for runtime-invariant
+ * divisors use the invdiv API instead.
+ */
+
+static inline_function uint64_t div64_const(uint64_t n, uint64_t base)
+{
+ DEBUGASSERT(base > 0);
+
+ if (IS_POWER_OF_2(base))
+ {
+ n >>= LOG2_FLOOR(base);
+ }
+ else if (base <= UINT32_MAX)
+ {
+ n = div64_const32(n, (uint32_t)base);
+ }
+ else
+ {
+ n /= base;
+ }
+
+ return n;
+}
+#endif
+
+#define div_const(n, base) \
+ ((is_constexpr(base) && !is_constexpr(n) && \
+ sizeof((n)) == sizeof(uint64_t)) ? div64_const(n, base) : \
+ ((n) / (base)))
+#define div_const_roundup(n, base) \
Review Comment:
Signed dividends now give wrong results: a negative 64-bit value goes
through `div64_const()` as `uint64_t`, and `1u`/`2u` make a 32-bit `int`
dividend unsigned. On a 64-bit host, `div_const((int64_t)-15, 10)` gives
1844674407370955160 (master: -1) and `div_const_roundup((int32_t)-100, 10)`
gives 429496720 (master: -9). The `*2TICK()` macros take `int`, `long` and
`time_t` arguments.
Fix: keep `- 1` and `/ 2` signed here, and take the `div64_const()` path
only for an unsigned dividend.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
