Use __BYTE_ORDER__ to test the endianness when available

Reviewed-by: NPaul Dale <paul.dale@oracle.com>
GH: #13085

include/internal/endian.h

@@ -10,13 +10,41 @@
 #ifndef OSSL_INTERNAL_ENDIAN_H
 # define OSSL_INTERNAL_ENDIAN_H
 
-# define DECLARE_IS_ENDIAN \
+/*
+ * IS_LITTLE_ENDIAN and IS_BIG_ENDIAN can be used to detect the endiannes
+ * at compile time. To use it, DECLARE_IS_ENDIAN must be used to declare
+ * a variable.
+ *
+ * L_ENDIAN and B_ENDIAN can be used at preprocessor time. They can be set
+ * in the configarion using the lib_cppflags variable. If neither is
+ * set, it will fall back to code works with either endianness.
+ */
+
+# if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__)
+#  define DECLARE_IS_ENDIAN const int ossl_is_little_endian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#  define IS_LITTLE_ENDIAN (ossl_is_little_endian)
+#  define IS_BIG_ENDIAN (!ossl_is_little_endian)
+#  if defined(L_ENDIAN) && (__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__)
+#   error "L_ENDIAN defined on a big endian machine"
+#  endif
+#  if defined(B_ENDIAN) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#   error "B_ENDIAN defined on a little endian machine"
+#  endif
+#  if !defined(L_ENDIAN) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#   define L_ENDIAN
+#  endif
+#  if !defined(B_ENDIAN) && (__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__)
+#   define B_ENDIAN
+#  endif
+# else
+#  define DECLARE_IS_ENDIAN \
     const union { \
         long one; \
         char little; \
     } ossl_is_endian = { 1 }
 
-# define IS_LITTLE_ENDIAN (ossl_is_endian.little != 0)
-# define IS_BIG_ENDIAN    (ossl_is_endian.little == 0)
+#  define IS_LITTLE_ENDIAN (ossl_is_endian.little != 0)
+#  define IS_BIG_ENDIAN    (ossl_is_endian.little == 0)
+# endif
 
 #endif