Subject: bug#65674: Build failure with <openssl-3

From: Bruno Haible <[email-masked]>
Date: Fri, 01 Sep 2023 15:00:29 +0200

Sam James wrote:
> Forwarding a downstream report at https://bugs.gentoo.org/913368
> of coreutils-9.4 failing to build with openssl-1.1.x:
> """
> x86_64-pc-linux-gnu-gcc  -I. -I./lib  -DHASH_ALGO_BLAKE2=1 -DHAVE_CONFIG_H -Ilib -I./lib -Isrc -I./src    -O2 -march=x86-64 -pipe -pipe -frecord-gcc-switches -fno-diagnostics-color -fmessage-length=0 -c -o src/b2sum-digest.o `test -f 'src/digest.c' || echo './'`src/digest.c
> In file included from src/digest.c:41:
> ./lib/md5.h:36:12: fatal error: openssl/configuration.h: No such file or directory
>    36 | #  include <openssl/configuration.h>
>       |            ^~~~~~~~~~~~~~~~~~~~~~~~~
> compilation terminated.
> """
> 
> This appears to have been introduced with gnulib commit
> a436f5f498d7e747864d40d4450fa8330dd44d12.
> 
> configuration.h is only available with >=openssl-3.

Thanks for the report.

We did test on platforms with OpenSSL 1.1.x before the coreutils-9.4
release (namely, on AIX 7.1 and FreeBSD 13.1 [1]), and did not encounter
this problem. That is because, by default, on such platforms, the
configure test
  checking whether openssl is GPL compatible... no
determines that OpenSSL should not be used. More regarding this license
incompatibility at
  https://www.gnu.org/licenses/license-list.en.html#OpenSSL

Apparently your user's build is using the configure option
'--with-openssl' or 'with-openssl=yes'.

The patch below fixes the compilation failure.

However, since OpenSSL 1.1.x is out-of-support 11 days from now [2]
and it is quite dangerous to use a security-centered software when
it is no longer supported, I would suggest that you (Gentoo) migrate
to OpenSSL version 3.0.*.

Bruno

[1] https://lists.gnu.org/archive/html/coreutils/2023-08/msg00096.html
[2] https://en.wikipedia.org/wiki/OpenSSL#Major_version_releases

2023-09-01  Bruno Haible  <[email-masked]>

	crypto/{sha*,md5,sm3}-buffer: Fix --with-openssl (regr. 2023-08-26).
	Reported by Agostino Sarubbo via Sam James <[email-masked]> in
	<https://lists.gnu.org/archive/html/bug-gnulib/2023-09/msg00000.html>.
	* lib/sha1.h: Test the OpenSSL major version before attempting to
	include <openssl/configuration.h>.
	* lib/sha256.h: Likewise.
	* lib/sha512.h: Likewise.
	* lib/md5.h: Likewise.
	* lib/sm3.h: Likewise.

diff --git a/lib/md5.h b/lib/md5.h
index 6ddf009148..b298fc4cc3 100644
--- a/lib/md5.h
+++ b/lib/md5.h
@@ -33,14 +33,18 @@
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
 /* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  */
-#  include <openssl/configuration.h>
-#  if (OPENSSL_CONFIGURED_API \
-       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
-          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
-          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
-          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
-#   undef HAVE_OPENSSL_MD5
-#  else
+#  include <openssl/opensslv.h>
+#  if OPENSSL_VERSION_MAJOR >= 3
+#   include <openssl/configuration.h>
+#   if (OPENSSL_CONFIGURED_API \
+        < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+           ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+           + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+           + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#    undef HAVE_OPENSSL_MD5
+#   endif
+#  endif
+#  if HAVE_OPENSSL_MD5
 #   include <openssl/md5.h>
 #  endif
 # endif
diff --git a/lib/sha1.h b/lib/sha1.h
index d5a6e72e2c..cf67997f3d 100644
--- a/lib/sha1.h
+++ b/lib/sha1.h
@@ -32,14 +32,18 @@
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
 /* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  */
-#  include <openssl/configuration.h>
-#  if (OPENSSL_CONFIGURED_API \
-       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
-          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
-          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
-          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
-#   undef HAVE_OPENSSL_SHA1
-#  else
+#  include <openssl/opensslv.h>
+#  if OPENSSL_VERSION_MAJOR >= 3
+#   include <openssl/configuration.h>
+#   if (OPENSSL_CONFIGURED_API \
+        < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+           ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+           + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+           + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#    undef HAVE_OPENSSL_SHA1
+#   endif
+#  endif
+#  if HAVE_OPENSSL_SHA1
 #   include <openssl/sha.h>
 #  endif
 # endif
diff --git a/lib/sha256.h b/lib/sha256.h
index 508bce7de8..5a0b652b78 100644
--- a/lib/sha256.h
+++ b/lib/sha256.h
@@ -31,14 +31,18 @@
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
 /* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  */
-#  include <openssl/configuration.h>
-#  if (OPENSSL_CONFIGURED_API \
-       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
-          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
-          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
-          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
-#   undef HAVE_OPENSSL_SHA256
-#  else
+#  include <openssl/opensslv.h>
+#  if OPENSSL_VERSION_MAJOR >= 3
+#   include <openssl/configuration.h>
+#   if (OPENSSL_CONFIGURED_API \
+        < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+           ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+           + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+           + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#    undef HAVE_OPENSSL_SHA256
+#   endif
+#  endif
+#  if HAVE_OPENSSL_SHA256
 #   include <openssl/sha.h>
 #  endif
 # endif
diff --git a/lib/sha512.h b/lib/sha512.h
index 3259f1c7b8..37832da750 100644
--- a/lib/sha512.h
+++ b/lib/sha512.h
@@ -31,14 +31,18 @@
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
 /* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  */
-#  include <openssl/configuration.h>
-#  if (OPENSSL_CONFIGURED_API \
-       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
-          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
-          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
-          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
-#   undef HAVE_OPENSSL_SHA512
-#  else
+#  include <openssl/opensslv.h>
+#  if OPENSSL_VERSION_MAJOR >= 3
+#   include <openssl/configuration.h>
+#   if (OPENSSL_CONFIGURED_API \
+        < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+           ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+           + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+           + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#    undef HAVE_OPENSSL_SHA512
+#   endif
+#  endif
+#  if HAVE_OPENSSL_SHA512
 #   include <openssl/sha.h>
 #  endif
 # endif
diff --git a/lib/sm3.h b/lib/sm3.h
index f60efdfe9a..36f1cd5297 100644
--- a/lib/sm3.h
+++ b/lib/sm3.h
@@ -40,14 +40,18 @@
 #   define OPENSSL_API_COMPAT 0x10101000L /* FIXME: Use OpenSSL 1.1+ API.  */
 #  endif
 /* If <openssl/macros.h> would give a compile-time error, don't use OpenSSL.  */
-#  include <openssl/configuration.h>
-#  if (OPENSSL_CONFIGURED_API \
-       < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
-          ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
-          + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
-          + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
-#   undef HAVE_OPENSSL_SM3
-#  else
+#  include <openssl/opensslv.h>
+#  if OPENSSL_VERSION_MAJOR >= 3
+#   include <openssl/configuration.h>
+#   if (OPENSSL_CONFIGURED_API \
+        < (OPENSSL_API_COMPAT < 0x900000L ? OPENSSL_API_COMPAT : \
+           ((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000 \
+           + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+           + ((OPENSSL_API_COMPAT >> 12) & 0xFF)))
+#    undef HAVE_OPENSSL_SM3
+#   endif
+#  endif
+#  if HAVE_OPENSSL_SM3
 #   include <openssl/sm3.h>
 #  endif
 # endif