Hello.
As mentioned by Honza, it's using cmake and to be honest I prefer to use a
shared
library than a statically build library. Moreover, it's an optional requirement
and
so that we don't have to include that to contrib/download_prerequisites.
I like the idea of marking of compression algorithm in 'LTO_header'. However,
we do compress the header as well. Proper solution would be to make a new
section .gnu.lto_.header where we'll put:
struct lto_header
{
int16_t major_version;
int16_t minor_version;
};
I don't see a reason why why should have that information in each LTO ELF
section?
In the time being, I've written the code so that I fallback in decompression to
zlib
if ZSTD detects that LTO bytecode was compressed with zlib. On the contrary,
decompression
of zstd with zlib will end with:
lto1: internal compiler error: compressed stream: data error
I'm sending updated version of the patch that can properly detect zstd.
Martin
>From 869b630139676fb740fb5296d68086a8ef7f03ae Mon Sep 17 00:00:00 2001
From: Martin Liska <[email protected]>
Date: Wed, 19 Jun 2019 09:40:35 +0200
Subject: [PATCH 2/2] Add optional support for zstd.
---
gcc/common.opt | 4 +-
gcc/lto-compress.c | 139 ++++++++++++++++++++++++++++++++++++++-------
gcc/timevar.def | 4 +-
3 files changed, 122 insertions(+), 25 deletions(-)
diff --git a/gcc/common.opt b/gcc/common.opt
index a1544d06824..3b71a36552b 100644
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -1888,8 +1888,8 @@ Specify the algorithm to partition symbols and vars at linktime.
; The initial value of -1 comes from Z_DEFAULT_COMPRESSION in zlib.h.
flto-compression-level=
-Common Joined RejectNegative UInteger Var(flag_lto_compression_level) Init(-1) IntegerRange(0, 9)
--flto-compression-level=<number> Use zlib compression level <number> for IL.
+Common Joined RejectNegative UInteger Var(flag_lto_compression_level) Init(-1) IntegerRange(0, 19)
+-flto-compression-level=<number> Use zlib/zstd compression level <number> for IL.
flto-odr-type-merging
Common Ignore
diff --git a/gcc/lto-compress.c b/gcc/lto-compress.c
index 3287178f257..327ff9e07b7 100644
--- a/gcc/lto-compress.c
+++ b/gcc/lto-compress.c
@@ -35,6 +35,10 @@ along with GCC; see the file COPYING3. If not see
#include "lto-compress.h"
#include "timevar.h"
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
/* Compression stream structure, holds the flush callback and opaque token,
the buffered data, and a note of whether compressing or uncompressing. */
@@ -92,6 +96,95 @@ lto_normalized_zlib_level (void)
return level;
}
+/* Free the buffer and memory associated with STREAM. */
+
+static void
+lto_destroy_compression_stream (struct lto_compression_stream *stream)
+{
+ free (stream->buffer);
+ free (stream);
+}
+
+#ifdef HAVE_ZSTD_H
+/* Return a zstd compression level that zstd will not reject. Normalizes
+ the compression level from the command line flag, clamping non-default
+ values to the appropriate end of their valid range. */
+
+static int
+lto_normalized_zstd_level (void)
+{
+ int level = flag_lto_compression_level;
+
+ if (level != ZSTD_CLEVEL_DEFAULT)
+ {
+ if (level < 1)
+ level = 1;
+ else if (level > ZSTD_maxCLevel ())
+ level = ZSTD_maxCLevel ();
+ }
+
+ return level;
+}
+
+/* Compress STREAM using ZSTD algorithm. */
+
+static void
+lto_compression_zstd (struct lto_compression_stream *stream)
+{
+ unsigned char *cursor = (unsigned char *) stream->buffer;
+ size_t size = stream->bytes;
+
+ timevar_push (TV_IPA_LTO_COMPRESS);
+ size_t const outbuf_length = ZSTD_compressBound (size);
+ char *outbuf = (char *) xmalloc (outbuf_length);
+
+ size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size,
+ lto_normalized_zstd_level ());
+
+ if (ZSTD_isError (csize))
+ internal_error ("compressed stream: %s", ZSTD_getErrorName (csize));
+
+ stream->callback (outbuf, csize, NULL);
+
+ lto_destroy_compression_stream (stream);
+ free (outbuf);
+ timevar_pop (TV_IPA_LTO_COMPRESS);
+}
+
+/* Uncompress STREAM using ZSTD algorithm. */
+
+static bool
+lto_uncompression_zstd (struct lto_compression_stream *stream)
+{
+ unsigned char *cursor = (unsigned char *) stream->buffer;
+ size_t size = stream->bytes;
+
+ timevar_push (TV_IPA_LTO_DECOMPRESS);
+ unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size);
+ if (rsize == ZSTD_CONTENTSIZE_ERROR)
+ {
+ /* The content is probably using zlib. */
+ return false;
+ }
+ else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN)
+ internal_error ("original size unknown");
+
+ char *outbuf = (char *) xmalloc (rsize);
+ size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size);
+
+ if (ZSTD_isError (dsize))
+ internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize));
+
+ stream->callback (outbuf, dsize, stream->opaque);
+
+ lto_destroy_compression_stream (stream);
+ free (outbuf);
+ timevar_pop (TV_IPA_LTO_DECOMPRESS);
+ return true;
+}
+
+#endif
+
/* Create a new compression stream, with CALLBACK flush function passed
OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing. */
@@ -132,15 +225,6 @@ lto_append_to_compression_stream (struct lto_compression_stream *stream,
stream->bytes += num_chars;
}
-/* Free the buffer and memory associated with STREAM. */
-
-static void
-lto_destroy_compression_stream (struct lto_compression_stream *stream)
-{
- free (stream->buffer);
- free (stream);
-}
-
/* Return a new compression stream, with CALLBACK flush function passed
OPAQUE token. */
@@ -163,10 +247,8 @@ lto_compress_block (struct lto_compression_stream *stream,
lto_stats.num_output_il_bytes += num_chars;
}
-/* Finalize STREAM compression, and free stream allocations. */
-
-void
-lto_end_compression (struct lto_compression_stream *stream)
+static void
+lto_compression_zlib (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t remaining = stream->bytes;
@@ -226,6 +308,16 @@ lto_end_compression (struct lto_compression_stream *stream)
timevar_pop (TV_IPA_LTO_COMPRESS);
}
+void
+lto_end_compression (struct lto_compression_stream *stream)
+{
+#ifdef HAVE_ZSTD_H
+ lto_compression_zstd (stream);
+#else
+ lto_compression_zlib (stream);
+#endif
+}
+
/* Return a new uncompression stream, with CALLBACK flush function passed
OPAQUE token. */
@@ -248,14 +340,8 @@ lto_uncompress_block (struct lto_compression_stream *stream,
lto_stats.num_input_il_bytes += num_chars;
}
-/* Finalize STREAM uncompression, and free stream allocations.
-
- Because of the way LTO IL streams are compressed, there may be several
- concatenated compressed segments in the accumulated data, so for this
- function we iterate decompressions until no data remains. */
-
-void
-lto_end_uncompression (struct lto_compression_stream *stream)
+static void
+lto_uncompression_zlib (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t remaining = stream->bytes;
@@ -318,3 +404,14 @@ lto_end_uncompression (struct lto_compression_stream *stream)
free (outbuf);
timevar_pop (TV_IPA_LTO_DECOMPRESS);
}
+
+void
+lto_end_uncompression (struct lto_compression_stream *stream)
+{
+#ifdef HAVE_ZSTD_H
+ bool done = lto_uncompression_zstd (stream);
+ if (done)
+ return;
+#endif
+ lto_uncompression_zlib (stream);
+}
diff --git a/gcc/timevar.def b/gcc/timevar.def
index 13cb470b688..626ce493b76 100644
--- a/gcc/timevar.def
+++ b/gcc/timevar.def
@@ -78,8 +78,8 @@ DEFTIMEVAR (TV_IPA_INLINING , "ipa inlining heuristics")
DEFTIMEVAR (TV_IPA_FNSPLIT , "ipa function splitting")
DEFTIMEVAR (TV_IPA_COMDATS , "ipa comdats")
DEFTIMEVAR (TV_IPA_OPT , "ipa various optimizations")
-DEFTIMEVAR (TV_IPA_LTO_DECOMPRESS , "lto stream inflate")
-DEFTIMEVAR (TV_IPA_LTO_COMPRESS , "lto stream deflate")
+DEFTIMEVAR (TV_IPA_LTO_DECOMPRESS , "lto stream decompression")
+DEFTIMEVAR (TV_IPA_LTO_COMPRESS , "lto stream compression")
DEFTIMEVAR (TV_IPA_LTO_OUTPUT , "lto stream output")
DEFTIMEVAR (TV_IPA_LTO_GIMPLE_IN , "ipa lto gimple in")
DEFTIMEVAR (TV_IPA_LTO_GIMPLE_OUT , "ipa lto gimple out")
--
2.21.0
>From 7294489d924dbe6c01ba5c66e83793757677465f Mon Sep 17 00:00:00 2001
From: Martin Liska <[email protected]>
Date: Thu, 20 Jun 2019 10:08:17 +0200
Subject: [PATCH 1/2] Configure detection for zstd.
---
gcc/Makefile.in | 4 +-
gcc/config.in | 6 +++
gcc/configure | 101 ++++++++++++++++++++++++++++++++++++++++++++++-
gcc/configure.ac | 21 ++++++++++
4 files changed, 129 insertions(+), 3 deletions(-)
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index d9e0885b96b..597dc01328b 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1065,7 +1065,7 @@ BUILD_LIBDEPS= $(BUILD_LIBIBERTY)
LIBS = @LIBS@ libcommon.a $(CPPLIB) $(LIBINTL) $(LIBICONV) $(LIBBACKTRACE) \
$(LIBIBERTY) $(LIBDECNUMBER) $(HOST_LIBS)
BACKENDLIBS = $(ISLLIBS) $(GMPLIBS) $(PLUGINLIBS) $(HOST_LIBS) \
- $(ZLIB)
+ $(ZLIB) $(ZSTD_LIB)
# Any system libraries needed just for GNAT.
SYSLIBS = @GNAT_LIBEXC@
@@ -1076,6 +1076,8 @@ GNATMAKE = @GNATMAKE@
# Libs needed (at present) just for jcf-dump.
LDEXP_LIB = @LDEXP_LIB@
+ZSTD_LIB = @ZSTD_LIB@
+
# Likewise, for use in the tools that must run on this machine
# even if we are cross-building GCC.
BUILD_LIBS = $(BUILD_LIBIBERTY)
diff --git a/gcc/config.in b/gcc/config.in
index a718ceaf3da..13fd7959dd7 100644
--- a/gcc/config.in
+++ b/gcc/config.in
@@ -1926,6 +1926,12 @@
#endif
+/* Define if you have a working <zstd.h> header file. */
+#ifndef USED_FOR_TARGET
+#undef HAVE_ZSTD_H
+#endif
+
+
/* Define if isl is in use. */
#ifndef USED_FOR_TARGET
#undef HAVE_isl
diff --git a/gcc/configure b/gcc/configure
index 955e9ccc09b..9c3439c156b 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -782,6 +782,7 @@ manext
LIBICONV_DEP
LTLIBICONV
LIBICONV
+ZSTD_LIB
DL_LIB
LDEXP_LIB
EXTRA_GCC_LIBS
@@ -9798,6 +9799,70 @@ DL_LIB="$LIBS"
LIBS="$save_LIBS"
+# LTO can use zstd compression algorithm
+save_LIBS="$LIBS"
+LIBS=
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing ZSTD_compress" >&5
+$as_echo_n "checking for library containing ZSTD_compress... " >&6; }
+if ${ac_cv_search_ZSTD_compress+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ZSTD_compress ();
+int
+main ()
+{
+return ZSTD_compress ();
+ ;
+ return 0;
+}
+_ACEOF
+for ac_lib in '' zstd; do
+ if test -z "$ac_lib"; then
+ ac_res="none required"
+ else
+ ac_res=-l$ac_lib
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
+ fi
+ if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_search_ZSTD_compress=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext
+ if ${ac_cv_search_ZSTD_compress+:} false; then :
+ break
+fi
+done
+if ${ac_cv_search_ZSTD_compress+:} false; then :
+
+else
+ ac_cv_search_ZSTD_compress=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZSTD_compress" >&5
+$as_echo "$ac_cv_search_ZSTD_compress" >&6; }
+ac_res=$ac_cv_search_ZSTD_compress
+if test "$ac_res" != no; then :
+ test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+fi
+
+ZSTD_LIB="$LIBS"
+LIBS="$save_LIBS"
+
+
+
# Use <inttypes.h> only if it exists,
# doesn't clash with <sys/types.h>, declares intmax_t and defines
# PRId64
@@ -9838,6 +9903,38 @@ $as_echo "#define HAVE_INTTYPES_H 1" >>confdefs.h
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for zstd.h" >&5
+$as_echo_n "checking for zstd.h... " >&6; }
+if ${gcc_cv_header_zstd_h+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <zstd.h>
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_compile "$LINENO"; then :
+ gcc_cv_header_zstd_h=yes
+else
+ gcc_cv_header_zstd_h=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_header_zstd_h" >&5
+$as_echo "$gcc_cv_header_zstd_h" >&6; }
+if test $gcc_cv_header_zstd_h = yes; then
+
+$as_echo "#define HAVE_ZSTD_H 1" >>confdefs.h
+
+fi
+
for ac_func in times clock kill getrlimit setrlimit atoq \
@@ -18655,7 +18752,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 18658 "configure"
+#line 18755 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -18761,7 +18858,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 18764 "configure"
+#line 18861 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
diff --git a/gcc/configure.ac b/gcc/configure.ac
index 066a6f4c958..3ee4fd2e2ee 100644
--- a/gcc/configure.ac
+++ b/gcc/configure.ac
@@ -1237,6 +1237,15 @@ DL_LIB="$LIBS"
LIBS="$save_LIBS"
AC_SUBST(DL_LIB)
+# LTO can use zstd compression algorithm
+save_LIBS="$LIBS"
+LIBS=
+AC_SEARCH_LIBS(ZSTD_compress, zstd)
+ZSTD_LIB="$LIBS"
+LIBS="$save_LIBS"
+AC_SUBST(ZSTD_LIB)
+
+
# Use <inttypes.h> only if it exists,
# doesn't clash with <sys/types.h>, declares intmax_t and defines
# PRId64
@@ -1258,6 +1267,18 @@ if test $gcc_cv_header_inttypes_h = yes; then
[Define if you have a working <inttypes.h> header file.])
fi
+AC_MSG_CHECKING(for zstd.h)
+AC_CACHE_VAL(gcc_cv_header_zstd_h,
+[AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+[[#include <zstd.h>]])],
+ [gcc_cv_header_zstd_h=yes],
+ [gcc_cv_header_zstd_h=no])])
+AC_MSG_RESULT($gcc_cv_header_zstd_h)
+if test $gcc_cv_header_zstd_h = yes; then
+ AC_DEFINE(HAVE_ZSTD_H, 1,
+ [Define if you have a working <zstd.h> header file.])
+fi
+
dnl Disabled until we have a complete test for buggy enum bitfields.
dnl gcc_AC_C_ENUM_BF_UNSIGNED
--
2.21.0
