Re: [Mesa-dev] [PATCH 1/2] intel: genxml: compress all gen files into one

Lionel Landwerlin Fri, 31 Mar 2017 03:04:21 -0700

Hi,

Sorry I broke the out of tree build. I have a fix coming up.


Thanks,

-
Lionel

On 31/03/17 02:06, Mike Lothian wrote:

This prevents me building master

PYTHONPATH=/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/compiler/nir/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/compiler/brw_nir_trig_workarounds.py> compiler/brw_nir_trig_workarounds.c || (rm -f compi

ler/brw_nir_trig_workarounds.c; false)
/bin/mkdir -p genxml

/bin/mkdir -p genxml
/bin/mkdir -p genxml

/bin/mkdir -p genxml

/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen_pack_header.py/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen6.xml> genxml/gen6_pack.h || (rm -f genxml/gen6_pack.h; false)/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen_pack_header.py/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen7.xml> genxml/gen7_pack.h || (rm -f genxml/gen7_pack.h; false)/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen_pack_header.py/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen75.xml> genxml/gen75_pack.h || (rm -f genxml/gen75_pack.h; false)

/bin/mkdir -p genxml
/bin/mkdir -p genxml

/bin/mkdir -p genxml
/bin/mkdir -p genxml

/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen_bits_header.py-o genxml/genX_bits.h/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen4.xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen45.xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen5.xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen6.xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen7.xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen75.xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen8.xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen9.x

ml

/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen_zipped_file.pygenxml/gen4.xml genxml/gen45.xml genxml/gen5.xml genxml/gen6.xmlgenxml/gen7.xml genxml/gen75.xml genxml/gen8.xml genxml/gen9.xml >genxml/genX_xm

l.h || (rm -f genxml/genX_xml.h; false)
/bin/mkdir -p isl
/bin/mkdir -p vulkan

/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/isl/gen_format_layout.py\--csv/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/isl/isl_format_layout.csv--out isl/isl_format_layout.c/usr/bin/python2.7/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/vulkan/anv_entrypoints_gen.py\--xml/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/vulkan/registry/vk.xml--outdir ./vulkan

Traceback (most recent call last):

File"/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen_zipped_file.py",line 71, in <module>

   main()

File"/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999/src/intel/genxml/gen_zipped_file.py",line 48, in main

   xml = open(filename).read()
IOError: [Errno 2] No such file or directory: 'genxml/gen4.xml'
make[3]: *** [Makefile:4275: genxml/genX_xml.h] Error 1
make[3]: *** Waiting for unfinished jobs....

make[3]: Leaving directory'/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999-abi_x86_32.x86/src/intel'

make[2]: *** [Makefile:852: all-recursive] Error 1

make[2]: Leaving directory'/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999-abi_x86_32.x86/src'

make[1]: *** [Makefile:643: all] Error 2

make[1]: Leaving directory'/var/tmp/portage/media-libs/mesa-9999/work/mesa-9999-abi_x86_32.x86/src'

make: *** [Makefile:643: all-recursive] Error 1
* ERROR: media-libs/mesa-9999::gentoo failed (compile phase):
*   emake failed
*

* If you need support, post the output of `emerge --info'=media-libs/mesa-9999::gentoo'`,

* the complete build log and the output of `emerge -pqv '=med

On Thu, 30 Mar 2017 at 19:56 Jordan Justen <[email protected]<mailto:[email protected]>> wrote:


    Series Reviewed-by: Jordan Justen <[email protected]
    <mailto:[email protected]>>

    On 2017-03-25 14:57:15, Lionel Landwerlin wrote:
    > Combining all the files into a single string didn't make any
    > difference in the size of the aubinator binary.
    >
    > With this change we now also embed gen4/4.5/5 descriptions, which
    > increases the aubinator size by ~16Kb.
    >
    > Signed-off-by: Lionel Landwerlin <[email protected]
    <mailto:[email protected]>>
    > ---
    >  src/intel/Makefile.genxml.am <http://Makefile.genxml.am>     |
    10 +++---
    >  src/intel/Makefile.sources          |  7 -----
    >  src/intel/common/gen_decoder.c      | 62
    +++++++++++++------------------------
    >  src/intel/genxml/.gitignore         |  2 +-
    >  src/intel/genxml/gen_zipped_file.py | 34 +++++++++++++++++---
    >  5 files changed, 56 insertions(+), 59 deletions(-)
    >
    > diff --git a/src/intel/Makefile.genxml.am
    <http://Makefile.genxml.am> b/src/intel/Makefile.genxml.am
    <http://Makefile.genxml.am>
    > index 01a02b63b4..c5cc843191 100644
    > --- a/src/intel/Makefile.genxml.am <http://Makefile.genxml.am>
    > +++ b/src/intel/Makefile.genxml.am <http://Makefile.genxml.am>
    > @@ -21,12 +21,12 @@
    >
    >  BUILT_SOURCES += \
    >         $(GENXML_GENERATED_FILES) \
    > -       $(AUBINATOR_GENERATED_FILES)
    > +       genxml/genX_xml.h
    >
    >  EXTRA_DIST += \
    >         $(GENXML_XML_FILES) \
    >         $(GENXML_GENERATED_FILES) \
    > -       $(AUBINATOR_GENERATED_FILES)
    > +       genxml/genX_xml.h
    >
    >  SUFFIXES = _pack.h _xml.h .xml
    >
    > @@ -36,11 +36,9 @@ $(GENXML_GENERATED_FILES):
    genxml/gen_pack_header.py
    >         $(MKDIR_GEN)
    >         $(PYTHON_GEN) $(srcdir)/genxml/gen_pack_header.py $< >
    $@ || ($(RM) $@; false)
    >
    > -$(AUBINATOR_GENERATED_FILES): genxml/gen_zipped_file.py
    > -
    > -.xml_xml.h:
    > +genxml/genX_xml.h: $(GENXML_XML_FILES) genxml/gen_zipped_file.py
    >         $(MKDIR_GEN)
    > -       $(AM_V_GEN) $(PYTHON2)
    $(srcdir)/genxml/gen_zipped_file.py $< > $@ || ($(RM) $@; false)
    > +       $(AM_V_GEN) $(PYTHON2)
    $(srcdir)/genxml/gen_zipped_file.py $(GENXML_XML_FILES) > $@
    >
    >  EXTRA_DIST += \
    >         genxml/genX_pack.h \
    > diff --git a/src/intel/Makefile.sources b/src/intel/Makefile.sources
    > index 88bcf60f6e..b5992c8d35 100644
    > --- a/src/intel/Makefile.sources
    > +++ b/src/intel/Makefile.sources
    > @@ -129,13 +129,6 @@ GENXML_GENERATED_FILES = \
    >         genxml/gen8_pack.h \
    >         genxml/gen9_pack.h
    >
    > -AUBINATOR_GENERATED_FILES = \
    > -       genxml/gen6_xml.h \
    > -       genxml/gen7_xml.h \
    > -       genxml/gen75_xml.h \
    > -       genxml/gen8_xml.h \
    > -       genxml/gen9_xml.h
    > -
    >  ISL_FILES = \
    >         isl/isl.c \
    >         isl/isl.h \
    > diff --git a/src/intel/common/gen_decoder.c
    b/src/intel/common/gen_decoder.c
    > index 1c3246f265..7b04ac051b 100644
    > --- a/src/intel/common/gen_decoder.c
    > +++ b/src/intel/common/gen_decoder.c
    > @@ -34,11 +34,7 @@
    >
    >  #include "gen_decoder.h"
    >
    > -#include "genxml/gen6_xml.h"
    > -#include "genxml/gen7_xml.h"
    > -#include "genxml/gen75_xml.h"
    > -#include "genxml/gen8_xml.h"
    > -#include "genxml/gen9_xml.h"
    > +#include "genxml/genX_xml.h"
    >
    >  #define XML_BUFFER_SIZE 4096
    >
    > @@ -481,35 +477,6 @@ devinfo_to_gen(const struct gen_device_info
    *devinfo)
    >     return value;
    >  }
    >
    > -static const struct {
    > -   int gen;
    > -   const uint8_t *data;
    > -   size_t data_length;
    > -} gen_data[] = {
    > -   { .gen = 60, .data = gen6_xml, .data_length =
    sizeof(gen6_xml) },
    > -   { .gen = 70, .data = gen7_xml, .data_length =
    sizeof(gen7_xml) },
    > -   { .gen = 75, .data = gen75_xml, .data_length =
    sizeof(gen75_xml) },
    > -   { .gen = 80, .data = gen8_xml, .data_length =
    sizeof(gen8_xml) },
    > -   { .gen = 90, .data = gen9_xml, .data_length = sizeof(gen9_xml) }
    > -};
    > -
    > -static const uint8_t *
    > -devinfo_to_xml_data(const struct gen_device_info *devinfo,
    > -                    uint32_t *data_length)
    > -{
    > -   int i, gen = devinfo_to_gen(devinfo);
    > -
    > -   for (i = 0; i < ARRAY_SIZE(gen_data); i++) {
    > -      if (gen_data[i].gen == gen) {
    > -         *data_length = gen_data[i].data_length;
    > -         return gen_data[i].data;
    > -      }
    > -   }
    > -
    > -   unreachable("Unknown generation");
    > -   return NULL;
    > -}
    > -
    >  static uint32_t zlib_inflate(const void *compressed_data,
    >                               uint32_t compressed_len,
    >                               void **out_ptr)
    > @@ -563,9 +530,22 @@ gen_spec_load(const struct gen_device_info
    *devinfo)
    >  {
    >     struct parser_context ctx;
    >     void *buf;
    > -   const void *zlib_data;
    > -   void *text_data;
    > -   uint32_t zlib_length = 0, text_length;
    > +   uint8_t *text_data;
    > +   uint32_t text_offset = 0, text_length = 0, total_length;
    > +   uint32_t gen_10 = devinfo_to_gen(devinfo);
    > +
    > +   for (int i = 0; i < ARRAY_SIZE(genxml_files_table); i++) {
    > +      if (genxml_files_table[i].gen_10 == gen_10) {
    > +         text_offset = genxml_files_table[i].offset;
    > +         text_length = genxml_files_table[i].length;
    > +         break;
    > +      }
    > +   }
    > +
    > +   if (text_length == 0) {
    > +      fprintf(stderr, "unable to find gen (%u) data\n", gen_10);
    > +      return NULL;
    > +   }
    >
    >     memset(&ctx, 0, sizeof ctx);
    >     ctx.parser = XML_ParserCreate(NULL);
    > @@ -580,11 +560,13 @@ gen_spec_load(const struct gen_device_info
    *devinfo)
    >
    >     ctx.spec = xzalloc(sizeof(*ctx.spec));
    >
    > -   zlib_data = devinfo_to_xml_data(devinfo, &zlib_length);
    > -   text_length = zlib_inflate(zlib_data, zlib_length, &text_data);
    > +   total_length = zlib_inflate(compress_genxmls,
    > +                               sizeof(compress_genxmls),
    > +                               (void **) &text_data);
    > +   assert(text_offset + text_length <= total_length);
    >
    >     buf = XML_GetBuffer(ctx.parser, text_length);
    > -   memcpy(buf, text_data, text_length);
    > +   memcpy(buf, &text_data[text_offset], text_length);
    >
    >     if (XML_ParseBuffer(ctx.parser, text_length, true) == 0) {
    >        fprintf(stderr,
    > diff --git a/src/intel/genxml/.gitignore
    b/src/intel/genxml/.gitignore
    > index c5672b5595..4385170899 100644
    > --- a/src/intel/genxml/.gitignore
    > +++ b/src/intel/genxml/.gitignore
    > @@ -1,2 +1,2 @@
    >  gen*_pack.h
    > -gen*_xml.h
    > +genX_xml.h
    > diff --git a/src/intel/genxml/gen_zipped_file.py
    b/src/intel/genxml/gen_zipped_file.py
    > index 66222cabe7..af2008bea0 100644
    > --- a/src/intel/genxml/gen_zipped_file.py
    > +++ b/src/intel/genxml/gen_zipped_file.py
    > @@ -26,22 +26,46 @@ from __future__ import print_function
    >  import os
    >  import sys
    >  import zlib
    > +import xml.etree.cElementTree as et
    >
    >  def main():
    >      if len(sys.argv) < 2:
    >          print("No input xml file specified")
    >          sys.exit(1)
    >
    > -    with open(sys.argv[1]) as f:
    > -        compressed_data = zlib.compress(f.read())
    > +    compress = zlib.compressobj()
    >
    > -    gen_name = os.path.splitext(os.path.basename(sys.argv[1]))[0]
    > -    print("static const uint8_t %s_xml[] = {" % gen_name)
    > -    print("   ", end='')
    > +    print("static const struct {")
    > +    print("   uint32_t gen_10;")
    > +    print("   uint32_t offset;")
    > +    print("   uint32_t length;")
    > +    print("} genxml_files_table[] = {")
    > +
    > +    xml_offset = 0
    > +    compressed_data = ''
    > +    for i in range(1, len(sys.argv)):
    > +        filename = sys.argv[i]
    > +        xml = open(filename).read()
    > +        xml_length = len(xml)
    > +        root = et.fromstring(xml)
    > +
    > +        print("   { %i, %i, %i }," %
    > +              (int(float(root.attrib['gen']) * 10), xml_offset,
    xml_length))
    > +
    > +        compressed_data += compress.compress(xml)
    > +        xml_offset += xml_length
    >
    > +    print("};")
    > +
    > +    compressed_data += compress.flush()
    > +
    > +    print("")
    > +    print("static const uint8_t compress_genxmls[] = {")
    > +    print("   ", end='')
    >      for i, c in enumerate(compressed_data, start=1):
    >          print("0x%.2x, " % ord(c), end='\n   ' if not i % 12
    else '')
    >      print('\n};')
    >
    > +
    >  if __name__ == '__main__':
    >      main()
    > --
    > 2.11.0
    >
    > _______________________________________________
    > mesa-dev mailing list
    > [email protected]
    <mailto:[email protected]>
    > https://lists.freedesktop.org/mailman/listinfo/mesa-dev
    _______________________________________________
    mesa-dev mailing list
    [email protected] <mailto:[email protected]>
    https://lists.freedesktop.org/mailman/listinfo/mesa-dev

_______________________________________________
mesa-dev mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/mesa-dev

Re: [Mesa-dev] [PATCH 1/2] intel: genxml: compress all gen files into one

Reply via email to