For some recipes (llvm, chromium, chromium-wayland) I've noticed this function to fail with:
ERROR: llvm3.3-3.3-r0 do_unpack: Error executing a python function in exec_python_func() autogenerated: The stack trace of python calls that resulted in this exception/failure was: File: 'exec_python_func() autogenerated', lineno: 2, function: <module> 0001: *** 0002:base_do_unpack(d) 0003: File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass', lineno: 215, function: base_do_unpack 0211: 0212: try: 0213: fetcher = bb.fetch2.Fetch(src_uri, d) 0214: fetcher.unpack(d.getVar('WORKDIR')) *** 0215: create_src_date_epoch_stamp(d) 0216: except bb.fetch2.BBFetchException as e: 0217: bb.fatal(str(e)) 0218:} 0219: File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass', lineno: 34, function: create_src_date_epoch_stamp 0030: exclude = set(["temp", "licenses", "patches", "recipe-sysroot-native", "recipe-sysroot" ]) 0031: for root, dirs, files in os.walk(path, topdown=True): 0032: dirs[:] = [d for d in dirs if d not in exclude] 0033: if root.endswith('/git'): *** 0034: src_date_epoch = get_git_src_date_epoch(d, root) 0035: break 0036: 0037: for fname in files: 0038: filename = os.path.join(root, fname) File: '/home/jenkins/oe/world/shr-core/openembedded-core/meta/classes/base.bbclass', lineno: 17, function: get_git_src_date_epoch 0013:def get_git_src_date_epoch(d, path): 0014: import subprocess 0015: saved_cwd = os.getcwd() 0016: os.chdir(path) *** 0017: src_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'])) 0018: os.chdir(saved_cwd) 0019: return src_date_epoch 0020: 0021:def create_src_date_epoch_stamp(d): File: '/usr/lib/python3.5/subprocess.py', lineno: 626, function: check_output 0622: # empty string. That is maintained here for backwards compatibility. 0623: kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b'' 0624: 0625: return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, *** 0626: **kwargs).stdout 0627: 0628: 0629:class CompletedProcess(object): 0630: """A process that has finished running. File: '/usr/lib/python3.5/subprocess.py', lineno: 708, function: run 0704: raise 0705: retcode = process.poll() 0706: if check and retcode: 0707: raise CalledProcessError(retcode, process.args, *** 0708: output=stdout, stderr=stderr) 0709: return CompletedProcess(process.args, retcode, stdout, stderr) 0710: 0711: 0712:def list2cmdline(seq): Exception: subprocess.CalledProcessError: Command '['git', 'log', '-1', '--pretty=%ct']' returned non-zero exit status 128 ERROR: llvm3.3-3.3-r0 do_unpack: Function failed: base_do_unpack ERROR: Logfile of failure stored in: /home/jenkins/oe/world/shr-core/tmp-glibc/work/i586-oe-linux/llvm3.3/3.3-r0/temp/log.do_unpack.25005 NOTE: recipe llvm3.3-3.3-r0: task do_unpack: Failed ERROR: Task (/home/jenkins/oe/world/shr-core/meta-openembedded/meta-oe/recipes-core/llvm/llvm3.3_3.3.bb:do_unpack) failed with exit code '1' Maybe I don't have the latest version of this patch (I'm not using your poky-contrib branch yet), but it should fail with nicer message when git log fails for whatever reason. On Mon, May 1, 2017 at 10:59 PM, Juro Bystricky <juro.bystri...@intel.com> wrote: > Conditionally set some environment variables in order to achieve > improved binary reproducibility. Providing BUILD_REPRODUCIBLE_BINARIES is > set to "1", we set the following environment variables: > > export PYTHONHASHSEED=0 > export PERL_HASH_SEED=0 > export TZ="UTC" > > We also export and set SOURCE_DATE_EPOCH. The value for this variable > is obtained after source code for a recipe has been unpacked, but before > it is > patched. If the code comes from a GIT repo, we get the timestamp from the > top > commit. (This usually corresponds to the mktime of "changelog".) > Otherwise we go through all files and get the timestamp from the youngest > one. We create a timestamp for each recipe. The timestamp is stored in the > file > 'src_date_epoch.txt'. Later on, each task reads this file and sets > SOURCE_DATE_EPOCH > based on the value found in the file. > > [YOCTO#11178] > [YOCTO#11179] > > Signed-off-by: Juro Bystricky <juro.bystri...@intel.com> > --- > meta/classes/base.bbclass | 82 ++++++++++++++++++++++++++++++ > +++++++++++++++++ > 1 file changed, 82 insertions(+) > > diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass > index e29821f..f2b2d97 100644 > --- a/meta/classes/base.bbclass > +++ b/meta/classes/base.bbclass > @@ -10,6 +10,52 @@ inherit utility-tasks > inherit metadata_scm > inherit logging > > +def get_git_src_date_epoch(d, path): > + import subprocess > + saved_cwd = os.getcwd() > + os.chdir(path) > + src_date_epoch = int(subprocess.check_output([' > git','log','-1','--pretty=%ct'])) > + os.chdir(saved_cwd) > + return src_date_epoch > + > +def create_src_date_epoch_stamp(d): > + if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1': > + path = d.getVar('S') > + src_date_epoch = 0 > + filename_dbg = None > + > + if path.endswith('/git'): > + src_date_epoch = get_git_src_date_epoch(d, path) > + else: > + exclude = set(["temp", "licenses", "patches", > "recipe-sysroot-native", "recipe-sysroot" ]) > + for root, dirs, files in os.walk(path, topdown=True): > + dirs[:] = [d for d in dirs if d not in exclude] > + if root.endswith('/git'): > + src_date_epoch = get_git_src_date_epoch(d, root) > + break > + > + for fname in files: > + filename = os.path.join(root, fname) > + try: > + mtime = int(os.path.getmtime(filename)) > + except: > + mtime = 0 > + if mtime > src_date_epoch: > + src_date_epoch = mtime > + filename_dbg = filename > + > + # Most likely an empty folder > + if src_date_epoch == 0: > + bb.warn("Unable to determine src_date_epoch! path:%s" % path) > + > + f = open(os.path.join(path,'src_date_epoch.txt'), 'w') > + f.write(str(src_date_epoch)) > + f.close() > + > + if filename_dbg != None: > + bb.debug(1," src_date_epoch %d derived from: %s" % > (src_date_epoch, filename_dbg)) > + > + > OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package > oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license" > OE_IMPORTS[type] = "list" > > @@ -173,6 +219,7 @@ python base_do_unpack() { > try: > fetcher = bb.fetch2.Fetch(src_uri, d) > fetcher.unpack(d.getVar('WORKDIR')) > + create_src_date_epoch_stamp(d) > except bb.fetch2.BBFetchException as e: > bb.fatal(str(e)) > } > @@ -383,9 +430,43 @@ def set_packagetriplet(d): > > settriplet(d, "PKGMLTRIPLETS", archs, tos, tvs) > > + > +export PYTHONHASHSEED > +export PERL_HASH_SEED > +export SOURCE_DATE_EPOCH > + > +BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH PYTHONHASHSEED PERL_HASH_SEED > " > + > python () { > import string, re > > + # Create reproducible_environment > + > + if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1': > + import subprocess > + d.setVar('PYTHONHASHSEED', '0') > + d.setVar('PERL_HASH_SEED', '0') > + d.setVar('TZ', 'UTC') > + > + path = d.getVar('S') > + epochfile = os.path.join(path,'src_date_epoch.txt') > + if os.path.isfile(epochfile): > + f = open(epochfile, 'r') > + src_date_epoch = f.read() > + f.close() > + bb.debug(1, "src_date_epoch stamp found ---> stamp %s" % > src_date_epoch) > + d.setVar('SOURCE_DATE_EPOCH', src_date_epoch) > + else: > + bb.debug(1, "src_date_epoch stamp not found.") > + d.setVar('SOURCE_DATE_EPOCH', '0') > + else: > + if 'PYTHONHASHSEED' in os.environ: > + del os.environ['PYTHONHASHSEED'] > + if 'PERL_HASH_SEED' in os.environ: > + del os.environ['PERL_HASH_SEED'] > + if 'SOURCE_DATE_EPOCH' in os.environ: > + del os.environ['SOURCE_DATE_EPOCH'] > + > # Handle PACKAGECONFIG > # > # These take the form: > @@ -678,6 +759,7 @@ python () { > bb.warn("Recipe %s is marked as only being architecture > specific but seems to have machine specific packages?! The recipe may as > well mark itself as machine specific directly." % d.getVar("PN")) > } > > + > addtask cleansstate after do_clean > python do_cleansstate() { > sstate_clean_cachefiles(d) > -- > 2.7.4 > >
-- _______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core