[buildstream] branch master updated: Raise ContentTooShortError if we don't get enough bytes

[akitouni]

This is an automated email from the ASF dual-hosted git repository.

akitouni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/buildstream.git


The following commit(s) were added to refs/heads/master by this push:
     new 1993f0bec Raise ContentTooShortError if we don't get enough bytes
     new 52d0899c6 Merge pull request #1846 from nanonyme/nanonyme/download
1993f0bec is described below

commit 1993f0beccff80c757aa70a6d1d06de2faa0432f
Author: Seppo Yli-Olli <seppo.ylio...@gmail.com>
AuthorDate: Sun Jun 11 23:41:45 2023 +0300

    Raise ContentTooShortError if we don't get enough bytes
    
    This can happen if connection is aborted before we finish the
    download. urlretrieve higher level API would handle this but
    we don't use it.
---
 src/buildstream/downloadablefilesource.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/buildstream/downloadablefilesource.py 
b/src/buildstream/downloadablefilesource.py
index e31033506..3c7e05763 100644
--- a/src/buildstream/downloadablefilesource.py
+++ b/src/buildstream/downloadablefilesource.py
@@ -107,6 +107,7 @@ def _download_file(opener_creator, url, etag, directory):
                 return None, None, None
 
             etag = info["ETag"]
+            length = info.get("Content-Length")
 
             filename = info.get_filename(default_name)
             filename = os.path.basename(filename)
@@ -114,6 +115,10 @@ def _download_file(opener_creator, url, etag, directory):
             with open(local_file, "wb") as dest:
                 shutil.copyfileobj(response, dest)
 
+                actual_length = dest.tell()
+                if length and actual_length < int(length):
+                    raise ValueError(f"Partial file {actual_length}/{length}")
+
     except urllib.error.HTTPError as e:
         if e.code == 304:
             # 304 Not Modified.
@@ -122,7 +127,7 @@ def _download_file(opener_creator, url, etag, directory):
             return None, None, None
 
         return None, None, str(e)
-    except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError, 
ValueError) as e:
+    except (urllib.error.URLError, OSError, ValueError) as e:
         # Note that urllib.request.Request in the try block may throw a
         # ValueError for unknown url types, so we handle it here.
         return None, None, str(e)