This is an automated email from the ASF dual-hosted git repository. tomaz pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit 3d16e9e1ed24bbdd4984a5fd53cbda8f261e036a Author: Tomaz Muraus <[email protected]> AuthorDate: Sat Jul 11 18:03:42 2020 +0200 Update docs and location from which we retrieve pricing data. --- docs/compute/pricing.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ libcloud/pricing.py | 8 +++++--- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/docs/compute/pricing.rst b/docs/compute/pricing.rst index a01b879..1511dd3 100644 --- a/docs/compute/pricing.rst +++ b/docs/compute/pricing.rst @@ -37,6 +37,50 @@ JSON file (``data/pricing.json``) which is bundled with each release. This pricing data is only updated once you install a new release which means it could be out of date. +Downloading latest pricing data from an S3 Bucket +------------------------------------------------- + +Since July 2020, we now run a daily job as part of our CI/CD system which +scrapes pricing data for various providers and publishes pricing data to a +public read-only S3 bucket. + +Pricing file data is available at the following locations: + +* https://libcloud-pricing-data.s3.amazonaws.com/pricing.json +* https://libcloud-pricing-data.s3.amazonaws.com/pricing.json.sha256 +* https://libcloud-pricing-data.s3.amazonaws.com/pricing.json.sha512 + +First file contains actual pricing JSON file and the second and third contain +SHA 256 and SHA 512 sum of that file content. + +You can use the content of the sha sum files to implement efficient file +downloads and only download pricing.json file if the content has changed. + +You can do that by fetching the sha sum file, caching the sha sum and only +downloading ``pricing.json`` file is the sha sum value has changed. + +An alternative to using the content of the sha sum file is caching the value +of the ``ETag`` HTTP response header which you can retrieve by issuing HTTP +``HEAD`` request against the ``pricing.json`` URL. HEAD request will only +return the object metdata without the actual content. + +For example: + +.. sourcecode:: bash + + curl --head https://libcloud-pricing-data.s3.amazonaws.com/pricing.json + + HTTP/1.1 200 OK + x-amz-id-2: c8Mer3VtRYWGeKtKlbgwebn3BsVQt+Z/WKKPjk3NcsRSK23BzE6OQDIogzIR2oJGJRmOtS4ydjA= + x-amz-request-id: 9A790A3B3587478D + Date: Sat, 11 Jul 2020 16:01:39 GMT + Last-Modified: Sat, 11 Jul 2020 15:55:50 GMT + ETag: "e46324663d76dedafc7d9b09537b18a7" + Accept-Ranges: bytes + Content-Type: application/json + Content-Length: 549390 + Server: AmazonS3 + .. _using-custom-pricing-file: Using a custom pricing file diff --git a/libcloud/pricing.py b/libcloud/pricing.py index 695c1c6..70f1fb9 100644 --- a/libcloud/pricing.py +++ b/libcloud/pricing.py @@ -43,8 +43,10 @@ __all__ = [ 'download_pricing_file' ] -# Default URL to the pricing file -DEFAULT_FILE_URL = 'https://git-wip-us.apache.org/repos/asf?p=libcloud.git;a=blob_plain;f=libcloud/data/pricing.json' # NOQA +# Default URL to the pricing file in a git repo +DEFAULT_FILE_URL_GIT = 'https://git-wip-us.apache.org/repos/asf?p=libcloud.git;a=blob_plain;f=libcloud/data/pricing.json' # NOQA + +DEFAULT_FILE_URL_S3_BUCKET = 'https://libcloud-pricing-data.s3.amazonaws.com/pricing.json' # NOQA CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) DEFAULT_PRICING_FILE_PATH = pjoin(CURRENT_DIRECTORY, 'data/pricing.json') @@ -190,7 +192,7 @@ def invalidate_module_pricing_cache(driver_type, driver_name): del PRICING_DATA[driver_type][driver_name] -def download_pricing_file(file_url=DEFAULT_FILE_URL, +def download_pricing_file(file_url=DEFAULT_FILE_URL_S3_BUCKET, file_path=CUSTOM_PRICING_FILE_PATH): """ Download pricing file from the file_url and save it to file_path.
