Repository: libcloud Updated Branches: refs/heads/trunk 23127e387 -> c07dbe8c3
[google compute] add pricing data update script Also used the script to update `pricing.json` with current prices on a per-region basis. Added numeric sorting to the JSON output; as a result, all other pricing is now sorted numerically, e.g., "12GB" sorts after "3GB" and "150" sorts after "20" whereas previously, it would be the opposite. Closes #464 Signed-off-by: Eric Johnson <erjoh...@google.com> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/c07dbe8c Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/c07dbe8c Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/c07dbe8c Branch: refs/heads/trunk Commit: c07dbe8c359b8a1ac760c2b168ffbd4f00dca952 Parents: 23127e3 Author: Misha Brukman <mbruk...@google.com> Authored: Tue Feb 17 12:00:16 2015 -0500 Committer: Eric Johnson <erjoh...@google.com> Committed: Mon Nov 9 15:34:11 2015 +0000 ---------------------------------------------------------------------- CHANGES.rst | 4 + contrib/update_google_prices.py | 89 ++++++++++++++++++++ contrib/utils.py | 48 +++++++++++ contrib/utils_test.py | 68 +++++++++++++++ libcloud/data/pricing.json | 158 ++++++++++++++++++++++++++++++----- 5 files changed, 345 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index ebc76e3..39db15a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,6 +26,10 @@ General Compute ~~~~~~~ +- [google compute] add pricing data update script + (GITHUB-464) + [Misha Brukman] + - Fix a bug in the ``list_volumes`` method in the CloudStack driver so it returns an empty list if no volumes are found. (GITHUB-617) http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/contrib/update_google_prices.py ---------------------------------------------------------------------- diff --git a/contrib/update_google_prices.py b/contrib/update_google_prices.py new file mode 100755 index 0000000..e69ffe1 --- /dev/null +++ b/contrib/update_google_prices.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Loads Google Cloud Platform prices and updates the `pricing.json` data file. +""" + +import os +try: + import simplejson as json +except ImportError: + import json +import sys +import time +import urllib2 +import utils + +BASE_PATH = os.path.dirname(os.path.abspath(__file__)) +PRICING_FILE_PATH = os.path.join(BASE_PATH, '../libcloud/data/pricing.json') +PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH) + +GOOGLE_CLOUD_PRICES = 'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json' + + +def main(argv): + # Read the current pricing data. + libcloud_data = {} + with open(PRICING_FILE_PATH, 'r') as libcloud_in: + libcloud_data = json.loads(libcloud_in.read()) + + # Download the current Google Cloud Platform pricing. + req = urllib2.Request(GOOGLE_CLOUD_PRICES, '') + google_ext_prices = json.loads(urllib2.urlopen(req).read()) + if 'gcp_price_list' not in google_ext_prices: + sys.stderr.write('Google Cloud pricing data missing "gcp_price_list" node\n') + sys.exit(1) + + # This is a map from regions used in the pricing JSON file to the regions as + # reflected in the Google Cloud Platform documentation and APIs. + pricing_to_region = { + 'us': 'us', + 'eu': 'europe', # alias for 'europe' + 'europe': 'europe', + 'apac': 'asia', # alias for 'asia' + 'asia': 'asia' + } + + # Initialize Google Cloud Platform regions. + for _, region in pricing_to_region.iteritems(): + libcloud_data['compute']['google_%s' % region] = {} + + # Update Google Compute Engine pricing. + gcp_price_list = google_ext_prices['gcp_price_list'] + gce_vm_prefix = 'CP-COMPUTEENGINE-VMIMAGE-' + for name, prices in gcp_price_list.iteritems(): + if not name.startswith(gce_vm_prefix): + continue + short_name = name[len(gce_vm_prefix):] + machine_type = short_name.lower() + for key, price in prices.iteritems(): + if key in pricing_to_region: + region = pricing_to_region[key] + libcloud_data['compute']['google_%s' % region][machine_type] = price + + # Update last-modified timestamp. + libcloud_data['updated'] = int(time.time()) + + # Write updated price list. + with open(PRICING_FILE_PATH, 'w') as libcloud_out: + json_str = json.dumps(libcloud_data, indent=4 * ' ', + item_sort_key=utils.sortKeysNumerically) + libcloud_out.write(json_str) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/contrib/utils.py ---------------------------------------------------------------------- diff --git a/contrib/utils.py b/contrib/utils.py new file mode 100644 index 0000000..2028721 --- /dev/null +++ b/contrib/utils.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +import re + + +def splitStringWithNumbers(string): + """Splits input string into a list of items, numeric and otherwise. + + Returns a list of values, each either an interpreted number, or a substring + of the original input. + + E.g., 'abc-123-def' => ['abc-', 123, '-def'] + """ + rawParts = re.split(r'(\d+)', string) + + # Filter out empty strings. + nonEmptyParts = filter(None, rawParts) + + # Convert any numeric strings to numbers. + def splitHelper(nonEmptyParts): + for part in nonEmptyParts: + if re.match(r'\d+', part): + yield int(part) + else: + yield part + + return list(splitHelper(nonEmptyParts)) + + +def sortKeysNumerically(key_value): + key, value = key_value + return splitStringWithNumbers(key) http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/contrib/utils_test.py ---------------------------------------------------------------------- diff --git a/contrib/utils_test.py b/contrib/utils_test.py new file mode 100644 index 0000000..98016a4 --- /dev/null +++ b/contrib/utils_test.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +try: + import simplejson as json +except ImportError: + import json +import unittest +import utils + + +class SplitStringToAlphaNumTest(unittest.TestCase): + + def testInitial(self): + self.assertEqual(utils.splitStringWithNumbers('12-abc'), + [12, '-abc']) + + def testMiddle(self): + self.assertEqual(utils.splitStringWithNumbers('abc-345-def'), + ['abc-', 345, '-def']) + + def testFinal(self): + self.assertEqual(utils.splitStringWithNumbers('xyz-42'), + ['xyz-', 42]) + + def testMultiple(self): + self.assertEqual(utils.splitStringWithNumbers('Aaa-123-Bbb-456-Ccc'), + ['Aaa-', 123, '-Bbb-', 456, '-Ccc']) + + +class SortKeysNumericallyTest(unittest.TestCase): + + def testSimple(self): + input = { + 'a-1': 1, + 'a-12': 12, + 'a-2': 2, + } + output = """\ +{ + "a-1": 1, + "a-2": 2, + "a-12": 12 +}\ +""" + self.assertEqual( + json.dumps(input, indent=4 * ' ', + item_sort_key=utils.sortKeysNumerically), + output) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/libcloud/data/pricing.json ---------------------------------------------------------------------- diff --git a/libcloud/data/pricing.json b/libcloud/data/pricing.json index ab85715..5cb17e3 100644 --- a/libcloud/data/pricing.json +++ b/libcloud/data/pricing.json @@ -467,28 +467,142 @@ "x-large": 0.12 }, "gogrid": { - "16GB": 3.04, "1GB": 0.19, - "24GB": 4.56, "2GB": 0.38, "4GB": 0.76, - "512MB": 0.095, - "8GB": 1.52 + "8GB": 1.52, + "16GB": 3.04, + "24GB": 4.56, + "512MB": 0.095 + }, + "google_asia": { + "f1-micro": 0.009, + "f1-micro-preemptible": 0.005, + "g1-small": 0.03, + "g1-small-preemptible": 0.01, + "n1-highcpu-2": 0.084, + "n1-highcpu-2-preemptible": 0.022, + "n1-highcpu-4": 0.168, + "n1-highcpu-4-preemptible": 0.044, + "n1-highcpu-8": 0.336, + "n1-highcpu-8-preemptible": 0.088, + "n1-highcpu-16": 0.672, + "n1-highcpu-16-preemptible": 0.176, + "n1-highcpu-32": 1.344, + "n1-highcpu-32-preemptible": 0.352, + "n1-highmem-2": 0.139, + "n1-highmem-2-preemptible": 0.0385, + "n1-highmem-4": 0.278, + "n1-highmem-4-preemptible": 0.077, + "n1-highmem-8": 0.556, + "n1-highmem-8-preemptible": 0.154, + "n1-highmem-16": 1.112, + "n1-highmem-16-preemptible": 0.308, + "n1-highmem-32": 2.224, + "n1-highmem-32-preemptible": 0.616, + "n1-standard-1": 0.055, + "n1-standard-1-preemptible": 0.0165, + "n1-standard-2": 0.11, + "n1-standard-2-preemptible": 0.033, + "n1-standard-4": 0.22, + "n1-standard-4-preemptible": 0.066, + "n1-standard-8": 0.44, + "n1-standard-8-preemptible": 0.132, + "n1-standard-16": 0.88, + "n1-standard-16-preemptible": 0.264, + "n1-standard-32": 1.76, + "n1-standard-32-preemptible": 0.528 + }, + "google_europe": { + "f1-micro": 0.009, + "f1-micro-preemptible": 0.005, + "g1-small": 0.03, + "g1-small-preemptible": 0.01, + "n1-highcpu-2": 0.084, + "n1-highcpu-2-preemptible": 0.022, + "n1-highcpu-4": 0.168, + "n1-highcpu-4-preemptible": 0.044, + "n1-highcpu-8": 0.336, + "n1-highcpu-8-preemptible": 0.088, + "n1-highcpu-16": 0.672, + "n1-highcpu-16-preemptible": 0.176, + "n1-highcpu-32": 1.344, + "n1-highcpu-32-preemptible": 0.352, + "n1-highmem-2": 0.139, + "n1-highmem-2-preemptible": 0.0385, + "n1-highmem-4": 0.278, + "n1-highmem-4-preemptible": 0.077, + "n1-highmem-8": 0.556, + "n1-highmem-8-preemptible": 0.154, + "n1-highmem-16": 1.112, + "n1-highmem-16-preemptible": 0.308, + "n1-highmem-32": 2.224, + "n1-highmem-32-preemptible": 0.616, + "n1-standard-1": 0.055, + "n1-standard-1-preemptible": 0.0165, + "n1-standard-2": 0.11, + "n1-standard-2-preemptible": 0.033, + "n1-standard-4": 0.22, + "n1-standard-4-preemptible": 0.066, + "n1-standard-8": 0.44, + "n1-standard-8-preemptible": 0.132, + "n1-standard-16": 0.88, + "n1-standard-16-preemptible": 0.264, + "n1-standard-32": 1.76, + "n1-standard-32-preemptible": 0.528 + }, + "google_us": { + "f1-micro": 0.008, + "f1-micro-preemptible": 0.005, + "g1-small": 0.027, + "g1-small-preemptible": 0.01, + "n1-highcpu-2": 0.076, + "n1-highcpu-2-preemptible": 0.02, + "n1-highcpu-4": 0.152, + "n1-highcpu-4-preemptible": 0.04, + "n1-highcpu-8": 0.304, + "n1-highcpu-8-preemptible": 0.08, + "n1-highcpu-16": 0.608, + "n1-highcpu-16-preemptible": 0.16, + "n1-highcpu-32": 1.216, + "n1-highcpu-32-preemptible": 0.32, + "n1-highmem-2": 0.126, + "n1-highmem-2-preemptible": 0.035, + "n1-highmem-4": 0.252, + "n1-highmem-4-preemptible": 0.07, + "n1-highmem-8": 0.504, + "n1-highmem-8-preemptible": 0.14, + "n1-highmem-16": 1.008, + "n1-highmem-16-preemptible": 0.28, + "n1-highmem-32": 2.016, + "n1-highmem-32-preemptible": 0.56, + "n1-standard-1": 0.05, + "n1-standard-1-preemptible": 0.015, + "n1-standard-2": 0.1, + "n1-standard-2-preemptible": 0.03, + "n1-standard-4": 0.2, + "n1-standard-4-preemptible": 0.06, + "n1-standard-8": 0.4, + "n1-standard-8-preemptible": 0.12, + "n1-standard-16": 0.8, + "n1-standard-16-preemptible": 0.24, + "n1-standard-32": 1.6, + "n1-standard-32-preemptible": 0.48 }, "nephoscale": { "1": 0.6, + "3": 0.063, + "5": 0.031, + "7": 0.125, + "9": 0.188, "11": 0.35, "27": 0.0, - "3": 0.063, "46": 0.1, "48": 0.15, - "5": 0.031, "50": 0.28, "52": 0.48, "54": 0.938, - "56": 0.75, - "7": 0.125, - "9": 0.188 + "56": 0.75 }, "nimbus": { "m1.large": 0.0, @@ -640,11 +754,11 @@ "performance1-2": 0.08, "performance1-4": 0.16, "performance1-8": 0.32, - "performance2-120": 5.44, "performance2-15": 0.68, "performance2-30": 1.36, "performance2-60": 2.72, - "performance2-90": 4.08 + "performance2-90": 4.08, + "performance2-120": 5.44 }, "rackspacenovalon": { "2": 0.032, @@ -658,11 +772,11 @@ "performance1-2": 0.08, "performance1-4": 0.16, "performance1-8": 0.32, - "performance2-120": 5.44, "performance2-15": 0.68, "performance2-30": 1.36, "performance2-60": 2.72, - "performance2-90": 4.08 + "performance2-90": 4.08, + "performance2-120": 5.44 }, "rackspacenovasyd": { "2": 0.026, @@ -676,11 +790,11 @@ "performance1-2": 0.08, "performance1-4": 0.16, "performance1-8": 0.32, - "performance2-120": 5.44, "performance2-15": 0.68, "performance2-30": 1.36, "performance2-60": 2.72, - "performance2-90": 4.08 + "performance2-90": 4.08, + "performance2-120": 5.44 }, "rackspacenovaus": { "2": 0.022, @@ -694,11 +808,11 @@ "performance1-2": 0.08, "performance1-4": 0.16, "performance1-8": 0.32, - "performance2-120": 5.44, "performance2-15": 0.68, "performance2-30": 1.36, "performance2-60": 2.72, - "performance2-90": 4.08 + "performance2-90": 4.08, + "performance2-120": 5.44 }, "serverlove": { "extra-large": 0.615, @@ -724,10 +838,10 @@ "4": 0.045, "5": 0.045, "6": 0.045, - "7": 0.090, - "8": 0.090, - "9": 0.090, - "10": 0.090, + "7": 0.09, + "8": 0.09, + "9": 0.09, + "10": 0.09, "11": 0.205, "12": 0.205 }, @@ -736,5 +850,5 @@ } }, "storage": {}, - "updated": 1427829652 + "updated": 1447082902 } \ No newline at end of file
