Repository: incubator-airflow Updated Branches: refs/heads/master f1d72e53b -> 644f5d43a
[AIRFLOW-1343] Fix dataproc label format Dataproc label must conform to the following regex: [a-z]([-a-z0-9]*[a-z0-9])?. Current "airflow_version" label violates this format. This commit fixes the format and updates the unittest to prevent future violations. Closes #2413 from fenglu-g/master Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/644f5d43 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/644f5d43 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/644f5d43 Branch: refs/heads/master Commit: 644f5d43a49d621dbd2e3f4d4731bf85c240f080 Parents: f1d72e5 Author: Feng Lu <[email protected]> Authored: Wed Jul 5 10:52:20 2017 -0700 Committer: Chris Riccomini <[email protected]> Committed: Wed Jul 5 10:52:30 2017 -0700 ---------------------------------------------------------------------- airflow/contrib/operators/dataproc_operator.py | 6 +++++- tests/contrib/operators/test_dataproc_operator.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/644f5d43/airflow/contrib/operators/dataproc_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py index 3e006ac..4b6b61c 100644 --- a/airflow/contrib/operators/dataproc_operator.py +++ b/airflow/contrib/operators/dataproc_operator.py @@ -229,7 +229,11 @@ class DataprocClusterCreateOperator(BaseOperator): } cluster_data['labels'] = self.labels if self.labels else {} - cluster_data['labels'].update({'airflow_version': version}) + # Dataproc labels must conform to the following regex: + # [a-z]([-a-z0-9]*[a-z0-9])? (current airflow version string follows + # semantic versioning spec: x.y.z). + cluster_data['labels'].update({'airflow-version': + 'v' + version.replace('.', '-')}) if self.storage_bucket: cluster_data['config']['configBucket'] = self.storage_bucket if self.metadata: http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/644f5d43/tests/contrib/operators/test_dataproc_operator.py ---------------------------------------------------------------------- diff --git a/tests/contrib/operators/test_dataproc_operator.py b/tests/contrib/operators/test_dataproc_operator.py index a441e47..923fecc 100644 --- a/tests/contrib/operators/test_dataproc_operator.py +++ b/tests/contrib/operators/test_dataproc_operator.py @@ -13,6 +13,7 @@ # limitations under the License. # +import re import unittest from airflow.contrib.operators.dataproc_operator import DataprocClusterCreateOperator @@ -33,12 +34,12 @@ WORKER_MACHINE_TYPE = 'n1-standard-2' WORKER_DISK_SIZE = 100 NUM_PREEMPTIBLE_WORKERS = 2 LABEL1 = {} -LABEL2 = {'application':'test', 'year': 2017} +LABEL2 = {'application':'test', 'year': 2017} class DataprocClusterCreateOperatorTest(unittest.TestCase): # Unitest for the DataprocClusterCreateOperator def setUp(self): - # instantiate two different test cases with different labels + # instantiate two different test cases with different labels. self.labels = [LABEL1, LABEL2] self.dataproc_operators = [] for labels in self.labels: @@ -86,8 +87,11 @@ class DataprocClusterCreateOperatorTest(unittest.TestCase): self.assertEqual(cluster_data['config']['workerConfig']['numInstances'], NUM_WORKERS) self.assertEqual(cluster_data['config']['secondaryWorkerConfig']['numInstances'], NUM_PREEMPTIBLE_WORKERS) - # test whether the default airflow_version label has been properly set to the dataproc operator + # test whether the default airflow-version label has been properly + # set to the dataproc operator. merged_labels = {} merged_labels.update(self.labels[suffix]) - merged_labels.update({'airflow_version': version}) + merged_labels.update({'airflow-version': 'v' + version.replace('.', '-')}) + self.assertTrue(re.match(r'[a-z]([-a-z0-9]*[a-z0-9])?', + cluster_data['labels']['airflow-version'])) self.assertEqual(cluster_data['labels'], merged_labels)
