This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f5ad26d Fixup string concatenations (#19099)
f5ad26d is described below
commit f5ad26dcdd7bcb724992528dce71056965b94d26
Author: blag <[email protected]>
AuthorDate: Thu Oct 21 15:56:51 2021 -0700
Fixup string concatenations (#19099)
---
.pre-commit-config.yaml | 2 +-
airflow/models/dagcode.py | 7 +-
airflow/models/taskinstance.py | 4 +-
airflow/operators/google_api_to_s3_transfer.py | 8 +-
airflow/providers/alibaba/cloud/sensors/oss_key.py | 4 +-
airflow/providers/amazon/aws/hooks/s3.py | 8 +-
airflow/providers/amazon/aws/sensors/s3_key.py | 4 +-
airflow/providers/databricks/hooks/databricks.py | 2 +-
.../example_dags/example_docker_copy_data.py | 2 +-
airflow/providers/yandex/hooks/yandex.py | 2 +-
airflow/sensors/smart_sensor.py | 2 +-
dev/provider_packages/prepare_provider_packages.py | 2 +-
.../pre_commit_check_extras_have_providers.py | 2 +-
.../amazon/aws/utils/eks_test_constants.py | 61 ++++++------
.../apache/hive/operators/test_hive_stats.py | 16 +--
.../apache/spark/hooks/test_spark_submit.py | 109 +++++++++++----------
16 files changed, 121 insertions(+), 114 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e652f95..72a5ec6 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -563,7 +563,7 @@ repos:
additional_dependencies:
- 'PyYAML==5.3.1'
- 'jsonschema==3.2.0'
- - 'tabulate==0.8.7'
+ - 'tabulate==0.8.8'
- 'jsonpath-ng==1.5.3'
- 'rich==10.9.0'
- id: mermaid
diff --git a/airflow/models/dagcode.py b/airflow/models/dagcode.py
index fa5f7e1..7322ba9 100644
--- a/airflow/models/dagcode.py
+++ b/airflow/models/dagcode.py
@@ -94,10 +94,11 @@ class DagCode(Base):
hashes_to_filelocs = {DagCode.dag_fileloc_hash(fileloc): fileloc
for fileloc in filelocs}
message = ""
for fileloc in conflicting_filelocs:
+ filename =
hashes_to_filelocs[DagCode.dag_fileloc_hash(fileloc)]
message += (
- "Filename '{}' causes a hash collision in the "
- + "database with '{}'. Please rename the file."
-
).format(hashes_to_filelocs[DagCode.dag_fileloc_hash(fileloc)], fileloc)
+ f"Filename '{filename}' causes a hash collision in the "
+ f"database with '{fileloc}'. Please rename the file."
+ )
raise AirflowException(message)
existing_filelocs = {dag_code.fileloc for dag_code in
existing_orm_dag_codes}
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index c3484ee..b9745cc 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -1267,8 +1267,8 @@ class TaskInstance(Base, LoggingMixin):
def _log_state(self, lead_msg: str = ''):
self.log.info(
'%sMarking task as %s.'
- + ' dag_id=%s, task_id=%s,'
- + ' execution_date=%s, start_date=%s, end_date=%s',
+ ' dag_id=%s, task_id=%s,'
+ ' execution_date=%s, start_date=%s, end_date=%s',
lead_msg,
self.state.upper(),
self.dag_id,
diff --git a/airflow/operators/google_api_to_s3_transfer.py
b/airflow/operators/google_api_to_s3_transfer.py
index b08ba75..9566cdd 100644
--- a/airflow/operators/google_api_to_s3_transfer.py
+++ b/airflow/operators/google_api_to_s3_transfer.py
@@ -40,10 +40,10 @@ class GoogleApiToS3Transfer(GoogleApiToS3Operator):
def __init__(self, **kwargs):
warnings.warn(
- """This class is deprecated.
- Please use
- `airflow.providers.amazon.aws.transfers."""
- + "google_api_to_s3_transfer.GoogleApiToS3Operator`.",
+ "This class is deprecated. "
+ "Please use "
+ "`airflow.providers.amazon.aws.transfers."
+ "google_api_to_s3_transfer.GoogleApiToS3Operator`.",
DeprecationWarning,
stacklevel=3,
)
diff --git a/airflow/providers/alibaba/cloud/sensors/oss_key.py
b/airflow/providers/alibaba/cloud/sensors/oss_key.py
index 22bd8e9..9949aff 100644
--- a/airflow/providers/alibaba/cloud/sensors/oss_key.py
+++ b/airflow/providers/alibaba/cloud/sensors/oss_key.py
@@ -76,8 +76,8 @@ class OSSKeySensor(BaseSensorOperator):
if parsed_url.scheme != '' or parsed_url.netloc != '':
raise AirflowException(
'If bucket_name is provided, bucket_key'
- + ' should be relative path from root'
- + ' level, rather than a full oss:// url'
+ ' should be relative path from root'
+ ' level, rather than a full oss:// url'
)
self.log.info('Poking for key : oss://%s/%s', self.bucket_name,
self.bucket_key)
diff --git a/airflow/providers/amazon/aws/hooks/s3.py
b/airflow/providers/amazon/aws/hooks/s3.py
index 84f3794..4a2a15b 100644
--- a/airflow/providers/amazon/aws/hooks/s3.py
+++ b/airflow/providers/amazon/aws/hooks/s3.py
@@ -716,8 +716,8 @@ class S3Hook(AwsBaseHook):
if parsed_url.scheme != '' or parsed_url.netloc != '':
raise AirflowException(
'If dest_bucket_name is provided, '
- + 'dest_bucket_key should be relative path '
- + 'from root level, rather than a full s3:// url'
+ 'dest_bucket_key should be relative path '
+ 'from root level, rather than a full s3:// url'
)
if source_bucket_name is None:
@@ -727,8 +727,8 @@ class S3Hook(AwsBaseHook):
if parsed_url.scheme != '' or parsed_url.netloc != '':
raise AirflowException(
'If source_bucket_name is provided, '
- + 'source_bucket_key should be relative path '
- + 'from root level, rather than a full s3:// url'
+ 'source_bucket_key should be relative path '
+ 'from root level, rather than a full s3:// url'
)
copy_source = {'Bucket': source_bucket_name, 'Key': source_bucket_key,
'VersionId': source_version_id}
diff --git a/airflow/providers/amazon/aws/sensors/s3_key.py
b/airflow/providers/amazon/aws/sensors/s3_key.py
index 9a7af08..57cc0df 100644
--- a/airflow/providers/amazon/aws/sensors/s3_key.py
+++ b/airflow/providers/amazon/aws/sensors/s3_key.py
@@ -89,8 +89,8 @@ class S3KeySensor(BaseSensorOperator):
if parsed_url.scheme != '' or parsed_url.netloc != '':
raise AirflowException(
'If bucket_name is provided, bucket_key'
- + ' should be relative path from root'
- + ' level, rather than a full s3:// url'
+ ' should be relative path from root'
+ ' level, rather than a full s3:// url'
)
self.log.info('Poking for key : s3://%s/%s', self.bucket_name,
self.bucket_key)
diff --git a/airflow/providers/databricks/hooks/databricks.py
b/airflow/providers/databricks/hooks/databricks.py
index 5e7fc35..5e843a4 100644
--- a/airflow/providers/databricks/hooks/databricks.py
+++ b/airflow/providers/databricks/hooks/databricks.py
@@ -216,7 +216,7 @@ class DatabricksHook(BaseHook):
if attempt_num == self.retry_limit:
raise AirflowException(
- ('API requests to Databricks failed {} times. ' + 'Giving
up.').format(self.retry_limit)
+ f'API requests to Databricks failed {self.retry_limit}
times. Giving up.'
)
attempt_num += 1
diff --git a/airflow/providers/docker/example_dags/example_docker_copy_data.py
b/airflow/providers/docker/example_dags/example_docker_copy_data.py
index f03c4be..5ce78d0 100644
--- a/airflow/providers/docker/example_dags/example_docker_copy_data.py
+++ b/airflow/providers/docker/example_dags/example_docker_copy_data.py
@@ -75,7 +75,7 @@ t_move = DockerOperator(
"/bin/bash",
"-c",
"/bin/sleep 30; "
- "/bin/mv {{ params.source_location }}/" + f"{t_view.output}" + " {{
params.target_location }};"
+ "/bin/mv {{ params.source_location }}/" + str(t_view.output) + " {{
params.target_location }};"
"/bin/echo '{{ params.target_location }}/" + f"{t_view.output}';",
],
task_id="move_data",
diff --git a/airflow/providers/yandex/hooks/yandex.py
b/airflow/providers/yandex/hooks/yandex.py
index ee1ae0d..06049e5 100644
--- a/airflow/providers/yandex/hooks/yandex.py
+++ b/airflow/providers/yandex/hooks/yandex.py
@@ -119,7 +119,7 @@ class YandexCloudBaseHook(BaseHook):
if not (service_account_json or oauth_token or
service_account_json_path):
raise AirflowException(
'No credentials are found in connection. Specify either
service account '
- + 'authentication JSON or user OAuth token in Yandex.Cloud
connection'
+ 'authentication JSON or user OAuth token in Yandex.Cloud
connection'
)
if service_account_json_path:
with open(service_account_json_path) as infile:
diff --git a/airflow/sensors/smart_sensor.py b/airflow/sensors/smart_sensor.py
index 1e8c827..e042c61 100644
--- a/airflow/sensors/smart_sensor.py
+++ b/airflow/sensors/smart_sensor.py
@@ -125,7 +125,7 @@ class SensorWork:
log_id = "-".join(
[si.dag_id, si.task_id,
si.execution_date.strftime("%Y_%m_%dT%H_%M_%S_%f"), str(si.try_number)]
)
- logger = logging.getLogger('airflow.task' + '.' + log_id)
+ logger = logging.getLogger(f'airflow.task.{log_id}')
if len(logger.handlers) == 0:
handler = self.create_new_task_handler()
diff --git a/dev/provider_packages/prepare_provider_packages.py
b/dev/provider_packages/prepare_provider_packages.py
index a56b529..c3c7554 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -1238,7 +1238,7 @@ def
validate_provider_info_with_runtime_schema(provider_info: Dict[str, Any]) ->
console.print("[red]Provider info not validated against runtime
schema[/]")
raise Exception(
"Error when validating schema. The schema must be compatible with "
- + "airflow/provider_info.schema.json.",
+ "airflow/provider_info.schema.json.",
ex,
)
diff --git a/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
b/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
index 20088b7..7c1b7eb 100755
--- a/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
+++ b/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
@@ -71,7 +71,7 @@ def check_all_providers_are_listed_in_setup_py() -> None:
if provider_name not in ALL_PROVIDERS:
errors.append(
f"The provider {provider_name} is missing in setup.py "
- + "[bold]PROVIDERS_REQUIREMENTS[/]: [red]NOK[/]"
+ "[bold]PROVIDERS_REQUIREMENTS[/]: [red]NOK[/]"
)
diff --git a/tests/providers/amazon/aws/utils/eks_test_constants.py
b/tests/providers/amazon/aws/utils/eks_test_constants.py
index 1a5985c..4624cd4 100644
--- a/tests/providers/amazon/aws/utils/eks_test_constants.py
+++ b/tests/providers/amazon/aws/utils/eks_test_constants.py
@@ -187,10 +187,10 @@ class PageCount:
FARGATE_PROFILE_UUID_PATTERN: str = (
-
"(?P<fargate_uuid>[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})"
+
r"(?P<fargate_uuid>[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})"
)
NODEGROUP_UUID_PATTERN: str = (
-
"(?P<nodegroup_uuid>[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})"
+
r"(?P<nodegroup_uuid>[-0-9a-z]{8}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{4}-[-0-9a-z]{12})"
)
@@ -198,38 +198,41 @@ class RegExTemplates:
"""The compiled RegEx patterns used in testing."""
CLUSTER_ARN: Pattern = re.compile(
- "arn:"
- + "(?P<partition>.+):"
- + "eks:"
- + "(?P<region>[-0-9a-zA-Z]+):"
- + "(?P<account_id>[0-9]{12}):"
- + "cluster/"
- + "(?P<cluster_name>.+)"
+ r"""arn:
+ (?P<partition>.+):
+ eks:
+ (?P<region>[-0-9a-zA-Z]+):
+ (?P<account_id>[0-9]{12}):
+ cluster/
+ (?P<cluster_name>.+)""",
+ re.VERBOSE,
)
FARGATE_PROFILE_ARN: Pattern = re.compile(
- "arn:"
- + "(?P<partition>.+):"
- + "eks:"
- + "(?P<region>[-0-9a-zA-Z]+):"
- + "(?P<account_id>[0-9]{12}):"
- + "fargateprofile/"
- + "(?P<cluster_name>.+)/"
- + "(?P<fargate_name>.+)/"
- + FARGATE_PROFILE_UUID_PATTERN
+ r"""arn:
+ (?P<partition>.+):
+ eks:
+ (?P<region>[-0-9a-zA-Z]+):
+ (?P<account_id>[0-9]{12}):
+ fargateprofile/
+ (?P<cluster_name>.+)/
+ (?P<fargate_name>.+)/"""
+ + FARGATE_PROFILE_UUID_PATTERN,
+ re.VERBOSE,
)
NODEGROUP_ARN: Pattern = re.compile(
- "arn:"
- + "(?P<partition>.+):"
- + "eks:"
- + "(?P<region>[-0-9a-zA-Z]+):"
- + "(?P<account_id>[0-9]{12}):"
- + "nodegroup/"
- + "(?P<cluster_name>.+)/"
- + "(?P<nodegroup_name>.+)/"
- + NODEGROUP_UUID_PATTERN
+ r"""arn:
+ (?P<partition>.+):
+ eks:
+ (?P<region>[-0-9a-zA-Z]+):
+ (?P<account_id>[0-9]{12}):
+ nodegroup/
+ (?P<cluster_name>.+)/
+ (?P<nodegroup_name>.+)/"""
+ + NODEGROUP_UUID_PATTERN,
+ re.VERBOSE,
)
- NODEGROUP_ASG_NAME_PATTERN: Pattern = re.compile("eks-" +
NODEGROUP_UUID_PATTERN)
- NODEGROUP_SECURITY_GROUP_NAME_PATTERN: Pattern = re.compile("sg-" +
"([-0-9a-z]{17})")
+ NODEGROUP_ASG_NAME_PATTERN: Pattern =
re.compile(f"eks-{NODEGROUP_UUID_PATTERN}")
+ NODEGROUP_SECURITY_GROUP_NAME_PATTERN: Pattern =
re.compile(r"sg-([-0-9a-z]{17})")
class MethodNames:
diff --git a/tests/providers/apache/hive/operators/test_hive_stats.py
b/tests/providers/apache/hive/operators/test_hive_stats.py
index 02dbdd8..b51420b 100644
--- a/tests/providers/apache/hive/operators/test_hive_stats.py
+++ b/tests/providers/apache/hive/operators/test_hive_stats.py
@@ -308,17 +308,19 @@ class
TestHiveStatsCollectionOperator(TestHiveEnvironment):
op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE,
ignore_ti_state=True)
select_count_query = (
- "SELECT COUNT(*) AS __count FROM airflow."
- + "static_babynames_partitioned WHERE ds = '2015-01-01';"
+ "SELECT COUNT(*) AS __count "
+ "FROM airflow.static_babynames_partitioned "
+ "WHERE ds = '2015-01-01';"
)
mock_presto_hook.get_first.assert_called_with(hql=select_count_query)
expected_stats_select_query = (
- "SELECT 1 FROM hive_stats WHERE table_name='airflow."
- + "static_babynames_partitioned' AND "
- + "partition_repr='{\"ds\": \"2015-01-01\"}' AND "
- + "dttm='2015-01-01T00:00:00+00:00' "
- + "LIMIT 1;"
+ "SELECT 1 "
+ "FROM hive_stats "
+ "WHERE table_name='airflow.static_babynames_partitioned' "
+ " AND partition_repr='{\"ds\": \"2015-01-01\"}' "
+ " AND dttm='2015-01-01T00:00:00+00:00' "
+ "LIMIT 1;"
)
raw_stats_select_query =
mock_mysql_hook.get_records.call_args_list[0][0][0]
diff --git a/tests/providers/apache/spark/hooks/test_spark_submit.py
b/tests/providers/apache/spark/hooks/test_spark_submit.py
index 070719a..be47bfb 100644
--- a/tests/providers/apache/spark/hooks/test_spark_submit.py
+++ b/tests/providers/apache/spark/hooks/test_spark_submit.py
@@ -88,9 +88,11 @@ class TestSparkSubmitHook(unittest.TestCase):
conn_id='spark_k8s_cluster',
conn_type='spark',
host='k8s://https://k8s-master',
- extra='{"spark-home": "/opt/spark", '
- + '"deploy-mode": "cluster", '
- + '"namespace": "mynamespace"}',
+ extra=(
+ '{"spark-home": "/opt/spark", '
+ ' "deploy-mode": "cluster", '
+ ' "namespace": "mynamespace"}'
+ ),
)
)
db.merge_conn(
@@ -120,7 +122,7 @@ class TestSparkSubmitHook(unittest.TestCase):
conn_id='spark_binary_and_home_set',
conn_type='spark',
host='yarn',
- extra='{"spark-home": "/path/to/spark_home", ' +
'"spark-binary": "custom-spark-submit"}',
+ extra='{"spark-home": "/path/to/spark_home", "spark-binary":
"custom-spark-submit"}',
)
)
db.merge_conn(
@@ -628,11 +630,11 @@ class TestSparkSubmitHook(unittest.TestCase):
log_lines = [
'SPARK_MAJOR_VERSION is set to 2, using Spark2',
'WARN NativeCodeLoader: Unable to load native-hadoop library for
your '
- + 'platform... using builtin-java classes where applicable',
+ 'platform... using builtin-java classes where applicable',
'WARN DomainSocketFactory: The short-circuit local reads feature
cannot '
'be used because libhadoop cannot be loaded.',
'INFO Client: Requesting a new application from cluster with 10
NodeManagers',
- 'INFO Client: Submitting application
application_1486558679801_1820 ' + 'to ResourceManager',
+ 'INFO Client: Submitting application
application_1486558679801_1820 to ResourceManager',
]
# When
hook._process_spark_submit_log(log_lines)
@@ -646,25 +648,25 @@ class TestSparkSubmitHook(unittest.TestCase):
hook = SparkSubmitHook(conn_id='spark_k8s_cluster')
log_lines = [
'INFO LoggingPodStatusWatcherImpl:54 - State changed, new state:'
- + 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
- + 'namespace: default'
- + 'labels: spark-app-selector ->
spark-465b868ada474bda82ccb84ab2747fcd,'
- + 'spark-role -> driver'
- + 'pod uid: ba9c61f6-205f-11e8-b65f-d48564c88e42'
- + 'creation time: 2018-03-05T10:26:55Z'
- + 'service account name: spark'
- + 'volumes: spark-init-properties, download-jars-volume,'
- + 'download-files-volume, spark-token-2vmlm'
- + 'node name: N/A'
- + 'start time: N/A'
- + 'container images: N/A'
- + 'phase: Pending'
- + 'status: []'
- + '2018-03-05 11:26:56 INFO LoggingPodStatusWatcherImpl:54 -
State changed,'
- + ' new state:'
- + 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
- + 'namespace: default'
- + 'Exit code: 999'
+ 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
+ 'namespace: default'
+ 'labels: spark-app-selector ->
spark-465b868ada474bda82ccb84ab2747fcd,'
+ 'spark-role -> driver'
+ 'pod uid: ba9c61f6-205f-11e8-b65f-d48564c88e42'
+ 'creation time: 2018-03-05T10:26:55Z'
+ 'service account name: spark'
+ 'volumes: spark-init-properties, download-jars-volume,'
+ 'download-files-volume, spark-token-2vmlm'
+ 'node name: N/A'
+ 'start time: N/A'
+ 'container images: N/A'
+ 'phase: Pending'
+ 'status: []'
+ '2018-03-05 11:26:56 INFO LoggingPodStatusWatcherImpl:54 - State
changed,'
+ ' new state:'
+ 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
+ 'namespace: default'
+ 'Exit code: 999'
]
# When
@@ -693,7 +695,7 @@ class TestSparkSubmitHook(unittest.TestCase):
'17/11/28 11:14:15 INFO RestSubmissionClient: Submitting a request
'
'to launch an application in spark://spark-standalone-master:6066',
'17/11/28 11:14:15 INFO RestSubmissionClient: Submission
successfully '
- + 'created as driver-20171128111415-0001. Polling submission
state...',
+ 'created as driver-20171128111415-0001. Polling submission
state...',
]
# When
hook._process_spark_submit_log(log_lines)
@@ -707,9 +709,8 @@ class TestSparkSubmitHook(unittest.TestCase):
hook = SparkSubmitHook(conn_id='spark_standalone_cluster')
log_lines = [
'Submitting a request for the status of submission '
- + 'driver-20171128111415-0001 in
spark://spark-standalone-master:6066',
- '17/11/28 11:15:37 INFO RestSubmissionClient: Server responded
with '
- + 'SubmissionStatusResponse:',
+ 'driver-20171128111415-0001 in
spark://spark-standalone-master:6066',
+ '17/11/28 11:15:37 INFO RestSubmissionClient: Server responded
with SubmissionStatusResponse:',
'{',
'"action" : "SubmissionStatusResponse",',
'"driverState" : "RUNNING",',
@@ -738,12 +739,12 @@ class TestSparkSubmitHook(unittest.TestCase):
log_lines = [
'SPARK_MAJOR_VERSION is set to 2, using Spark2',
'WARN NativeCodeLoader: Unable to load native-hadoop library for
your '
- + 'platform... using builtin-java classes where applicable',
+ 'platform... using builtin-java classes where applicable',
'WARN DomainSocketFactory: The short-circuit local reads feature
cannot '
- + 'be used because libhadoop cannot be loaded.',
+ 'be used because libhadoop cannot be loaded.',
'INFO Client: Requesting a new application from cluster with 10 '
- + 'NodeManagerapplication_1486558679801_1820s',
- 'INFO Client: Submitting application
application_1486558679801_1820 ' + 'to ResourceManager',
+ 'NodeManagerapplication_1486558679801_1820s',
+ 'INFO Client: Submitting application
application_1486558679801_1820 to ResourceManager',
]
env = {"PATH": "hadoop/bin"}
hook = SparkSubmitHook(conn_id='spark_yarn_cluster', env_vars=env)
@@ -792,9 +793,9 @@ class TestSparkSubmitHook(unittest.TestCase):
log_lines = [
'Running Spark using the REST application submission protocol.',
'17/11/28 11:14:15 INFO RestSubmissionClient: Submitting a request
'
- + 'to launch an application in
spark://spark-standalone-master:6066',
+ 'to launch an application in spark://spark-standalone-master:6066',
'17/11/28 11:14:15 INFO RestSubmissionClient: Submission
successfully '
- + 'created as driver-20171128111415-0001. Polling submission
state...',
+ 'created as driver-20171128111415-0001. Polling submission
state...',
]
hook = SparkSubmitHook(conn_id='spark_standalone_cluster')
hook._process_spark_submit_log(log_lines)
@@ -821,25 +822,25 @@ class TestSparkSubmitHook(unittest.TestCase):
hook = SparkSubmitHook(conn_id='spark_k8s_cluster')
log_lines = [
'INFO LoggingPodStatusWatcherImpl:54 - State changed, new state:'
- + 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
- + 'namespace: default'
- + 'labels: spark-app-selector ->
spark-465b868ada474bda82ccb84ab2747fcd,'
- + 'spark-role -> driver'
- + 'pod uid: ba9c61f6-205f-11e8-b65f-d48564c88e42'
- + 'creation time: 2018-03-05T10:26:55Z'
- + 'service account name: spark'
- + 'volumes: spark-init-properties, download-jars-volume,'
- + 'download-files-volume, spark-token-2vmlm'
- + 'node name: N/A'
- + 'start time: N/A'
- + 'container images: N/A'
- + 'phase: Pending'
- + 'status: []'
- + '2018-03-05 11:26:56 INFO LoggingPodStatusWatcherImpl:54 -
State changed,'
- + ' new state:'
- + 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
- + 'namespace: default'
- + 'Exit code: 0'
+ 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
+ 'namespace: default'
+ 'labels: spark-app-selector ->
spark-465b868ada474bda82ccb84ab2747fcd,'
+ 'spark-role -> driver'
+ 'pod uid: ba9c61f6-205f-11e8-b65f-d48564c88e42'
+ 'creation time: 2018-03-05T10:26:55Z'
+ 'service account name: spark'
+ 'volumes: spark-init-properties, download-jars-volume,'
+ 'download-files-volume, spark-token-2vmlm'
+ 'node name: N/A'
+ 'start time: N/A'
+ 'container images: N/A'
+ 'phase: Pending'
+ 'status: []'
+ '2018-03-05 11:26:56 INFO LoggingPodStatusWatcherImpl:54 - State
changed,'
+ ' new state:'
+ 'pod name: spark-pi-edf2ace37be7353a958b38733a12f8e6-driver'
+ 'namespace: default'
+ 'Exit code: 0'
]
hook._process_spark_submit_log(log_lines)
hook.submit()
