mik-laj commented on a change in pull request #6541: BashOperator does not
create temporary shell script
URL: https://github.com/apache/airflow/pull/6541#discussion_r344789351
##########
File path: airflow/operators/bash_operator.py
##########
@@ -105,42 +105,37 @@ def execute(self, context):
self.lineage_data = self.bash_command
with TemporaryDirectory(prefix='airflowtmp') as tmp_dir:
- with NamedTemporaryFile(dir=tmp_dir, prefix=self.task_id) as
tmp_file:
- tmp_file.write(bytes(self.bash_command, 'utf_8'))
- tmp_file.flush()
- script_location = os.path.abspath(tmp_file.name)
- self.log.info('Temporary script location: %s', script_location)
-
- def pre_exec():
- # Restore default signal disposition and invoke setsid
- for sig in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'):
- if hasattr(signal, sig):
- signal.signal(getattr(signal, sig), signal.SIG_DFL)
- os.setsid()
-
- self.log.info('Running command: %s', self.bash_command)
- sub_process = Popen(
- ['bash', tmp_file.name],
- stdout=PIPE,
- stderr=STDOUT,
- cwd=tmp_dir,
- env=env,
- preexec_fn=pre_exec)
-
- self.sub_process = sub_process
-
- self.log.info('Output:')
- line = ''
- for raw_line in iter(sub_process.stdout.readline, b''):
- line = raw_line.decode(self.output_encoding).rstrip()
- self.log.info(line)
-
- sub_process.wait()
-
- self.log.info('Command exited with return code %s',
sub_process.returncode)
-
- if sub_process.returncode:
- raise AirflowException('Bash command failed')
+
+ def pre_exec():
+ # Restore default signal disposition and invoke setsid
+ for sig in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'):
+ if hasattr(signal, sig):
+ signal.signal(getattr(signal, sig), signal.SIG_DFL)
+ os.setsid()
+
+ self.log.info('Running command: %s', self.bash_command)
+ sub_process = Popen(
+ ['bash', "-c", self.bash_command],
+ stdout=PIPE,
+ stderr=STDOUT,
+ cwd=tmp_dir,
+ env=env,
+ preexec_fn=pre_exec)
+
+ self.sub_process = sub_process
+
+ self.log.info('Output:')
+ line = ''
+ for raw_line in iter(sub_process.stdout.readline, b''):
+ line = raw_line.decode(self.output_encoding).rstrip()
+ self.log.info("%s", line)
Review comment:
I changed the logging method, because data should be passed in arguments,
because then it look better in the console - different a color.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
