Repository: incubator-airflow Updated Branches: refs/heads/master 80210b2bd -> 0bd5515a4
[AIRFLOW-218] Added option to enable webserver gunicorn access/err logs Closes #1577 from aoen/ddavydov/better_http_response_logging Added an option to enable gunicorn access/error logs. The default config will now log webserver errors/accesses to stderr. Also made the 404 page hostname text default page color instead of white, since white text is pretty hard to see against a white background. Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/0bd5515a Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/0bd5515a Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/0bd5515a Branch: refs/heads/master Commit: 0bd5515a42f7912b0d4ac8bf33dec2f01539b555 Parents: 80210b2 Author: Dan Davydov <dan.davy...@airbnb.com> Authored: Tue Jun 7 22:13:48 2016 -0700 Committer: Dan Davydov <dan.davy...@airbnb.com> Committed: Tue Jun 7 22:14:01 2016 -0700 ---------------------------------------------------------------------- airflow/bin/cli.py | 55 ++++++++++++++++++------- airflow/configuration.py | 10 ++++- airflow/www/templates/airflow/circles.html | 2 +- 3 files changed, 49 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0bd5515a/airflow/bin/cli.py ---------------------------------------------------------------------- diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py index 87400eb..3430164 100755 --- a/airflow/bin/cli.py +++ b/airflow/bin/cli.py @@ -416,6 +416,8 @@ def webserver(args): from airflow.www.app import cached_app app = cached_app(conf) + access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile') + error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile') workers = args.workers or conf.get('webserver', 'workers') worker_timeout = (args.worker_timeout or conf.get('webserver', 'webserver_worker_timeout')) @@ -427,23 +429,36 @@ def webserver(args): else: pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid) print( - 'Running the Gunicorn server with {workers} {args.workerclass}' - 'workers on host {args.hostname} and port ' - '{args.port} with a timeout of {worker_timeout}...'.format(**locals())) - - run_args = ['gunicorn', - '-w ' + str(args.workers), - '-k ' + str(args.workerclass), - '-t ' + str(args.worker_timeout), - '-b ' + args.hostname + ':' + str(args.port), - '-n ' + 'airflow-webserver', - '-p ' + str(pid)] + textwrap.dedent('''\ + Running the Gunicorn Server with: + Workers: {workers} {args.workerclass} + Host: {args.hostname}:{args.port} + Timeout: {worker_timeout} + Logfiles: {access_logfile} {error_logfile} + =================================================================\ + '''.format(**locals()))) + + run_args = [ + 'gunicorn', + '-w ' + str(args.workers), + '-k ' + str(args.workerclass), + '-t ' + str(args.worker_timeout), + '-b ' + args.hostname + ':' + str(args.port), + '-n ' + 'airflow-webserver', + '-p ' + str(pid), + ] + + if args.access_logfile: + run_args += ['--access-logfile', str(args.access_logfile)] + + if args.error_logfile: + run_args += ['--error-logfile', str(args.error_logfile)] if args.daemon: - run_args.append("-D") + run_args += ["-D"] module = "airflow.www.app:cached_app()".encode() - run_args.append(module) + run_args += [module] os.execvp( 'gunicorn', run_args ) @@ -799,6 +814,16 @@ class CLIFactory(object): ("-d", "--debug"), "Use the server that ships with Flask in debug mode", "store_true"), + 'access_logfile': Arg( + ("-A", "--access_logfile"), + default=conf.get('webserver', 'ACCESS_LOGFILE'), + help="The logfile to store the webserver access log. Use '-' to print to " + "stderr."), + 'error_logfile': Arg( + ("-E", "--error_logfile"), + default=conf.get('webserver', 'ERROR_LOGFILE'), + help="The logfile to store the webserver error log. Use '-' to print to " + "stderr."), # resetdb 'yes': Arg( ("-y", "--yes"), @@ -925,8 +950,8 @@ class CLIFactory(object): 'func': webserver, 'help': "Start a Airflow webserver instance", 'args': ('port', 'workers', 'workerclass', 'worker_timeout', 'hostname', - 'pid', 'daemon', 'stdout', 'stderr', 'log_file', - 'debug'), + 'pid', 'daemon', 'stdout', 'stderr', 'access_logfile', + 'error_logfile', 'log_file', 'debug'), }, { 'func': resetdb, 'help': "Burn down and rebuild the metadata database", http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0bd5515a/airflow/configuration.py ---------------------------------------------------------------------- diff --git a/airflow/configuration.py b/airflow/configuration.py index 3193fbf..3403656 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -98,7 +98,7 @@ defaults = { 'remote_base_log_folder': '', 'remote_log_conn_id': '', 'encrypt_s3_logs': False, - 's3_log_folder': '', # deprecated! + 's3_log_folder': '', # deprecated! 'dag_concurrency': 16, 'max_active_runs_per_dag': 16, 'executor': 'SequentialExecutor', @@ -122,7 +122,9 @@ defaults = { 'secret_key': 'airflowified', 'expose_config': False, 'workers': 4, - 'worker_class': 'sync' + 'worker_class': 'sync', + 'access_logfile': '', + 'error_logfile': '', }, 'scheduler': { 'statsd_on': False, @@ -277,6 +279,10 @@ workers = 4 # sync (default), eventlet, gevent worker_class = sync +# Log files for the gunicorn webserver. '-' means log to stderr. +access_logfile = - +error_logfile = - + # Expose the configuration file in the web server expose_config = true http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0bd5515a/airflow/www/templates/airflow/circles.html ---------------------------------------------------------------------- diff --git a/airflow/www/templates/airflow/circles.html b/airflow/www/templates/airflow/circles.html index 44e1349..7407d8d 100644 --- a/airflow/www/templates/airflow/circles.html +++ b/airflow/www/templates/airflow/circles.html @@ -1,6 +1,6 @@ <div style="font-family: verdana;"> <h1>Airflow 404 = lots of circles</h1> - <div style="color: white">{{ hostname }}</div> + <div>{{ hostname }}</div> <div id="div_svg" class="content"