Github user JoshRosen commented on a diff in the pull request:
https://github.com/apache/spark/pull/7401#discussion_r34615640
--- Diff: dev/run-tests-jenkins.py ---
@@ -0,0 +1,253 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import print_function
+import os
+import sys
+import json
+import subprocess
+
+from sparktestsupport import SPARK_HOME, ERROR_CODES
+from sparktestsupport.shellutils import exit_from_command_with_retcode,
run_cmd, rm_r
+
+
+def print_err(*args):
+ """
+ Given a set of arguments, will print them to the STDERR stream
+ """
+ print(*args, file=sys.stderr)
+
+
+def post_message(mssg, comments_url):
+ http_code_header = "HTTP Response Code: "
+ posted_message = json.dumps({"body": mssg})
+
+ print("Attempting to post to Github...")
+
+ # we don't want to call `run_cmd` here as, in the event of an error,
we DO NOT
+ # want to print the GITHUB_OAUTH_KEY into the public Jenkins logs
+ curl_proc = subprocess.Popen(['curl',
+ '--silent',
+ '--user', 'x-oauth-basic:' +
os.environ['GITHUB_OATH_KEY'],
+ '--request', 'POST',
+ '--data', posted_message,
+ '--write-out', http_code_header +
'%{http_code}',
+ '--header', 'Content-Type:
application/json',
+ comments_url],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ curl_stdout, curl_stderr = curl_proc.communicate()
+ curl_returncode = curl_proc.returncode
+ # find all lines relevant to the Github API response
+ api_response = "\n".join([l for l in curl_stdout.split('\n')
+ if l and not l.startswith(http_code_header)])
+ # find the line where `http_code_header` exists, split on ':' to get
the
+ # HTTP response code, and cast to an int
+ http_code =
int(curl_stdout[curl_stdout.find(http_code_header):].split(':')[1])
+
+ if not curl_returncode == 0:
+ print_err("Failed to post message to GitHub.")
+ print_err(" > curl_status:", curl_returncode)
+ print_err(" > curl_output:", curl_stdout)
+ print_err(" > data:", posted_message)
+
+ if http_code and not http_code == 201:
+ print_err(" > http_code:", http_code)
+ print_err(" > api_response:", api_response)
+ print_err(" > data:", posted_message)
+
+ if curl_returncode == 0 and http_code == 201:
+ print(" > Post successful.")
+
+
+def send_archived_logs():
+ print("Archiving unit tests logs...")
+
+ log_files = run_cmd(['find', '.',
+ '-name', 'unit-tests.log',
+ '-o', '-path',
'./sql/hive/target/HiveCompatibilitySuite.failed',
+ '-o', '-path',
'./sql/hive/target/HiveCompatibilitySuite.hiveFailed',
+ '-o', '-path',
'./sql/hive/target/HiveCompatibilitySuite.wrong'],
+ return_output=True)
+
+ if log_files:
+ log_archive = "unit-tests-logs.tar.gz"
+
+ run_cmd(['tar', 'czf', log-archive] +
log_files.strip().split('\n'))
+
+ jenkins_build_dir = os.environ["JENKINS_HOME"] + "/jobs/" +
os.environ["JOB_NAME"] + \
+ "/builds/" + os.environ["BUILD_NUMBER"]
+
+ scp_proc = subprocess.Popen(['scp', log_archive,
+ 'amp-jenkins-master:' +
jenkins_build_dir + '/' + log_archive],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ scp_stdout, scp_stderr = scp_proc.communicate()
+ scp_returncode = scp_proc.returncode
+
+ if not scp_returncode == 0:
+ print_err("Failed to send archived unit tests logs to Jenkins
master.")
+ print_err(" > scp_status:", scp_returncode)
+ print_err(" > scp_output:", scp_stdout)
+ else:
+ print(" > Send successful.")
+ else:
+ print_err(" > No log files found.")
+
+ rm_r(log_archive)
+
+
+def run_pr_tests(pr_tests, ghprb_actual_commit, sha1):
+ # Ensure we save off the current HEAD to revert to
+ current_pr_head = run_cmd(['git', 'rev-parse', 'HEAD'],
return_output=True).strip()
+ pr_results = list()
+
+ for pr_test in pr_tests:
+ pr_results.append(run_cmd(['bash', os.path.join(SPARK_HOME, 'dev',
'tests', pr_test),
+ ghprb_actual_commit, sha1],
+ return_output=True).strip())
+ # Ensure, after each test, that we're back on the current PR
+ run_cmd(['git', 'checkout', '-f', current_pr_head])
+ return pr_results
+
+
+def bind_message_base(build_display_name, build_url, ghprb_pull_id,
short_commit_hash, commit_url):
+ """
+ Given base parameters to generate a strong Github message response,
binds those
+ parameters into a closure without the specific message and returns a
function
+ able to generate strong messages for a specific description.
+ """
+ return lambda mssg, post_mssg="": \
+ '**[Test build ' + build_display_name + ' ' + mssg + '](' +
build_url + \
+ 'console)** for PR ' + ghprb_pull_id + ' at commit [\`' +
short_commit_hash + '\`](' + \
+ commit_url + ')' + str(' ' + post_mssg + '.') if post_mssg else '.'
+
+
+def success_result_note(mssg):
--- End diff --
This is only called in one place; can you inline it there?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
