This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 9150216 submit topology file by uri (#2886)
9150216 is described below
commit 915021630404d9a05d4ae27aa8e48b9a0b071029
Author: bed debug <[email protected]>
AuthorDate: Fri May 11 11:28:29 2018 -0700
submit topology file by uri (#2886)
* move downloader registry map to yaml
* fix neng comment
* staging
* fix path
* fix type
* refactor #2274
* revert comment
* fix downloader command
---
heron/tools/cli/src/python/submit.py | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/heron/tools/cli/src/python/submit.py
b/heron/tools/cli/src/python/submit.py
index c70b3fe..ffb0ae4 100644
--- a/heron/tools/cli/src/python/submit.py
+++ b/heron/tools/cli/src/python/submit.py
@@ -24,6 +24,8 @@ import logging
import os
import tempfile
import requests
+import subprocess
+import urlparse
from heron.common.src.python.utils.log import Log
from heron.proto import topology_pb2
@@ -370,6 +372,22 @@ def submit_cpp(cl_args, unknown_args, tmp_dir):
return launch_topologies(cl_args, topology_file, tmp_dir)
+def download(uri, cluster):
+ tmp_dir = tempfile.mkdtemp()
+ cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh"
+ cmd_uri = "-u " + uri
+ cmd_destination = "-f " + tmp_dir
+ cmd_heron_root = "-d " + config.get_heron_dir()
+ cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster,
config.get_heron_conf_dir())
+ cmd_mode = "-m local"
+ cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root,
cmd_heron_config, cmd_mode]
+ Log.debug("download uri command: %s", cmd)
+ subprocess.call(cmd)
+ suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so")
+ for f in os.listdir(tmp_dir):
+ if f.endswith(suffix):
+ return os.path.join(tmp_dir, f)
+
################################################################################
# pylint: disable=unused-argument
def run(command, parser, cl_args, unknown_args):
@@ -391,6 +409,11 @@ def run(command, parser, cl_args, unknown_args):
# get the topology file name
topology_file = cl_args['topology-file-name']
+ if urlparse.urlparse(topology_file).scheme:
+ cl_args['topology-file-name'] = download(topology_file, cl_args['cluster'])
+ topology_file = cl_args['topology-file-name']
+ Log.debug("download uri to local file: %s", topology_file)
+
# check to see if the topology file exists
if not os.path.isfile(topology_file):
err_context = "Topology file '%s' does not exist" % topology_file
--
To stop receiving notification emails like this one, please contact
[email protected].
