Repository: zeppelin Updated Branches: refs/heads/master ecb688f45 -> 87480056a
[ZEPPELIN-2152] Added proxy config support in Helium bundle factory ### What is this PR for? Proxy configuration implementation in Helium bundle factory. Used by Node, NPM and YarnPkg installers. ### What type of PR is it? [Improvement] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2152 ### How should this be tested? Trigger Helium plugins behind proxy ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Nelson Costa <[email protected]> Closes #2363 from necosta/zeppelin2152 and squashes the following commits: f681e73 [Nelson Costa] [ZEPPELIN-2152] Added documentation 5e29cca [Nelson Costa] [ZEPPELIN-2152] Fixed format issues f55e6e2 [Nelson Costa] [ZEPPELIN-2152] Fixed bug in secure/insecure proxy. Added proxy for HeliumOnlineRegistry d9a086a [Nelson Costa] [ZEPPELIN-2152] Added proxy config support in Helium bundle factory Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/87480056 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/87480056 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/87480056 Branch: refs/heads/master Commit: 87480056a0b10effaf5556d000f8d0c5186848d0 Parents: ecb688f Author: Nelson Costa <[email protected]> Authored: Wed May 24 17:03:53 2017 +0100 Committer: Lee moon soo <[email protected]> Committed: Sun May 28 09:35:53 2017 +0900 ---------------------------------------------------------------------- docs/_includes/themes/zeppelin/_navigation.html | 1 + docs/index.md | 1 + docs/security/helium_authorization.md | 28 ++++++++ .../zeppelin/helium/HeliumBundleFactory.java | 71 +++++++++++++++++--- .../zeppelin/helium/HeliumOnlineRegistry.java | 32 ++++++++- .../helium/HeliumBundleFactoryTest.java | 2 +- 6 files changed, 124 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/docs/_includes/themes/zeppelin/_navigation.html ---------------------------------------------------------------------- diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index caa1627..296be25 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -117,6 +117,7 @@ <li><a href="{{BASE_PATH}}/security/shiroauthentication.html">Shiro Authentication</a></li> <li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li> <li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li> + <li><a href="{{BASE_PATH}}/security/helium_authorization.html">Helium Authorization</a></li> <li role="separator" class="divider"></li> <li class="title"><span><b>Helium Framework (Experimental)</b></span></li> <li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application</a></li> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/docs/index.md ---------------------------------------------------------------------- diff --git a/docs/index.md b/docs/index.md index 7477070..bb2222b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -173,6 +173,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor * [Shiro Authentication](./security/shiroauthentication.html) * [Notebook Authorization](./security/notebook_authorization.html) * [Data Source Authorization](./security/datasource_authorization.html) + * [Helium Authorization](./security/helium_authorization.html) * Helium Framework (Experimental) * [Writing Zeppelin Application](./development/writingzeppelinapplication.html) * [Writing Zeppelin Spell](./development/writingzeppelinspell.html) http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/docs/security/helium_authorization.md ---------------------------------------------------------------------- diff --git a/docs/security/helium_authorization.md b/docs/security/helium_authorization.md new file mode 100644 index 0000000..fdfb435 --- /dev/null +++ b/docs/security/helium_authorization.md @@ -0,0 +1,28 @@ +--- +layout: page +title: "Helium Authorization in Apache Zeppelin" +description: "Apache Zeppelin supports Helium plugins which fetch required installer packages from remote registry/repositories" +group: security +--- +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +{% include JB/setup %} + +# Helium Authorization in Apache Zeppelin + +<div id="toc"></div> + +## How to configure proxies? + +Set **http_proxy** and **https_proxy** env variables to allow connection to npm registry behind a corporate firewall. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java index ef032bd..c585567 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java @@ -29,6 +29,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Appender; import org.apache.log4j.PatternLayout; import org.apache.log4j.WriterAppender; @@ -38,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.net.URI; import java.util.*; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -50,6 +52,7 @@ public class HeliumBundleFactory { private final String NODE_VERSION = "v6.9.1"; private final String NPM_VERSION = "3.10.8"; private final String YARN_VERSION = "v0.21.3"; + private static final String NPM_PACKAGE_NAME = "npm"; public static final String HELIUM_LOCAL_REPO = "helium-bundle"; public static final String HELIUM_BUNDLES_DIR = "bundles"; public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules"; @@ -74,7 +77,9 @@ public class HeliumBundleFactory { private File tabledataModulePath; private File visualizationModulePath; private File spellModulePath; + private String defaultNodeRegistryUrl; private String defaultNpmRegistryUrl; + private String defaultYarnRegistryUrl; private Gson gson; private boolean nodeAndNpmInstalled = false; @@ -102,7 +107,10 @@ public class HeliumBundleFactory { this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR); this.yarnCacheDir = new File(heliumLocalRepoDirectory, YARN_CACHE_DIR); this.conf = conf; + // To be done in ZEPPELIN-2214: Soft-code installer urls + this.defaultNodeRegistryUrl = "https://nodejs.org/dist/";; this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry(); + this.defaultYarnRegistryUrl = "https://github.com/yarnpkg/yarn/releases/download/";; nodeInstallationDirectory = (nodeInstallationDir == null) ? heliumLocalRepoDirectory : nodeInstallationDir; @@ -118,16 +126,22 @@ public class HeliumBundleFactory { return; } try { - NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig()); + NodeInstaller nodeInstaller = frontEndPluginFactory + .getNodeInstaller(getProxyConfig(isSecure(defaultNodeRegistryUrl))); nodeInstaller.setNodeVersion(NODE_VERSION); + nodeInstaller.setNodeDownloadRoot(defaultNodeRegistryUrl); nodeInstaller.install(); - NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig()); + NPMInstaller npmInstaller = frontEndPluginFactory + .getNPMInstaller(getProxyConfig(isSecure(defaultNpmRegistryUrl))); npmInstaller.setNpmVersion(NPM_VERSION); + npmInstaller.setNpmDownloadRoot(defaultNpmRegistryUrl + "/" + NPM_PACKAGE_NAME + "/-/"); npmInstaller.install(); - YarnInstaller yarnInstaller = frontEndPluginFactory.getYarnInstaller(getProxyConfig()); + YarnInstaller yarnInstaller = frontEndPluginFactory + .getYarnInstaller(getProxyConfig(isSecure(defaultYarnRegistryUrl))); yarnInstaller.setYarnVersion(YARN_VERSION); + yarnInstaller.setYarnDownloadRoot(defaultYarnRegistryUrl); yarnInstaller.install(); yarnCacheDir.mkdirs(); String yarnCacheDirPath = yarnCacheDir.getAbsolutePath(); @@ -140,9 +154,48 @@ public class HeliumBundleFactory { } } - private ProxyConfig getProxyConfig() { - List<ProxyConfig.Proxy> proxy = new LinkedList<>(); - return new ProxyConfig(proxy); + private ProxyConfig getProxyConfig(boolean isSecure) { + List<ProxyConfig.Proxy> proxies = new LinkedList<>(); + + String httpProxy = StringUtils.isBlank(System.getenv("http_proxy")) ? + System.getenv("HTTP_PROXY") : System.getenv("http_proxy"); + + String httpsProxy = StringUtils.isBlank(System.getenv("https_proxy")) ? + System.getenv("HTTPS_PROXY") : System.getenv("https_proxy"); + + try { + if (isSecure) + proxies.add(generateProxy("secure", new URI(httpsProxy))); + else proxies.add(generateProxy("insecure", new URI(httpProxy))); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + } + return new ProxyConfig(proxies); + } + + private ProxyConfig.Proxy generateProxy(String proxyId, URI uri) { + + String protocol = uri.getScheme(); + String host = uri.getHost(); + int port = uri.getPort() <= 0 ? 80 : uri.getPort(); + + String username = null, password = null; + if (uri.getUserInfo() != null) { + String[] authority = uri.getUserInfo().split(":"); + if (authority.length == 2) { + username = authority[0]; + password = authority[1]; + } else if (authority.length == 1) { + username = authority[0]; + } + } + String nonProxyHosts = StringUtils.isBlank(System.getenv("no_proxy")) ? + System.getenv("NO_PROXY") : System.getenv("no_proxy"); + return new ProxyConfig.Proxy(proxyId, protocol, host, port, username, password, nonProxyHosts); + } + + private boolean isSecure(String url) { + return url.toLowerCase().startsWith("https"); } public void buildAllPackages(List<HeliumPackage> pkgs) throws IOException { @@ -609,7 +662,8 @@ public class HeliumBundleFactory { } private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException { - NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl); + NpmRunner npm = frontEndPluginFactory.getNpmRunner( + getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl); npm.execute(args, env); } @@ -623,7 +677,8 @@ public class HeliumBundleFactory { private void yarnCommand(FrontendPluginFactory fpf, String args, Map<String, String> env) throws TaskRunnerException { - YarnRunner yarn = fpf.getYarnRunner(getProxyConfig(), defaultNpmRegistryUrl); + YarnRunner yarn = fpf.getYarnRunner( + getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl); yarn.execute(args, env); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java index b1c3a83..165d70a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java @@ -19,6 +19,8 @@ package org.apache.zeppelin.helium; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -28,7 +30,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -67,6 +70,7 @@ public class HeliumOnlineRegistry extends HeliumRegistry { public synchronized List<HeliumPackage> getAll() throws IOException { HttpClient client = HttpClientBuilder.create() .setUserAgent("ApacheZeppelin/" + Util.getVersion()) + .setProxy(getProxy(uri())) .build(); HttpGet get = new HttpGet(uri()); HttpResponse response; @@ -78,7 +82,6 @@ public class HeliumOnlineRegistry extends HeliumRegistry { return readFromCache(); } - if (response.getStatusLine().getStatusCode() != 200) { // try read from cache logger.error(uri() + " returned " + response.getStatusLine().toString()); @@ -107,6 +110,31 @@ public class HeliumOnlineRegistry extends HeliumRegistry { } } + private HttpHost getProxy(String uri) { + String httpProxy = StringUtils.isBlank(System.getenv("http_proxy")) ? + System.getenv("HTTP_PROXY") : System.getenv("http_proxy"); + + String httpsProxy = StringUtils.isBlank(System.getenv("https_proxy")) ? + System.getenv("HTTPS_PROXY") : System.getenv("https_proxy"); + + try { + String scheme = new URI(uri).getScheme(); + if (scheme.toLowerCase().startsWith("https")) { + URI httpsProxyUri = new URI(httpsProxy); + return new HttpHost(httpsProxyUri.getHost(), + httpsProxyUri.getPort(), httpsProxyUri.getScheme()); + } + else { + URI httpProxyUri = new URI(httpProxy); + return new HttpHost(httpProxyUri.getHost(), + httpProxyUri.getPort(), httpProxyUri.getScheme()); + } + } catch (URISyntaxException ex) { + logger.error(ex.getMessage(), ex); + return null; + } + } + private List<HeliumPackage> readFromCache() { synchronized (registryCacheFile) { if (registryCacheFile.isFile()) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/87480056/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java index 5feac1d..aaabec5 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java @@ -39,7 +39,7 @@ public class HeliumBundleFactoryTest { private File tmpDir; private ZeppelinConfiguration conf; private HeliumBundleFactory hbf; - static File nodeInstallationDir = new File( + private static File nodeInstallationDir = new File( System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_nodeCache"); @BeforeClass
