Repository: zeppelin Updated Branches: refs/heads/master 427d9d494 -> b4238b997
[ZEPPELIN-2214] Set npm installer default npm registry ### What is this PR for? Complements ZEPPELIN-2278, this is a fix for "connection timeout" bug when trying to run "npm install" behind a proxy. Instead of connecting to the official nodejs site from behind a proxy, we should use the already soft-coded env variable ZEPPELIN_HELIUM_NPM_REGISTRY repository. ### What type of PR is it? Bug Fix ### ToDos * [x] - Soft-code npm folder name as well (currently hardcoded "npm") - To be agreed on... ### What is the Jira issue? * [ZEPPELIN-2214 - Bug on connecting to the official nodejs site from behind a proxy] ### How should this be tested? Enable any Helium plugin from behind a proxy ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Nelson Costa <[email protected]> Closes #2349 from necosta/zeppelin2214 and squashes the following commits: 6d2a52e [Nelson Costa] [ZEPPELIN-2214] Soft-code Node, NPM and YarnPkg download/installer urls for Helium module Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/b4238b99 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/b4238b99 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/b4238b99 Branch: refs/heads/master Commit: b4238b997433a39202c48aa679f624e0e30a6858 Parents: 427d9d4 Author: Nelson Costa <[email protected]> Authored: Sun May 28 10:45:58 2017 +0100 Committer: Lee moon soo <[email protected]> Committed: Thu Jun 8 19:05:25 2017 -0700 ---------------------------------------------------------------------- conf/zeppelin-env.cmd.template | 4 +- conf/zeppelin-env.sh.template | 4 +- conf/zeppelin-site.xml.template | 16 ++++- docs/install/configuration.md | 18 +++++- .../zeppelin/conf/ZeppelinConfiguration.java | 19 +++++- .../zeppelin/helium/HeliumBundleFactory.java | 63 ++++++++++---------- .../helium/HeliumBundleFactoryTest.java | 1 - 7 files changed, 81 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b4238b99/conf/zeppelin-env.cmd.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index cd1541e..e40429a 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -39,7 +39,9 @@ REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zep REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading -REM set ZEPPELIN_HELIUM_NPM_REGISTRY REM Remote Npm registry for Helium dependency loader +REM set ZEPPELIN_HELIUM_NODE_INSTALLER_URL REM Remote Node installer url for Helium dependency loader +REM set ZEPPELIN_HELIUM_NPM_INSTALLER_URL REM Remote Npm installer url for Helium dependency loader +REM set ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL REM Remote Yarn package installer url for Helium dependency loader REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth? http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b4238b99/conf/zeppelin-env.sh.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 754c8ef..ce55346 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -44,7 +44,9 @@ # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading # export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading -# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader +# export ZEPPELIN_HELIUM_NODE_INSTALLER_URL # Remote Node installer url for Helium dependency loader +# export ZEPPELIN_HELIUM_NPM_INSTALLER_URL # Remote Npm installer url for Helium dependency loader +# export ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL # Remote Yarn package installer url for Helium dependency loader # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? # export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b4238b99/conf/zeppelin-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 2a03cd9..584434b 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -252,9 +252,21 @@ </property> <property> - <name>zeppelin.helium.npm.registry</name> + <name>zeppelin.helium.node.installer.url</name> + <value>https://nodejs.org/dist/</value> + <description>Remote Node installer url for Helium dependency loader</description> +</property> + +<property> + <name>zeppelin.helium.npm.installer.url</name> <value>http://registry.npmjs.org/</value> - <description>Remote Npm registry for Helium dependency loader</description> + <description>Remote Npm installer url for Helium dependency loader</description> +</property> + +<property> + <name>zeppelin.helium.yarnpkg.installer.url</name> + <value>https://github.com/yarnpkg/yarn/releases/download/</value> + <description>Remote Yarn package installer url for Helium dependency loader</description> </property> <property> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b4238b99/docs/install/configuration.md ---------------------------------------------------------------------- diff --git a/docs/install/configuration.md b/docs/install/configuration.md index a35eb91..ca55397 100644 --- a/docs/install/configuration.md +++ b/docs/install/configuration.md @@ -282,10 +282,22 @@ If both are defined, then the **environment variables** will take priority. <td>Local repository for dependency loader.<br>ex)visualiztion modules of npm.</td> </tr> <tr> - <td><h6 class="properties">ZEPPELIN_HELIUM_NPM_REGISTRY</h6></td> - <td><h6 class="properties">zeppelin.helium.npm.registry</h6></td> + <td><h6 class="properties">ZEPPELIN_HELIUM_NODE_INSTALLER_URL</h6></td> + <td><h6 class="properties">zeppelin.helium.node.installer.url</h6></td> + <td>https://nodejs.org/dist/</td> + <td>Remote Node installer url for Helium dependency loader</td> + </tr> + <tr> + <td><h6 class="properties">ZEPPELIN_HELIUM_NPM_INSTALLER_URL</h6></td> + <td><h6 class="properties">zeppelin.helium.npm.installer.url</h6></td> <td>http://registry.npmjs.org/</td> - <td>Remote Npm registry for Helium dependency loader</td> + <td>Remote Npm installer url for Helium dependency loader</td> + </tr> + <tr> + <td><h6 class="properties">ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL</h6></td> + <td><h6 class="properties">zeppelin.helium.yarnpkg.installer.url</h6></td> + <td>https://github.com/yarnpkg/yarn/releases/download/</td> + <td>Remote Yarn package installer url for Helium dependency loader</td> </tr> <tr> <td><h6 class="properties">ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</h6></td> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b4238b99/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 66beb48..ade1fbd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -424,8 +424,16 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY); } - public String getHeliumNpmRegistry() { - return getString(ConfVars.ZEPPELIN_HELIUM_NPM_REGISTRY); + public String getHeliumNodeInstallerUrl() { + return getString(ConfVars.ZEPPELIN_HELIUM_NODE_INSTALLER_URL); + } + + public String getHeliumNpmInstallerUrl() { + return getString(ConfVars.ZEPPELIN_HELIUM_NPM_INSTALLER_URL); + } + + public String getHeliumYarnInstallerUrl() { + return getString(ConfVars.ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL); } public String getNotebookAuthorizationPath() { @@ -643,7 +651,12 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL), - ZEPPELIN_HELIUM_NPM_REGISTRY("zeppelin.helium.npm.registry", "http://registry.npmjs.org/";), + ZEPPELIN_HELIUM_NODE_INSTALLER_URL("zeppelin.helium.node.installer.url", + "https://nodejs.org/dist/";), + ZEPPELIN_HELIUM_NPM_INSTALLER_URL("zeppelin.helium.npm.installer.url", + "http://registry.npmjs.org/";), + ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL("zeppelin.helium.yarnpkg.installer.url", + "https://github.com/yarnpkg/yarn/releases/download/";), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b4238b99/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java index 52e4f6d..737fa55 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java @@ -49,23 +49,23 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; */ public class HeliumBundleFactory { Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class); - private final String NODE_VERSION = "v6.9.1"; - private final String NPM_VERSION = "3.10.8"; - private final String YARN_VERSION = "v0.21.3"; + private static final String NODE_VERSION = "v6.9.1"; + private static final String NPM_VERSION = "3.10.8"; + private static final String YARN_VERSION = "v0.21.3"; private static final String NPM_PACKAGE_NAME = "npm"; - public static final String HELIUM_LOCAL_REPO = "helium-bundle"; - public static final String HELIUM_BUNDLES_DIR = "bundles"; - public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules"; - public static final String HELIUM_BUNDLES_SRC_DIR = "src"; - public static final String HELIUM_BUNDLES_SRC = "load.js"; - public static final String YARN_CACHE_DIR = "yarn-cache"; - public static final String PACKAGE_JSON = "package.json"; - public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js"; - public static final String HELIUM_BUNDLE = "helium.bundle.js"; - public static final String HELIUM_BUNDLES_VAR = "heliumBundles"; - private final int FETCH_RETRY_COUNT = 2; - private final int FETCH_RETRY_FACTOR_COUNT = 1; - private final int FETCH_RETRY_MIN_TIMEOUT = 5000; // Milliseconds + protected static final String HELIUM_LOCAL_REPO = "helium-bundle"; + private static final String HELIUM_BUNDLES_DIR = "bundles"; + private static final String HELIUM_LOCAL_MODULE_DIR = "local_modules"; + private static final String HELIUM_BUNDLES_SRC_DIR = "src"; + private static final String HELIUM_BUNDLES_SRC = "load.js"; + private static final String YARN_CACHE_DIR = "yarn-cache"; + private static final String PACKAGE_JSON = "package.json"; + private static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js"; + private static final String HELIUM_BUNDLE = "helium.bundle.js"; + private static final String HELIUM_BUNDLES_VAR = "heliumBundles"; + private static final int FETCH_RETRY_COUNT = 2; + private static final int FETCH_RETRY_FACTOR_COUNT = 1; + private static final int FETCH_RETRY_MIN_TIMEOUT = 5000; // Milliseconds private final FrontendPluginFactory frontEndPluginFactory; private final File nodeInstallationDirectory; @@ -73,13 +73,12 @@ public class HeliumBundleFactory { private final File heliumBundleDirectory; private final File heliumLocalModuleDirectory; private final File yarnCacheDir; - private ZeppelinConfiguration conf; private File tabledataModulePath; private File visualizationModulePath; private File spellModulePath; - private String defaultNodeRegistryUrl; - private String defaultNpmRegistryUrl; - private String defaultYarnRegistryUrl; + private String defaultNodeInstallerUrl; + private String defaultNpmInstallerUrl; + private String defaultYarnInstallerUrl; private Gson gson; private boolean nodeAndNpmInstalled = false; @@ -106,11 +105,9 @@ public class HeliumBundleFactory { this.heliumBundleDirectory = new File(heliumLocalRepoDirectory, HELIUM_BUNDLES_DIR); this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR); this.yarnCacheDir = new File(heliumLocalRepoDirectory, YARN_CACHE_DIR); - this.conf = conf; - // To be done in ZEPPELIN-2214: Soft-code installer urls - this.defaultNodeRegistryUrl = "https://nodejs.org/dist/";; - this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry(); - this.defaultYarnRegistryUrl = "https://github.com/yarnpkg/yarn/releases/download/";; + this.defaultNodeInstallerUrl = conf.getHeliumNodeInstallerUrl(); + this.defaultNpmInstallerUrl = conf.getHeliumNpmInstallerUrl(); + this.defaultYarnInstallerUrl = conf.getHeliumYarnInstallerUrl(); nodeInstallationDirectory = (nodeInstallationDir == null) ? heliumLocalRepoDirectory : nodeInstallationDir; @@ -127,21 +124,21 @@ public class HeliumBundleFactory { } try { NodeInstaller nodeInstaller = frontEndPluginFactory - .getNodeInstaller(getProxyConfig(isSecure(defaultNodeRegistryUrl))); + .getNodeInstaller(getProxyConfig(isSecure(defaultNodeInstallerUrl))); nodeInstaller.setNodeVersion(NODE_VERSION); - nodeInstaller.setNodeDownloadRoot(defaultNodeRegistryUrl); + nodeInstaller.setNodeDownloadRoot(defaultNodeInstallerUrl); nodeInstaller.install(); NPMInstaller npmInstaller = frontEndPluginFactory - .getNPMInstaller(getProxyConfig(isSecure(defaultNpmRegistryUrl))); + .getNPMInstaller(getProxyConfig(isSecure(defaultNpmInstallerUrl))); npmInstaller.setNpmVersion(NPM_VERSION); - npmInstaller.setNpmDownloadRoot(defaultNpmRegistryUrl + "/" + NPM_PACKAGE_NAME + "/-/"); + npmInstaller.setNpmDownloadRoot(defaultNpmInstallerUrl + "/" + NPM_PACKAGE_NAME + "/-/"); npmInstaller.install(); YarnInstaller yarnInstaller = frontEndPluginFactory - .getYarnInstaller(getProxyConfig(isSecure(defaultYarnRegistryUrl))); + .getYarnInstaller(getProxyConfig(isSecure(defaultYarnInstallerUrl))); yarnInstaller.setYarnVersion(YARN_VERSION); - yarnInstaller.setYarnDownloadRoot(defaultYarnRegistryUrl); + yarnInstaller.setYarnDownloadRoot(defaultYarnInstallerUrl); yarnInstaller.install(); yarnCacheDir.mkdirs(); String yarnCacheDirPath = yarnCacheDir.getAbsolutePath(); @@ -664,7 +661,7 @@ public class HeliumBundleFactory { private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException { NpmRunner npm = frontEndPluginFactory.getNpmRunner( - getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl); + getProxyConfig(isSecure(defaultNpmInstallerUrl)), defaultNpmInstallerUrl); npm.execute(args, env); } @@ -679,7 +676,7 @@ public class HeliumBundleFactory { private void yarnCommand(FrontendPluginFactory fpf, String args, Map<String, String> env) throws TaskRunnerException { YarnRunner yarn = fpf.getYarnRunner( - getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl); + getProxyConfig(isSecure(defaultNpmInstallerUrl)), defaultNpmInstallerUrl); yarn.execute(args, env); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b4238b99/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java index aaabec5..1dafee1 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java @@ -119,7 +119,6 @@ public class HeliumBundleFactoryTest { assertEquals(lastModified, bundle.lastModified()); } - @Test public void bundleLocalPackage() throws IOException, TaskRunnerException { URL res = Resources.getResource("helium/webpack.config.js");
