Repository: zeppelin Updated Branches: refs/heads/master 019df1f6b -> 0589e27e7
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-web/test/spec/controllers/paragraph.js ---------------------------------------------------------------------- diff --git a/zeppelin-web/test/spec/controllers/paragraph.js b/zeppelin-web/test/spec/controllers/paragraph.js index 4089d2d..1aa4a4a 100644 --- a/zeppelin-web/test/spec/controllers/paragraph.js +++ b/zeppelin-web/test/spec/controllers/paragraph.js @@ -41,10 +41,6 @@ describe('Controller: ParagraphCtrl', function() { }); }); - it('should return "TEXT" by default when getResultType() is called with no parameter', function() { - expect(scope.getResultType()).toEqual('TEXT'); - }); - it('should have this array of values for "colWidthOption"', function() { expect(scope.colWidthOption).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); }); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-web/webpack.config.js ---------------------------------------------------------------------- diff --git a/zeppelin-web/webpack.config.js b/zeppelin-web/webpack.config.js index d3a2681..8198b55 100644 --- a/zeppelin-web/webpack.config.js +++ b/zeppelin-web/webpack.config.js @@ -210,7 +210,7 @@ module.exports = function makeWebpackConfig () { // Reference: https://webpack.github.io/docs/list-of-plugins.html#defineplugin new webpack.DefinePlugin({ 'process.env': { - HELIUM_VIS_DEV: process.env.HELIUM_VIS_DEV + HELIUM_BUNDLE_DEV: process.env.HELIUM_BUNDLE_DEV } }) ) http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java index 0ef3237..e2e1b49 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -45,20 +45,20 @@ public class Helium { private final File registryCacheDir; private final Gson gson; - private final HeliumVisualizationFactory visualizationFactory; + private final HeliumBundleFactory bundleFactory; private final HeliumApplicationFactory applicationFactory; public Helium( String heliumConfPath, String registryPaths, File registryCacheDir, - HeliumVisualizationFactory visualizationFactory, + HeliumBundleFactory bundleFactory, HeliumApplicationFactory applicationFactory) throws IOException { this.heliumConfPath = heliumConfPath; this.registryPaths = registryPaths; this.registryCacheDir = registryCacheDir; - this.visualizationFactory = visualizationFactory; + this.bundleFactory = bundleFactory; this.applicationFactory = applicationFactory; GsonBuilder builder = new GsonBuilder(); @@ -95,8 +95,8 @@ public class Helium { return applicationFactory; } - public HeliumVisualizationFactory getVisualizationFactory() { - return visualizationFactory; + public HeliumBundleFactory getBundleFactory() { + return bundleFactory; } private synchronized HeliumConf loadConf(String path) throws IOException { @@ -145,14 +145,14 @@ public class Helium { Map<String, List<HeliumPackageSearchResult>> all = getAllPackageInfo(); // clear visualization display order - List<String> packageOrder = heliumConf.getVisualizationDisplayOrder(); + List<String> packageOrder = heliumConf.getBundleDisplayOrder(); List<String> clearedOrder = new LinkedList<>(); for (String pkgName : packageOrder) { if (all.containsKey(pkgName)) { clearedOrder.add(pkgName); } } - heliumConf.setVisualizationDisplayOrder(clearedOrder); + heliumConf.setBundleDisplayOrder(clearedOrder); // clear enabled package Map<String, String> enabledPackages = heliumConf.getEnabledPackages(); @@ -215,8 +215,8 @@ public class Helium { return null; } - public File recreateVisualizationBundle() throws IOException { - return visualizationFactory.bundle(getVisualizationPackagesToBundle(), true); + public File recreateBundle() throws IOException { + return bundleFactory.buildBundle(getBundlePackagesToBundle(), true); } public void enable(String name, String artifact) throws IOException { @@ -231,8 +231,8 @@ public class Helium { heliumConf.enablePackage(name, artifact); // if package is visualization, rebuild bundle - if (pkgInfo.getPkg().getType() == HeliumPackage.Type.VISUALIZATION) { - visualizationFactory.bundle(getVisualizationPackagesToBundle()); + if (HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) { + bundleFactory.buildBundle(getBundlePackagesToBundle()); } save(); @@ -247,9 +247,9 @@ public class Helium { heliumConf.disablePackage(name); - HeliumPackageSearchResult pkg = getPackageInfo(name, artifact); - if (pkg == null || pkg.getPkg().getType() == HeliumPackage.Type.VISUALIZATION) { - visualizationFactory.bundle(getVisualizationPackagesToBundle()); + HeliumPackageSearchResult pkgInfo = getPackageInfo(name, artifact); + if (pkgInfo == null || HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) { + bundleFactory.buildBundle(getBundlePackagesToBundle()); } save(); @@ -278,7 +278,7 @@ public class Helium { for (List<HeliumPackageSearchResult> pkgs : getAllPackageInfo().values()) { for (HeliumPackageSearchResult pkg : pkgs) { - if (pkg.getPkg().getType() == HeliumPackage.Type.APPLICATION && pkg.isEnabled()) { + if (pkg.getPkg().getType() == HeliumType.APPLICATION && pkg.isEnabled()) { ResourceSet resources = ApplicationLoader.findRequiredResourceSet( pkg.getPkg().getResources(), paragraph.getNote().getId(), @@ -299,15 +299,15 @@ public class Helium { } /** - * Get enabled visualization packages + * Get enabled buildBundle packages * - * @return ordered list of enabled visualization package + * @return ordered list of enabled buildBundle package */ - public List<HeliumPackage> getVisualizationPackagesToBundle() { + public List<HeliumPackage> getBundlePackagesToBundle() { Map<String, List<HeliumPackageSearchResult>> allPackages = getAllPackageInfo(); - List<String> visOrder = heliumConf.getVisualizationDisplayOrder(); + List<String> visOrder = heliumConf.getBundleDisplayOrder(); - List<HeliumPackage> orderedVisualizationPackages = new LinkedList<>(); + List<HeliumPackage> orderedBundlePackages = new LinkedList<>(); // add enabled packages in visOrder for (String name : visOrder) { @@ -316,8 +316,8 @@ public class Helium { continue; } for (HeliumPackageSearchResult pkgInfo : versions) { - if (pkgInfo.getPkg().getType() == HeliumPackage.Type.VISUALIZATION && pkgInfo.isEnabled()) { - orderedVisualizationPackages.add(pkgInfo.getPkg()); + if (canBundle(pkgInfo)) { + orderedBundlePackages.add(pkgInfo.getPkg()); allPackages.remove(name); break; } @@ -325,28 +325,35 @@ public class Helium { } // add enabled packages not in visOrder - for (List<HeliumPackageSearchResult> pkgs : allPackages.values()) { - for (HeliumPackageSearchResult pkg : pkgs) { - if (pkg.getPkg().getType() == HeliumPackage.Type.VISUALIZATION && pkg.isEnabled()) { - orderedVisualizationPackages.add(pkg.getPkg()); + for (List<HeliumPackageSearchResult> pkgInfos : allPackages.values()) { + for (HeliumPackageSearchResult pkgInfo : pkgInfos) { + if (canBundle(pkgInfo)) { + orderedBundlePackages.add(pkgInfo.getPkg()); break; } } } - return orderedVisualizationPackages; + return orderedBundlePackages; + } + + public boolean canBundle(HeliumPackageSearchResult pkgInfo) { + return (pkgInfo.isEnabled() && + HeliumPackage.isBundleType(pkgInfo.getPkg().getType())); } /** * Get enabled package list in order * @return */ - public List<String> getVisualizationPackageOrder() { + public List<String> setVisualizationPackageOrder() { List orderedPackageList = new LinkedList<>(); - List<HeliumPackage> packages = getVisualizationPackagesToBundle(); + List<HeliumPackage> packages = getBundlePackagesToBundle(); for (HeliumPackage pkg : packages) { - orderedPackageList.add(pkg.getName()); + if (HeliumType.VISUALIZATION == pkg.getType()) { + orderedPackageList.add(pkg.getName()); + } } return orderedPackageList; @@ -354,10 +361,10 @@ public class Helium { public void setVisualizationPackageOrder(List<String> orderedPackageList) throws IOException { - heliumConf.setVisualizationDisplayOrder(orderedPackageList); + heliumConf.setBundleDisplayOrder(orderedPackageList); - // if package is visualization, rebuild bundle - visualizationFactory.bundle(getVisualizationPackagesToBundle()); + // if package is visualization, rebuild buildBundle + bundleFactory.buildBundle(getBundlePackagesToBundle()); save(); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java new file mode 100644 index 0000000..664030f --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.github.eirslett.maven.plugins.frontend.lib.*; +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import com.google.gson.Gson; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Appender; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.WriterAppender; +import org.apache.log4j.spi.Filter; +import org.apache.log4j.spi.LoggingEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URL; +import java.util.*; + +/** + * Load helium visualization + */ +public class HeliumBundleFactory { + Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class); + private final String NODE_VERSION = "v6.9.1"; + private final String NPM_VERSION = "3.10.8"; + private final String DEFAULT_NPM_REGISTRY_URL = "http://registry.npmjs.org/";; + public static final String HELIUM_LOCAL_REPO = "helium-bundle"; + public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js"; + public static final String HELIUM_BUNDLE = "helium.bundle.js"; + public static final String HELIUM_BUNDLES_VAR = "heliumBundles"; + + private final FrontendPluginFactory frontEndPluginFactory; + private final File workingDirectory; + private File tabledataModulePath; + private File visualizationModulePath; + private File spellModulePath; + private Gson gson; + + String bundleCacheKey = ""; + File currentCacheBundle; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + public HeliumBundleFactory( + File moduleDownloadPath, + File tabledataModulePath, + File visualizationModulePath, + File spellModulePath) throws TaskRunnerException { + this(moduleDownloadPath); + this.tabledataModulePath = tabledataModulePath; + this.visualizationModulePath = visualizationModulePath; + this.spellModulePath = spellModulePath; + } + + public HeliumBundleFactory(File moduleDownloadPath) throws TaskRunnerException { + this.workingDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO); + File installDirectory = workingDirectory; + + frontEndPluginFactory = new FrontendPluginFactory( + workingDirectory, installDirectory); + + currentCacheBundle = new File(workingDirectory, HELIUM_BUNDLE_CACHE); + gson = new Gson(); + installNodeAndNpm(); + configureLogger(); + } + + private void installNodeAndNpm() { + try { + NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig()); + npmInstaller.setNpmVersion(NPM_VERSION); + npmInstaller.install(); + + NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig()); + nodeInstaller.setNodeVersion(NODE_VERSION); + nodeInstaller.install(); + } catch (InstallationException e) { + logger.error(e.getMessage(), e); + } + } + + private ProxyConfig getProxyConfig() { + List<ProxyConfig.Proxy> proxy = new LinkedList<>(); + return new ProxyConfig(proxy); + } + + public File buildBundle(List<HeliumPackage> pkgs) throws IOException { + return buildBundle(pkgs, false); + } + + public synchronized File buildBundle(List<HeliumPackage> pkgs, boolean forceRefresh) + throws IOException { + // package.json + URL pkgUrl = Resources.getResource("helium/package.json"); + String pkgJson = Resources.toString(pkgUrl, Charsets.UTF_8); + StringBuilder dependencies = new StringBuilder(); + StringBuilder cacheKeyBuilder = new StringBuilder(); + + FileFilter npmPackageCopyFilter = new FileFilter() { + @Override + public boolean accept(File pathname) { + String fileName = pathname.getName(); + if (fileName.startsWith(".") || fileName.startsWith("#") || fileName.startsWith("~")) { + return false; + } else { + return true; + } + } + }; + + for (HeliumPackage pkg : pkgs) { + String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg); + if (moduleNameVersion == null) { + logger.error("Can't get module name and version of package " + pkg.getName()); + continue; + } + if (dependencies.length() > 0) { + dependencies.append(",\n"); + } + dependencies.append("\"" + moduleNameVersion[0] + "\": \"" + moduleNameVersion[1] + "\""); + cacheKeyBuilder.append(pkg.getName() + pkg.getArtifact()); + + File pkgInstallDir = new File(workingDirectory, "node_modules/" + pkg.getName()); + if (pkgInstallDir.exists()) { + FileUtils.deleteDirectory(pkgInstallDir); + } + + if (isLocalPackage(pkg)) { + FileUtils.copyDirectory( + new File(pkg.getArtifact()), + pkgInstallDir, + npmPackageCopyFilter); + } + } + pkgJson = pkgJson.replaceFirst("DEPENDENCIES", dependencies.toString()); + + // check if we can use previous buildBundle or not + if (cacheKeyBuilder.toString().equals(bundleCacheKey) && + currentCacheBundle.isFile() && !forceRefresh) { + return currentCacheBundle; + } + + // webpack.config.js + URL webpackConfigUrl = Resources.getResource("helium/webpack.config.js"); + String webpackConfig = Resources.toString(webpackConfigUrl, Charsets.UTF_8); + + // generate load.js + StringBuilder loadJsImport = new StringBuilder(); + StringBuilder loadJsRegister = new StringBuilder(); + + long idx = 0; + for (HeliumPackage pkg : pkgs) { + String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg); + if (moduleNameVersion == null) { + continue; + } + + String className = "bundles" + idx++; + loadJsImport.append( + "import " + className + " from \"" + moduleNameVersion[0] + "\"\n"); + + loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n"); + loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n"); + loadJsRegister.append("name: \"" + pkg.getName() + "\",\n"); + loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n"); + loadJsRegister.append("type: \"" + pkg.getType() + "\",\n"); + loadJsRegister.append("class: " + className + "\n"); + loadJsRegister.append("})\n"); + } + + FileUtils.write(new File(workingDirectory, "package.json"), pkgJson); + FileUtils.write(new File(workingDirectory, "webpack.config.js"), webpackConfig); + FileUtils.write(new File(workingDirectory, "load.js"), + loadJsImport.append(loadJsRegister).toString()); + + copyFrameworkModuleToInstallPath(npmPackageCopyFilter); + + try { + out.reset(); + npmCommand("install --loglevel=error"); + } catch (TaskRunnerException e) { + // ignore `(empty)` warning + String cause = new String(out.toByteArray()); + if (!cause.contains("(empty)")) { + throw new IOException(cause); + } + } + + try { + out.reset(); + npmCommand("run bundle"); + } catch (TaskRunnerException e) { + throw new IOException(new String(out.toByteArray())); + } + + String bundleStdoutResult = new String(out.toByteArray()); + + File heliumBundle = new File(workingDirectory, HELIUM_BUNDLE); + if (!heliumBundle.isFile()) { + throw new IOException( + "Can't create bundle: \n" + bundleStdoutResult); + } + + WebpackResult result = getWebpackResultFromOutput(bundleStdoutResult); + if (result.errors.length > 0) { + heliumBundle.delete(); + throw new IOException(result.errors[0]); + } + + synchronized (this) { + currentCacheBundle.delete(); + FileUtils.moveFile(heliumBundle, currentCacheBundle); + bundleCacheKey = cacheKeyBuilder.toString(); + } + return currentCacheBundle; + } + + private void copyFrameworkModuleToInstallPath(FileFilter npmPackageCopyFilter) + throws IOException { + // install tabledata module + File tabledataModuleInstallPath = new File(workingDirectory, + "node_modules/zeppelin-tabledata"); + if (tabledataModulePath != null) { + if (tabledataModuleInstallPath.exists()) { + FileUtils.deleteDirectory(tabledataModuleInstallPath); + } + FileUtils.copyDirectory( + tabledataModulePath, + tabledataModuleInstallPath, + npmPackageCopyFilter); + } + + // install visualization module + File visModuleInstallPath = new File(workingDirectory, + "node_modules/zeppelin-vis"); + if (visualizationModulePath != null) { + if (visModuleInstallPath.exists()) { + // when zeppelin-vis and zeppelin-table package is published to npm repository + // we don't need to remove module because npm install command will take care + // dependency version change. However, when two dependencies are copied manually + // into node_modules directory, changing vis package version results inconsistent npm + // install behavior. + // + // Remote vis package everytime and let npm download every time bundle as a workaround + FileUtils.deleteDirectory(visModuleInstallPath); + } + FileUtils.copyDirectory(visualizationModulePath, visModuleInstallPath, npmPackageCopyFilter); + } + + // install spell module + File spellModuleInstallPath = new File(workingDirectory, + "node_modules/zeppelin-spell"); + if (spellModulePath != null) { + if (spellModuleInstallPath.exists()) { + FileUtils.deleteDirectory(spellModuleInstallPath); + } + + FileUtils.copyDirectory( + spellModulePath, + spellModuleInstallPath, + npmPackageCopyFilter); + } + } + + private WebpackResult getWebpackResultFromOutput(String output) { + BufferedReader reader = new BufferedReader(new StringReader(output)); + + String line; + boolean webpackRunDetected = false; + boolean resultJsonDetected = false; + StringBuffer sb = new StringBuffer(); + try { + while ((line = reader.readLine()) != null) { + if (!webpackRunDetected) { + if (line.contains("webpack.js") && line.endsWith("--json")) { + webpackRunDetected = true; + } + continue; + } + + if (!resultJsonDetected) { + if (line.equals("{")) { + sb.append(line); + resultJsonDetected = true; + } + continue; + } + + if (resultJsonDetected && webpackRunDetected) { + sb.append(line); + } + } + + Gson gson = new Gson(); + return gson.fromJson(sb.toString(), WebpackResult.class); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return new WebpackResult(); + } + } + + public File getCurrentCacheBundle() { + synchronized (this) { + if (currentCacheBundle.isFile()) { + return currentCacheBundle; + } else { + return null; + } + } + } + + private boolean isLocalPackage(HeliumPackage pkg) { + return (pkg.getArtifact().startsWith(".") || pkg.getArtifact().startsWith("/")); + } + + private String[] getNpmModuleNameAndVersion(HeliumPackage pkg) { + String artifact = pkg.getArtifact(); + + if (isLocalPackage(pkg)) { + File packageJson = new File(artifact, "package.json"); + if (!packageJson.isFile()) { + return null; + } + Gson gson = new Gson(); + try { + NpmPackage npmPackage = gson.fromJson( + FileUtils.readFileToString(packageJson), + NpmPackage.class); + + String[] nameVersion = new String[2]; + nameVersion[0] = npmPackage.name; + nameVersion[1] = npmPackage.version; + return nameVersion; + } catch (IOException e) { + logger.error(e.getMessage(), e); + return null; + } + } else { + String[] nameVersion = new String[2]; + + int pos; + if ((pos = artifact.indexOf('@')) > 0) { + nameVersion[0] = artifact.substring(0, pos); + nameVersion[1] = artifact.substring(pos + 1); + } else if ( + (pos = artifact.indexOf('^')) > 0 || + (pos = artifact.indexOf('~')) > 0) { + nameVersion[0] = artifact.substring(0, pos); + nameVersion[1] = artifact.substring(pos); + } else { + nameVersion[0] = artifact; + nameVersion[1] = ""; + } + return nameVersion; + } + } + + public synchronized void install(HeliumPackage pkg) throws TaskRunnerException { + npmCommand("install " + pkg.getArtifact() + " npm install --loglevel=error"); + } + + private void npmCommand(String args) throws TaskRunnerException { + npmCommand(args, new HashMap<String, String>()); + } + + private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException { + NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), DEFAULT_NPM_REGISTRY_URL); + + npm.execute(args, env); + } + + private void configureLogger() { + org.apache.log4j.Logger npmLogger = org.apache.log4j.Logger.getLogger( + "com.github.eirslett.maven.plugins.frontend.lib.DefaultNpmRunner"); + Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders(); + + if (appenders != null) { + while (appenders.hasMoreElements()) { + Appender appender = (Appender) appenders.nextElement(); + appender.addFilter(new Filter() { + + @Override + public int decide(LoggingEvent loggingEvent) { + if (loggingEvent.getLoggerName().contains("DefaultNpmRunner")) { + return DENY; + } else { + return NEUTRAL; + } + } + }); + } + } + npmLogger.addAppender(new WriterAppender( + new PatternLayout("%m%n"), + out + )); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java index 5094934..d60aec7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java @@ -26,7 +26,7 @@ public class HeliumConf { Map<String, String> enabled = Collections.synchronizedMap(new HashMap<String, String>()); // enabled visualization package display order - List<String> visualizationDisplayOrder = new LinkedList<>(); + List<String> bundleDisplayOrder = new LinkedList<>(); public Map<String, String> getEnabledPackages() { return new HashMap<>(enabled); @@ -48,15 +48,15 @@ public class HeliumConf { enabled.remove(name); } - public List<String> getVisualizationDisplayOrder() { - if (visualizationDisplayOrder == null) { + public List<String> getBundleDisplayOrder() { + if (bundleDisplayOrder == null) { return new LinkedList<String>(); } else { - return visualizationDisplayOrder; + return bundleDisplayOrder; } } - public void setVisualizationDisplayOrder(List<String> orderedPackageList) { - visualizationDisplayOrder = orderedPackageList; + public void setBundleDisplayOrder(List<String> orderedPackageList) { + bundleDisplayOrder = orderedPackageList; } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java deleted file mode 100644 index 624f12a..0000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.zeppelin.helium; - -import com.github.eirslett.maven.plugins.frontend.lib.*; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; -import com.google.gson.Gson; -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Appender; -import org.apache.log4j.PatternLayout; -import org.apache.log4j.WriterAppender; -import org.apache.log4j.spi.Filter; -import org.apache.log4j.spi.LoggingEvent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.net.URL; -import java.util.*; - -/** - * Load helium visualization - */ -public class HeliumVisualizationFactory { - Logger logger = LoggerFactory.getLogger(HeliumVisualizationFactory.class); - private final String NODE_VERSION = "v6.9.1"; - private final String NPM_VERSION = "3.10.8"; - private final String DEFAULT_NPM_REGISTRY_URL = "http://registry.npmjs.org/";; - - private final FrontendPluginFactory frontEndPluginFactory; - private final File workingDirectory; - private File tabledataModulePath; - private File visualizationModulePath; - private Gson gson; - - String bundleCacheKey = ""; - File currentBundle; - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - - public HeliumVisualizationFactory( - File moduleDownloadPath, - File tabledataModulePath, - File visualizationModulePath) throws TaskRunnerException { - this(moduleDownloadPath); - this.tabledataModulePath = tabledataModulePath; - this.visualizationModulePath = visualizationModulePath; - } - - public HeliumVisualizationFactory(File moduleDownloadPath) throws TaskRunnerException { - this.workingDirectory = new File(moduleDownloadPath, "vis"); - File installDirectory = workingDirectory; - - frontEndPluginFactory = new FrontendPluginFactory( - workingDirectory, installDirectory); - - currentBundle = new File(workingDirectory, "vis.bundle.cache.js"); - gson = new Gson(); - installNodeAndNpm(); - configureLogger(); - } - - private void installNodeAndNpm() { - try { - NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig()); - npmInstaller.setNpmVersion(NPM_VERSION); - npmInstaller.install(); - - NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig()); - nodeInstaller.setNodeVersion(NODE_VERSION); - nodeInstaller.install(); - } catch (InstallationException e) { - logger.error(e.getMessage(), e); - } - } - - private ProxyConfig getProxyConfig() { - List<ProxyConfig.Proxy> proxy = new LinkedList<>(); - return new ProxyConfig(proxy); - } - - public File bundle(List<HeliumPackage> pkgs) throws IOException { - return bundle(pkgs, false); - } - - public synchronized File bundle(List<HeliumPackage> pkgs, boolean forceRefresh) - throws IOException { - // package.json - URL pkgUrl = Resources.getResource("helium/package.json"); - String pkgJson = Resources.toString(pkgUrl, Charsets.UTF_8); - StringBuilder dependencies = new StringBuilder(); - StringBuilder cacheKeyBuilder = new StringBuilder(); - - FileFilter npmPackageCopyFilter = new FileFilter() { - @Override - public boolean accept(File pathname) { - String fileName = pathname.getName(); - if (fileName.startsWith(".") || fileName.startsWith("#") || fileName.startsWith("~")) { - return false; - } else { - return true; - } - } - }; - - for (HeliumPackage pkg : pkgs) { - String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg); - if (moduleNameVersion == null) { - logger.error("Can't get module name and version of package " + pkg.getName()); - continue; - } - if (dependencies.length() > 0) { - dependencies.append(",\n"); - } - dependencies.append("\"" + moduleNameVersion[0] + "\": \"" + moduleNameVersion[1] + "\""); - cacheKeyBuilder.append(pkg.getName() + pkg.getArtifact()); - - File pkgInstallDir = new File(workingDirectory, "node_modules/" + pkg.getName()); - if (pkgInstallDir.exists()) { - FileUtils.deleteDirectory(pkgInstallDir); - } - - if (isLocalPackage(pkg)) { - FileUtils.copyDirectory( - new File(pkg.getArtifact()), - pkgInstallDir, - npmPackageCopyFilter); - } - } - pkgJson = pkgJson.replaceFirst("DEPENDENCIES", dependencies.toString()); - - // check if we can use previous bundle or not - if (cacheKeyBuilder.toString().equals(bundleCacheKey) - && currentBundle.isFile() && !forceRefresh) { - return currentBundle; - } - - // webpack.config.js - URL webpackConfigUrl = Resources.getResource("helium/webpack.config.js"); - String webpackConfig = Resources.toString(webpackConfigUrl, Charsets.UTF_8); - - // generate load.js - StringBuilder loadJsImport = new StringBuilder(); - StringBuilder loadJsRegister = new StringBuilder(); - - long idx = 0; - for (HeliumPackage pkg : pkgs) { - String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg); - if (moduleNameVersion == null) { - continue; - } - - String className = "vis" + idx++; - loadJsImport.append( - "import " + className + " from \"" + moduleNameVersion[0] + "\"\n"); - - loadJsRegister.append("visualizations.push({\n"); - loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n"); - loadJsRegister.append("name: \"" + pkg.getName() + "\",\n"); - loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n"); - loadJsRegister.append("class: " + className + "\n"); - loadJsRegister.append("})\n"); - } - - FileUtils.write(new File(workingDirectory, "package.json"), pkgJson); - FileUtils.write(new File(workingDirectory, "webpack.config.js"), webpackConfig); - FileUtils.write(new File(workingDirectory, "load.js"), - loadJsImport.append(loadJsRegister).toString()); - - // install tabledata module - File tabledataModuleInstallPath = new File(workingDirectory, - "node_modules/zeppelin-tabledata"); - if (tabledataModulePath != null) { - if (tabledataModuleInstallPath.exists()) { - FileUtils.deleteDirectory(tabledataModuleInstallPath); - } - FileUtils.copyDirectory( - tabledataModulePath, - tabledataModuleInstallPath, - npmPackageCopyFilter); - } - - // install visualization module - File visModuleInstallPath = new File(workingDirectory, - "node_modules/zeppelin-vis"); - if (visualizationModulePath != null) { - if (visModuleInstallPath.exists()) { - // when zeppelin-vis and zeppelin-table package is published to npm repository - // we don't need to remove module because npm install command will take care - // dependency version change. However, when two dependencies are copied manually - // into node_modules directory, changing vis package version results inconsistent npm - // install behavior. - // - // Remote vis package everytime and let npm download every time bundle as a workaround - FileUtils.deleteDirectory(visModuleInstallPath); - } - FileUtils.copyDirectory(visualizationModulePath, visModuleInstallPath, npmPackageCopyFilter); - } - - out.reset(); - try { - npmCommand("install"); - npmCommand("run bundle"); - } catch (TaskRunnerException e) { - throw new IOException(new String(out.toByteArray())); - } - - File visBundleJs = new File(workingDirectory, "vis.bundle.js"); - if (!visBundleJs.isFile()) { - throw new IOException( - "Can't create visualization bundle : \n" + new String(out.toByteArray())); - } - - WebpackResult result = getWebpackResultFromOutput(new String(out.toByteArray())); - if (result.errors.length > 0) { - visBundleJs.delete(); - throw new IOException(result.errors[0]); - } - - synchronized (this) { - currentBundle.delete(); - FileUtils.moveFile(visBundleJs, currentBundle); - bundleCacheKey = cacheKeyBuilder.toString(); - } - return currentBundle; - } - - private WebpackResult getWebpackResultFromOutput(String output) { - BufferedReader reader = new BufferedReader(new StringReader(output)); - - String line; - boolean webpackRunDetected = false; - boolean resultJsonDetected = false; - StringBuffer sb = new StringBuffer(); - try { - while ((line = reader.readLine()) != null) { - if (!webpackRunDetected) { - if (line.contains("webpack.js") && line.endsWith("--json")) { - webpackRunDetected = true; - } - continue; - } - - if (!resultJsonDetected) { - if (line.equals("{")) { - sb.append(line); - resultJsonDetected = true; - } - continue; - } - - if (resultJsonDetected && webpackRunDetected) { - sb.append(line); - } - } - - Gson gson = new Gson(); - return gson.fromJson(sb.toString(), WebpackResult.class); - } catch (IOException e) { - logger.error(e.getMessage(), e); - return new WebpackResult(); - } - } - - public File getCurrentBundle() { - synchronized (this) { - if (currentBundle.isFile()) { - return currentBundle; - } else { - return null; - } - } - } - - private boolean isLocalPackage(HeliumPackage pkg) { - return (pkg.getArtifact().startsWith(".") || pkg.getArtifact().startsWith("/")); - } - - private String[] getNpmModuleNameAndVersion(HeliumPackage pkg) { - String artifact = pkg.getArtifact(); - - if (isLocalPackage(pkg)) { - File packageJson = new File(artifact, "package.json"); - if (!packageJson.isFile()) { - return null; - } - Gson gson = new Gson(); - try { - NpmPackage npmPackage = gson.fromJson( - FileUtils.readFileToString(packageJson), - NpmPackage.class); - - String[] nameVersion = new String[2]; - nameVersion[0] = npmPackage.name; - nameVersion[1] = npmPackage.version; - return nameVersion; - } catch (IOException e) { - logger.error(e.getMessage(), e); - return null; - } - } else { - String[] nameVersion = new String[2]; - - int pos; - if ((pos = artifact.indexOf('@')) > 0) { - nameVersion[0] = artifact.substring(0, pos); - nameVersion[1] = artifact.substring(pos + 1); - } else if ( - (pos = artifact.indexOf('^')) > 0 || - (pos = artifact.indexOf('~')) > 0) { - nameVersion[0] = artifact.substring(0, pos); - nameVersion[1] = artifact.substring(pos); - } else { - nameVersion[0] = artifact; - nameVersion[1] = ""; - } - return nameVersion; - } - } - - public synchronized void install(HeliumPackage pkg) throws TaskRunnerException { - npmCommand("install " + pkg.getArtifact()); - } - - private void npmCommand(String args) throws TaskRunnerException { - npmCommand(args, new HashMap<String, String>()); - } - - private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException { - NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), DEFAULT_NPM_REGISTRY_URL); - - npm.execute(args, env); - } - - private void configureLogger() { - org.apache.log4j.Logger npmLogger = org.apache.log4j.Logger.getLogger( - "com.github.eirslett.maven.plugins.frontend.lib.DefaultNpmRunner"); - Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders(); - - if (appenders != null) { - while (appenders.hasMoreElements()) { - Appender appender = (Appender) appenders.nextElement(); - appender.addFilter(new Filter() { - - @Override - public int decide(LoggingEvent loggingEvent) { - if (loggingEvent.getLoggerName().contains("DefaultNpmRunner")) { - return DENY; - } else { - return NEUTRAL; - } - } - }); - } - } - npmLogger.addAppender(new WriterAppender( - new PatternLayout("%m%n"), - out - )); - } -} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java index 162baf8..a6d1546 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java @@ -172,7 +172,9 @@ public class Message { PARAGRAPH_REMOVED, // [s-c] paragraph deleted PARAGRAPH_MOVED, // [s-c] paragraph moved NOTE_UPDATED, // [s-c] paragraph updated(name, config) - RUN_ALL_PARAGRAPHS // [c-s] run all paragraphs + RUN_ALL_PARAGRAPHS, // [c-s] run all paragraphs + PARAGRAPH_EXECUTED_BY_SPELL, // [c-s] paragraph was executed by spell + RUN_PARAGRAPH_USING_SPELL // [s-c] run paragraph using spell } public static final Message EMPTY = new Message(null); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/main/resources/helium/package.json ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/resources/helium/package.json b/zeppelin-zengine/src/main/resources/helium/package.json index e6ec612..cd4e470 100644 --- a/zeppelin-zengine/src/main/resources/helium/package.json +++ b/zeppelin-zengine/src/main/resources/helium/package.json @@ -1,5 +1,5 @@ { - "name": "zeppelin-vis-bundle", + "name": "zeppelin-helium-bundle", "main": "load", "scripts": { "bundle": "node/node node_modules/webpack/bin/webpack.js --display-error-details --json" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/main/resources/helium/webpack.config.js ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/resources/helium/webpack.config.js b/zeppelin-zengine/src/main/resources/helium/webpack.config.js index 2b5015e..c318c10 100644 --- a/zeppelin-zengine/src/main/resources/helium/webpack.config.js +++ b/zeppelin-zengine/src/main/resources/helium/webpack.config.js @@ -18,7 +18,7 @@ module.exports = { entry: ['./'], output: { path: './', - filename: 'vis.bundle.js', + filename: 'helium.bundle.js', }, resolve: { root: __dirname + "/node_modules" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 2588c4c..99cdeca 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -129,7 +129,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { public void testLoadRunUnloadApplication() throws IOException, ApplicationException, InterruptedException { // given - HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + HeliumPackage pkg1 = new HeliumPackage(HeliumType.APPLICATION, "name1", "desc1", "", @@ -175,7 +175,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { @Test public void testUnloadOnParagraphRemove() throws IOException { // given - HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + HeliumPackage pkg1 = new HeliumPackage(HeliumType.APPLICATION, "name1", "desc1", "", @@ -215,7 +215,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { @Test public void testUnloadOnInterpreterUnbind() throws IOException { // given - HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + HeliumPackage pkg1 = new HeliumPackage(HeliumType.APPLICATION, "name1", "desc1", "", @@ -276,7 +276,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { @Test public void testUnloadOnInterpreterRestart() throws IOException { // given - HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + HeliumPackage pkg1 = new HeliumPackage(HeliumType.APPLICATION, "name1", "desc1", "", http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java new file mode 100644 index 0000000..503cc07 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.github.eirslett.maven.plugins.frontend.lib.InstallationException; +import com.github.eirslett.maven.plugins.frontend.lib.TaskRunnerException; +import com.google.common.io.Resources; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.*; + +public class HeliumBundleFactoryTest { + private File tmpDir; + private HeliumBundleFactory hbf; + + @Before + public void setUp() throws InstallationException, TaskRunnerException { + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir.mkdirs(); + + // get module dir + URL res = Resources.getResource("helium/webpack.config.js"); + String resDir = new File(res.getFile()).getParent(); + File moduleDir = new File(resDir + "/../../../../zeppelin-web/src/app/"); + + hbf = new HeliumBundleFactory(tmpDir, + new File(moduleDir, "tabledata"), + new File(moduleDir, "visualization"), + new File(moduleDir, "spell")); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + @Test + public void testInstallNpm() throws InstallationException { + assertTrue(new File(tmpDir, + HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile()); + assertTrue(new File(tmpDir, + HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile()); + } + + @Test + public void downloadPackage() throws TaskRunnerException { + HeliumPackage pkg = new HeliumPackage( + HeliumType.VISUALIZATION, + "lodash", + "lodash", + "[email protected]", + "", + null, + "license", + "icon" + ); + hbf.install(pkg); + assertTrue(new File(tmpDir, + HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node_modules/lodash").isDirectory()); + } + + @Test + public void bundlePackage() throws IOException, TaskRunnerException { + HeliumPackage pkg = new HeliumPackage( + HeliumType.VISUALIZATION, + "zeppelin-bubblechart", + "zeppelin-bubblechart", + "[email protected]", + "", + null, + "license", + "icon" + ); + List<HeliumPackage> pkgs = new LinkedList<>(); + pkgs.add(pkg); + File bundle = hbf.buildBundle(pkgs); + assertTrue(bundle.isFile()); + long lastModified = bundle.lastModified(); + + // buildBundle again and check if it served from cache + bundle = hbf.buildBundle(pkgs); + assertEquals(lastModified, bundle.lastModified()); + } + + + @Test + public void bundleLocalPackage() throws IOException, TaskRunnerException { + URL res = Resources.getResource("helium/webpack.config.js"); + String resDir = new File(res.getFile()).getParent(); + String localPkg = resDir + "/../../../src/test/resources/helium/vis1"; + + HeliumPackage pkg = new HeliumPackage( + HeliumType.VISUALIZATION, + "vis1", + "vis1", + localPkg, + "", + null, + "license", + "fa fa-coffee" + ); + List<HeliumPackage> pkgs = new LinkedList<>(); + pkgs.add(pkg); + File bundle = hbf.buildBundle(pkgs); + assertTrue(bundle.isFile()); + } + + @Test + public void bundleErrorPropagation() throws IOException, TaskRunnerException { + URL res = Resources.getResource("helium/webpack.config.js"); + String resDir = new File(res.getFile()).getParent(); + String localPkg = resDir + "/../../../src/test/resources/helium/vis2"; + + HeliumPackage pkg = new HeliumPackage( + HeliumType.VISUALIZATION, + "vis2", + "vis2", + localPkg, + "", + null, + "license", + "fa fa-coffee" + ); + List<HeliumPackage> pkgs = new LinkedList<>(); + pkgs.add(pkg); + File bundle = null; + try { + bundle = hbf.buildBundle(pkgs); + // should throw exception + assertTrue(false); + } catch (IOException e) { + assertTrue(e.getMessage().contains("error in the package")); + } + assertNull(bundle); + } + + @Test + public void switchVersion() throws IOException, TaskRunnerException { + URL res = Resources.getResource("helium/webpack.config.js"); + String resDir = new File(res.getFile()).getParent(); + + HeliumPackage pkgV1 = new HeliumPackage( + HeliumType.VISUALIZATION, + "zeppelin-bubblechart", + "zeppelin-bubblechart", + "[email protected]", + "", + null, + "license", + "icon" + ); + + HeliumPackage pkgV2 = new HeliumPackage( + HeliumType.VISUALIZATION, + "zeppelin-bubblechart", + "zeppelin-bubblechart", + "[email protected]", + "", + null, + "license", + "icon" + ); + List<HeliumPackage> pkgsV1 = new LinkedList<>(); + pkgsV1.add(pkgV1); + + List<HeliumPackage> pkgsV2 = new LinkedList<>(); + pkgsV2.add(pkgV2); + + File bundle1 = hbf.buildBundle(pkgsV1); + File bundle2 = hbf.buildBundle(pkgsV2); + + assertNotSame(bundle1.lastModified(), bundle2.lastModified()); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java index 03d77b7..0f490d1 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java @@ -50,7 +50,7 @@ public class HeliumLocalRegistryTest { // when Gson gson = new Gson(); - HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + HeliumPackage pkg1 = new HeliumPackage(HeliumType.APPLICATION, "app1", "desc1", "artifact1", http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java index 9db9477..1607c2c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java @@ -24,7 +24,6 @@ import org.junit.Test; import java.io.File; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; import static org.junit.Assert.assertEquals; @@ -79,7 +78,7 @@ public class HeliumTest { // when registry1.add(new HeliumPackage( - HeliumPackage.Type.APPLICATION, + HeliumType.APPLICATION, "name1", "desc1", "artifact1", @@ -89,7 +88,7 @@ public class HeliumTest { "")); registry2.add(new HeliumPackage( - HeliumPackage.Type.APPLICATION, + HeliumType.APPLICATION, "name2", "desc2", "artifact2", http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0589e27e/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java deleted file mode 100644 index e5a61ed..0000000 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.zeppelin.helium; - -import com.github.eirslett.maven.plugins.frontend.lib.InstallationException; -import com.github.eirslett.maven.plugins.frontend.lib.TaskRunnerException; -import com.google.common.io.Resources; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.LinkedList; -import java.util.List; - -import static org.junit.Assert.*; - -public class HeliumVisualizationFactoryTest { - private File tmpDir; - private HeliumVisualizationFactory hvf; - - @Before - public void setUp() throws InstallationException, TaskRunnerException { - tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); - tmpDir.mkdirs(); - - // get module dir - URL res = Resources.getResource("helium/webpack.config.js"); - String resDir = new File(res.getFile()).getParent(); - File moduleDir = new File(resDir + "/../../../../zeppelin-web/src/app/"); - - hvf = new HeliumVisualizationFactory(tmpDir, - new File(moduleDir, "tabledata"), - new File(moduleDir, "visualization")); - } - - @After - public void tearDown() throws IOException { - FileUtils.deleteDirectory(tmpDir); - } - - @Test - public void testInstallNpm() throws InstallationException { - assertTrue(new File(tmpDir, "vis/node/npm").isFile()); - assertTrue(new File(tmpDir, "vis/node/node").isFile()); - } - - @Test - public void downloadPackage() throws TaskRunnerException { - HeliumPackage pkg = new HeliumPackage( - HeliumPackage.Type.VISUALIZATION, - "lodash", - "lodash", - "[email protected]", - "", - null, - "license", - "icon" - ); - hvf.install(pkg); - assertTrue(new File(tmpDir, "vis/node_modules/lodash").isDirectory()); - } - - @Test - public void bundlePackage() throws IOException, TaskRunnerException { - HeliumPackage pkg = new HeliumPackage( - HeliumPackage.Type.VISUALIZATION, - "zeppelin-bubblechart", - "zeppelin-bubblechart", - "[email protected]", - "", - null, - "license", - "icon" - ); - List<HeliumPackage> pkgs = new LinkedList<>(); - pkgs.add(pkg); - File bundle = hvf.bundle(pkgs); - assertTrue(bundle.isFile()); - long lastModified = bundle.lastModified(); - - // bundle again and check if it served from cache - bundle = hvf.bundle(pkgs); - assertEquals(lastModified, bundle.lastModified()); - } - - - @Test - public void bundleLocalPackage() throws IOException, TaskRunnerException { - URL res = Resources.getResource("helium/webpack.config.js"); - String resDir = new File(res.getFile()).getParent(); - String localPkg = resDir + "/../../../src/test/resources/helium/vis1"; - - HeliumPackage pkg = new HeliumPackage( - HeliumPackage.Type.VISUALIZATION, - "vis1", - "vis1", - localPkg, - "", - null, - "license", - "fa fa-coffee" - ); - List<HeliumPackage> pkgs = new LinkedList<>(); - pkgs.add(pkg); - File bundle = hvf.bundle(pkgs); - assertTrue(bundle.isFile()); - } - - @Test - public void bundleErrorPropagation() throws IOException, TaskRunnerException { - URL res = Resources.getResource("helium/webpack.config.js"); - String resDir = new File(res.getFile()).getParent(); - String localPkg = resDir + "/../../../src/test/resources/helium/vis2"; - - HeliumPackage pkg = new HeliumPackage( - HeliumPackage.Type.VISUALIZATION, - "vis2", - "vis2", - localPkg, - "", - null, - "license", - "fa fa-coffee" - ); - List<HeliumPackage> pkgs = new LinkedList<>(); - pkgs.add(pkg); - File bundle = null; - try { - bundle = hvf.bundle(pkgs); - // should throw exception - assertTrue(false); - } catch (IOException e) { - assertTrue(e.getMessage().contains("error in the package")); - } - assertNull(bundle); - } - - @Test - public void switchVersion() throws IOException, TaskRunnerException { - URL res = Resources.getResource("helium/webpack.config.js"); - String resDir = new File(res.getFile()).getParent(); - - HeliumPackage pkgV1 = new HeliumPackage( - HeliumPackage.Type.VISUALIZATION, - "zeppelin-bubblechart", - "zeppelin-bubblechart", - "[email protected]", - "", - null, - "license", - "icon" - ); - - HeliumPackage pkgV2 = new HeliumPackage( - HeliumPackage.Type.VISUALIZATION, - "zeppelin-bubblechart", - "zeppelin-bubblechart", - "[email protected]", - "", - null, - "license", - "icon" - ); - List<HeliumPackage> pkgsV1 = new LinkedList<>(); - pkgsV1.add(pkgV1); - - List<HeliumPackage> pkgsV2 = new LinkedList<>(); - pkgsV2.add(pkgV2); - - File bundle1 = hvf.bundle(pkgsV1); - File bundle2 = hvf.bundle(pkgsV2); - - assertNotSame(bundle1.lastModified(), bundle2.lastModified()); - } -}
