AMBARI-7345 - Views : Exception from ambari-server startup, extract views
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19533465 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19533465 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19533465 Branch: refs/heads/branch-alerts-dev Commit: 19533465d6ccc4c57d00ed8c9af7815f32b2cead Parents: e83ab25 Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Tue Sep 16 17:41:13 2014 -0400 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Tue Sep 16 19:06:18 2014 -0400 ---------------------------------------------------------------------- .../apache/ambari/server/view/ViewRegistry.java | 67 +++++++++++++-- ambari-server/src/main/python/ambari-server.py | 42 +++++++-- .../ambari/server/view/ViewRegistryTest.java | 90 ++++++++++++++++++++ .../view/configuration/ViewConfigTest.java | 1 + 4 files changed, 186 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/19533465/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java b/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java index 8f6774e..5e652a1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java @@ -28,7 +28,6 @@ import org.apache.ambari.server.api.resources.ViewExternalSubResourceDefinition; import org.apache.ambari.server.api.services.ViewExternalSubResourceService; import org.apache.ambari.server.api.services.ViewSubResourceService; import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.controller.ControllerModule; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.orm.dao.MemberDAO; import org.apache.ambari.server.orm.dao.PrivilegeDAO; @@ -101,6 +100,7 @@ public class ViewRegistry { * Constants */ private static final String EXTRACTED_ARCHIVES_DIR = "work"; + private static final String EXTRACT_COMMAND = "extract"; /** * Thread pool @@ -219,15 +219,31 @@ public class ViewRegistry { * Registry main method. * * @param args the command line arguments - * - * @throws Exception if the registry command can not be completed */ - public static void main(String[] args) throws Exception { + public static void main(String[] args) { + + if (args.length >= 2) { + String archivePath = args[1]; + + try { + Injector injector = Guice.createInjector(); - Injector injector = Guice.createInjector(new ControllerModule()); - initInstance(injector.getInstance(ViewRegistry.class)); + ViewExtractor extractor = injector.getInstance(ViewExtractor.class); + ViewArchiveUtility archiveUtility = injector.getInstance(ViewArchiveUtility.class); + Configuration configuration = injector.getInstance(Configuration.class); - singleton.readViewArchives(true, false); + if (args[0].equals(EXTRACT_COMMAND)) { + if (extractViewArchive(archivePath, extractor, archiveUtility, configuration, true)) { + System.exit(0); + } + } + } catch (Exception e) { + String msg = "Caught exception extracting view archive " + archivePath + "."; + LOG.error(msg, e); + System.exit(2); + } + } + System.exit(1); } /** @@ -1229,6 +1245,43 @@ public class ViewRegistry { } } + // extract the view archive for the given path. + protected static boolean extractViewArchive(String archivePath, + ViewExtractor extractor, + ViewArchiveUtility archiveUtility, + Configuration configuration, + boolean systemOnly ) throws Exception { + + File viewDir = configuration.getViewsDir(); + + String extractedArchivesPath = viewDir.getAbsolutePath() + + File.separator + EXTRACTED_ARCHIVES_DIR; + + if (extractor.ensureExtractedArchiveDirectory(extractedArchivesPath)) { + + File archiveFile = archiveUtility.getFile(archivePath); + + ViewConfig viewConfig = archiveUtility.getViewConfigFromArchive(archiveFile); + + String commonName = viewConfig.getName(); + String version = viewConfig.getVersion(); + String viewName = ViewEntity.getViewName(commonName, version); + + String extractedArchiveDirPath = extractedArchivesPath + File.separator + viewName; + File extractedArchiveDirFile = archiveUtility.getFile(extractedArchiveDirPath); + + if (!extractedArchiveDirFile.exists()) { + ViewEntity viewDefinition = new ViewEntity(viewConfig, configuration, extractedArchiveDirPath); + + if (!systemOnly || viewDefinition.isSystem()) { + extractor.extractViewArchive(viewDefinition, archiveFile, extractedArchiveDirFile); + return true; + } + } + } + return false; + } + // set the status of the given view. private void setViewStatus(ViewEntity viewDefinition, ViewEntity.ViewStatus status, String statusDetail) { viewDefinition.setStatus(status); http://git-wip-us.apache.org/repos/asf/ambari/blob/19533465/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index 319d1d4..1ad906f 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -189,7 +189,7 @@ STACK_UPGRADE_HELPER_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" +\ VIEW_EXTRACT_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" +\ os.pathsep + "{2} " +\ - "org.apache.ambari.server.view.ViewRegistry " +\ + "org.apache.ambari.server.view.ViewRegistry extract {3} " +\ "> " + SERVER_OUT_FILE + " 2>&1" @@ -283,6 +283,9 @@ JDBC_HOSTNAME_PROPERTY = "server.jdbc.hostname" JDBC_PORT_PROPERTY = "server.jdbc.port" JDBC_POSTGRES_SCHEMA_PROPERTY = "server.jdbc.postgres.schema" # Only for postgres, defaults to same value as DB name +VIEWS_DIR_PROPERTY = "views.dir" +DEFAULT_VIEWS_DIR = "/var/lib/ambari-server/resources/views" + JDBC_USER_NAME_PROPERTY = "server.jdbc.user.name" JDBC_PASSWORD_PROPERTY = "server.jdbc.user.passwd" JDBC_PASSWORD_FILENAME = "password.dat" @@ -1225,6 +1228,7 @@ def prompt_db_properties(args): # extract the system views def extract_views(): + jdk_path = find_jdk() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " @@ -1232,12 +1236,36 @@ def extract_views(): "JDK manually to " + JDK_INSTALL_DIR) return 1 - command = VIEW_EXTRACT_CMD.format(jdk_path, get_conf_dir(), - get_ambari_classpath()) - (retcode, stdout, stderr) = run_os_command(command) - print_info_msg("Return code from view extraction: " + - str(retcode)) - return retcode + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + + if not VIEWS_DIR_PROPERTY in properties.keys(): + vdir = DEFAULT_VIEWS_DIR + else: + vdir = properties.get_property(VIEWS_DIR_PROPERTY) + + files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))] + for f in files: + + command = VIEW_EXTRACT_CMD.format(jdk_path, get_conf_dir(), + get_ambari_classpath(), os.path.join(vdir,f)) + + retcode, stdout, stderr = run_os_command(command) + if retcode == 0: + sys.stdout.write(f + "\n") + elif retcode == 2: + sys.stdout.write("Error extracting " + f + "\n") + else: + sys.stdout.write(".") + sys.stdout.flush() + + print_info_msg("Return code from extraction of view archive " + f + ": " + + str(retcode)) + + sys.stdout.write("\n") + return 0 # Store set of properties for remote database connection def store_remote_properties(args): http://git-wip-us.apache.org/repos/asf/ambari/blob/19533465/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java index eba0c25..56ee434 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java @@ -18,8 +18,10 @@ package org.apache.ambari.server.view; +import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; @@ -81,6 +83,7 @@ import org.apache.ambari.server.view.events.EventImplTest; import org.apache.ambari.view.ViewDefinition; import org.apache.ambari.view.events.Event; import org.apache.ambari.view.events.Listener; +import org.easymock.Capture; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -926,6 +929,93 @@ public class ViewRegistryTest { verify(securityHelper, viewEntity, configuration); } + @Test + public void testExtractViewArchive() throws Exception { + + File viewDir = createNiceMock(File.class); + File extractedArchiveDir = createNiceMock(File.class); + File viewArchive = createNiceMock(File.class); + File archiveDir = createNiceMock(File.class); + File entryFile = createNiceMock(File.class); + File classesDir = createNiceMock(File.class); + File libDir = createNiceMock(File.class); + File fileEntry = createNiceMock(File.class); + + JarFile viewJarFile = createNiceMock(JarFile.class); + Enumeration<JarEntry> enumeration = createMock(Enumeration.class); + JarEntry jarEntry = createNiceMock(JarEntry.class); + InputStream is = createMock(InputStream.class); + FileOutputStream fos = createMock(FileOutputStream.class); + ViewExtractor viewExtractor = createMock(ViewExtractor.class); + + ResourceTypeEntity resourceTypeEntity = new ResourceTypeEntity(); + resourceTypeEntity.setId(10); + resourceTypeEntity.setName("MY_VIEW{1.0.0}"); + + ViewEntity viewDefinition = ViewEntityTest.getViewEntity(); + viewDefinition.setResourceType(resourceTypeEntity); + + Set<ViewInstanceEntity> viewInstanceEntities = ViewInstanceEntityTest.getViewInstanceEntities(viewDefinition); + viewDefinition.setInstances(viewInstanceEntities); + + Map<File, ViewConfig> viewConfigs = + Collections.singletonMap(viewArchive, viewDefinition.getConfiguration()); + + long resourceId = 99L; + for (ViewInstanceEntity viewInstanceEntity : viewInstanceEntities) { + ResourceEntity resourceEntity = new ResourceEntity(); + resourceEntity.setId(resourceId); + resourceEntity.setResourceType(resourceTypeEntity); + viewInstanceEntity.setResource(resourceEntity); + } + + Map<String, File> files = new HashMap<String, File>(); + + files.put("/var/lib/ambari-server/resources/views/my_view-1.0.0.jar", viewArchive); + files.put("/var/lib/ambari-server/resources/views/work", extractedArchiveDir); + files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}", archiveDir); + files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}/view.xml", entryFile); + files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}/WEB-INF/classes", classesDir); + files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}/WEB-INF/lib", libDir); + + Map<File, FileOutputStream> outputStreams = new HashMap<File, FileOutputStream>(); + outputStreams.put(entryFile, fos); + + Map<File, JarFile> jarFiles = new HashMap<File, JarFile>(); + jarFiles.put(viewArchive, viewJarFile); + + // set expectations + expect(configuration.getViewsDir()).andReturn(viewDir); + expect(viewDir.getAbsolutePath()).andReturn("/var/lib/ambari-server/resources/views"); + + expect(configuration.getViewExtractionThreadPoolCoreSize()).andReturn(2).anyTimes(); + expect(configuration.getViewExtractionThreadPoolMaxSize()).andReturn(3).anyTimes(); + expect(configuration.getViewExtractionThreadPoolTimeout()).andReturn(10000L).anyTimes(); + + expect(viewArchive.getAbsolutePath()).andReturn("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}").anyTimes(); + + expect(archiveDir.exists()).andReturn(false); + expect(archiveDir.getAbsolutePath()).andReturn( + "/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}").anyTimes(); + + Capture<ViewEntity> viewEntityCapture = new Capture<ViewEntity>(); + expect(viewExtractor.ensureExtractedArchiveDirectory("/var/lib/ambari-server/resources/views/work")).andReturn(true); + expect(viewExtractor.extractViewArchive(capture(viewEntityCapture), eq(viewArchive), eq(archiveDir))).andReturn(null); + + // replay mocks + replay(configuration, viewDir, extractedArchiveDir, viewArchive, archiveDir, entryFile, classesDir, + libDir, fileEntry, viewJarFile, enumeration, jarEntry, is, fos, viewExtractor, resourceDAO, viewDAO, viewInstanceDAO); + + TestViewArchiveUtility archiveUtility = new TestViewArchiveUtility(viewConfigs, files, outputStreams, jarFiles); + + Assert.assertTrue(ViewRegistry.extractViewArchive("/var/lib/ambari-server/resources/views/my_view-1.0.0.jar", + viewExtractor, archiveUtility, configuration, true)); + + // verify mocks + verify(configuration, viewDir, extractedArchiveDir, viewArchive, archiveDir, entryFile, classesDir, + libDir, fileEntry, viewJarFile, enumeration, jarEntry, is, fos, viewExtractor, resourceDAO, viewDAO, viewInstanceDAO); + } + public static class TestViewArchiveUtility extends ViewArchiveUtility { private final Map<File, ViewConfig> viewConfigs; private final Map<String, File> files; http://git-wip-us.apache.org/repos/asf/ambari/blob/19533465/ambari-server/src/test/java/org/apache/ambari/server/view/configuration/ViewConfigTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/view/configuration/ViewConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/view/configuration/ViewConfigTest.java index 0e0ace8..f6ec403 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/view/configuration/ViewConfigTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/view/configuration/ViewConfigTest.java @@ -47,6 +47,7 @@ public class ViewConfigTest { " <label>My View!</label>\n" + " <description>Description</description>" + " <version>1.0.0</version>\n" + + " <system>true</system>\n" + " <icon64>/this/is/the/icon/url/icon64.png</icon64>\n" + " <icon>/this/is/the/icon/url/icon.png</icon>\n" + " <masker-class>org.apache.ambari.server.view.DefaultMasker</masker-class>" +
