Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-14714 294eddc42 -> e8c095402
AMBARI-21603: Minor fix to ensure mpack depoloyment. (mradhakrishnan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e8c09540 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e8c09540 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e8c09540 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: e8c09540287dc3b54ccdd25dc0fdd330262c99e4 Parents: 294eddc Author: Madhuvanthi Radhakrishnan <[email protected]> Authored: Fri Jul 28 16:19:38 2017 -0700 Committer: Madhuvanthi Radhakrishnan <[email protected]> Committed: Fri Jul 28 16:19:38 2017 -0700 ---------------------------------------------------------------------- .../ambari/server/mpack/MpackManager.java | 80 +++++++++++++++----- .../apache/ambari/server/stack/StackModule.java | 2 + .../src/test/resources/mpacks-v2/README.txt | 1 + 3 files changed, 63 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e8c09540/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java index a02ea45..a8a227b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java @@ -61,6 +61,8 @@ import com.google.inject.assistedinject.AssistedInject; public class MpackManager { private static final String MPACK_METADATA = "mpack.json"; private static final String MPACK_TAR_LOCATION = "staging"; + private static final String SERVICES_DIRECTORY = "services"; + private static final String PACKLETS_DIRECTORY = "packlets"; private final static Logger LOG = LoggerFactory.getLogger(MpackManager.class); protected Map<Long, Mpack> mpackMap = new ConcurrentHashMap<>(); private File mpacksStaging; @@ -97,13 +99,15 @@ public class MpackManager { if (file.isDirectory()) { String mpackVersion = file.getName(); List resultSet = mpackDAO.findByNameVersion(mpackName, mpackVersion); - MpackEntity mpackEntity = (MpackEntity) resultSet.get(0); - - //Read the mpack.json file into Mpack Object for further use. - String mpackJsonContents = new String((Files.readAllBytes(Paths.get(file + "/" + MPACK_METADATA))), "UTF-8"); - Gson gson = new Gson(); - Mpack existingMpack = gson.fromJson(mpackJsonContents, Mpack.class); - mpackMap.put(mpackEntity.getMpackId(), existingMpack); + if (resultSet.size() > 0) { + MpackEntity mpackEntity = (MpackEntity) resultSet.get(0); + + //Read the mpack.json file into Mpack Object for further use. + String mpackJsonContents = new String((Files.readAllBytes(Paths.get(file + "/" + MPACK_METADATA))), "UTF-8"); + Gson gson = new Gson(); + Mpack existingMpack = gson.fromJson(mpackJsonContents, Mpack.class); + mpackMap.put(mpackEntity.getMpackId(), existingMpack); + } } } } @@ -175,23 +179,19 @@ public class MpackManager { } } - /** - * Mpack is downloaded as a tar.gz file. It is extracted into mpack-v2-staging/{mpack-name}/{mpack-version}/ directory - * - * @param mpack Mpack to process - * @param mpackTarPath Path to mpack tarball - * @param mpackDirectory Mpack directory + /*** + * A generic method to extract tar files. + * @param tarPath * @throws IOException */ - private void extractMpackTar(Mpack mpack, Path mpackTarPath, String mpackDirectory) throws IOException { - - TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(String.valueOf(mpackTarPath)))))); + private void extractTar(Path tarPath, File tempOutputDirectory) throws IOException{ + TarArchiveInputStream tarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(String.valueOf(tarPath)))))); TarArchiveEntry entry = null; File outputFile = null; //Create a loop to read every single entry in TAR file - while ((entry = mpackTarFile.getNextTarEntry()) != null) { - outputFile = new File(mpacksStaging, entry.getName()); + while ((entry = tarFile.getNextTarEntry()) != null) { + outputFile = new File(tempOutputDirectory, entry.getName()); if (entry.isDirectory()) { LOG.debug("Attempting to write output directory" + outputFile.getAbsolutePath()); if (!outputFile.exists()) { @@ -203,21 +203,61 @@ public class MpackManager { } else { LOG.debug("Creating output file %s." + outputFile.getAbsolutePath()); final OutputStream outputFileStream = new FileOutputStream(outputFile); - IOUtils.copy(mpackTarFile, outputFileStream); + IOUtils.copy(tarFile, outputFileStream); outputFileStream.close(); } } - mpackTarFile.close(); + tarFile.close(); + } + + /** + * Mpack is downloaded as a tar.gz file. It is extracted into mpack-v2-staging/{mpack-name}/{mpack-version}/ directory + * + * @param mpack Mpack to process + * @param mpackTarPath Path to mpack tarball + * @param mpackDirectory Mpack directory + * @throws IOException + */ + private void extractMpackTar(Mpack mpack, Path mpackTarPath, String mpackDirectory) throws IOException { + + extractTar(mpackTarPath, mpacksStaging); String mpackTarDirectory = mpackTarPath.toString(); Path extractedMpackDirectory = Files.move (Paths.get(mpacksStaging + File.separator + mpackTarDirectory.substring(mpackTarDirectory.lastIndexOf('/') + 1, mpackTarDirectory.indexOf(".tar")) + File.separator), Paths.get(mpackDirectory), StandardCopyOption.REPLACE_EXISTING); + createServicesDirectory(extractedMpackDirectory); createSymLinks(mpack); } + /*** + * Create a services directory and extract all the services tar file inside it. This readies it for cluster deployment + * @param extractedMpackDirectory + * @throws IOException + */ + private void createServicesDirectory(Path extractedMpackDirectory) throws IOException { + File servicesDir = new File(extractedMpackDirectory.toAbsolutePath() + File.separator + SERVICES_DIRECTORY); + if (!servicesDir.exists()) { + servicesDir.mkdir(); + } + File packletDir = new File(extractedMpackDirectory.toAbsolutePath() + File.separator + PACKLETS_DIRECTORY); + for (final File serviceTar : packletDir.listFiles()) { + String serviceName = serviceTar.getName(); + if (serviceName.contains("tar.gz")) { + extractTar(serviceTar.toPath(), packletDir); + String serviceTarDirectory = serviceTar.toString(); + String serviceNameVersion = serviceTarDirectory.substring(serviceTarDirectory.lastIndexOf('/') + 1, serviceTarDirectory.indexOf(".tar")); + String serviceFolderName = serviceNameVersion.substring(0,serviceNameVersion.indexOf("-packlet")); //Can we assume the names of the tar files? + Path extractedServiceDirectory = Files.move + (Paths.get(packletDir + File.separator + serviceNameVersion), + Paths.get(servicesDir.toPath() + File.separator + serviceFolderName ), StandardCopyOption.REPLACE_EXISTING); + } + } + } + + /** * Reads the mpack.json file within the {mpack-name}.tar.gz file and populates Mpack object. * Extract the mpack-name and mpack-version from mpack.json to create the new mpack directory to hold the mpack files. http://git-wip-us.apache.org/repos/asf/ambari/blob/e8c09540/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index 7b2f87721..0d7ecf0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -356,6 +356,8 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V String parent = serviceInfo.getParent(); if (parent != null) { mergeServiceWithExplicitParent(service, parent, allStacks, commonServices, extensions); + }else { + serviceInfo.setServiceAdvisorType(ServiceInfo.ServiceAdvisorType.PYTHON); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e8c09540/ambari-server/src/test/resources/mpacks-v2/README.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/mpacks-v2/README.txt b/ambari-server/src/test/resources/mpacks-v2/README.txt new file mode 100644 index 0000000..00ca9bd --- /dev/null +++ b/ambari-server/src/test/resources/mpacks-v2/README.txt @@ -0,0 +1 @@ +This is a place holder for unit tests which try to parse mpacks-v2 directory. \ No newline at end of file
