Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-14714 4450fd7ef -> 5465aaa56
AMBARI-21529: Integrate Registries and Mpack for POST /mpacks using registry (mradhakrishnan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5465aaa5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5465aaa5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5465aaa5 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: 5465aaa56d7926206ba576911b2f5246231a79ca Parents: 4450fd7 Author: Madhuvanthi Radhakrishnan <mradhakrish...@hortonworks.com> Authored: Fri Jul 21 14:40:52 2017 -0700 Committer: Madhuvanthi Radhakrishnan <mradhakrish...@hortonworks.com> Committed: Fri Jul 21 14:40:52 2017 -0700 ---------------------------------------------------------------------- .../internal/MpackResourceProvider.java | 33 +++++++-- .../ambari/server/mpack/MpackManager.java | 78 +++++++++++++------- 2 files changed, 81 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5465aaa5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java index 1fd0343..e3afcde 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.controller.internal; import java.io.IOException; +import java.net.ConnectException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -28,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.StaticallyInject; import org.apache.ambari.server.api.services.parsers.BodyParseException; import org.apache.ambari.server.controller.AmbariManagementController; @@ -48,6 +50,9 @@ import org.apache.ambari.server.orm.dao.MpackDAO; import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.orm.entities.MpackEntity; import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.registry.Registry; +import org.apache.ambari.server.registry.RegistryMpack; +import org.apache.ambari.server.registry.RegistryMpackVersion; import org.apache.ambari.server.state.Packlet; import com.google.inject.Inject; @@ -121,9 +126,11 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider { public RequestStatus createResourcesAuthorized(final Request request) throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException, IllegalArgumentException { + + MpackRequest mpackRequest = null; Set<Resource> associatedResources = new HashSet<>(); try { - MpackRequest mpackRequest = getRequest(request); + mpackRequest = getRequest(request); if (mpackRequest == null) throw new BodyParseException("Please provide " + MPACK_NAME + " ," + MPACK_VERSION + " ," + MPACK_URI); MpackResponse response = getManagementController().registerMpack(mpackRequest); @@ -140,6 +147,8 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider { return getRequestStatus(null, associatedResources); } } catch (IOException e) { + if (e instanceof ConnectException) + throw new SystemException("The Mpack Uri : " + mpackRequest.getMpackUri() + " is not valid. Please try again"); e.printStackTrace(); } catch (BodyParseException e1) { e1.printStackTrace(); @@ -147,18 +156,19 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider { return null; } - public MpackRequest getRequest(Request request) { + public MpackRequest getRequest(Request request) throws AmbariException { MpackRequest mpackRequest = new MpackRequest(); Set<Map<String, Object>> properties = request.getProperties(); for (Map propertyMap : properties) { //Mpack Download url is either given in the request body or is fetched using the registry id if (!propertyMap.containsKey(MPACK_URI) && !propertyMap.containsKey(REGISTRY_ID)) return null; - //Fetch Mpack Download Url using the given registry id + //Fetch Mpack Download Url using the given registry id else if (!propertyMap.containsKey(MPACK_URI)) { - mpackRequest.setRegistryId((Long) propertyMap.get(REGISTRY_ID)); + mpackRequest.setRegistryId(Long.valueOf ((String) propertyMap.get(REGISTRY_ID))); mpackRequest.setMpackName((String) propertyMap.get(MPACK_NAME)); mpackRequest.setMpackVersion((String) propertyMap.get(MPACK_VERSION)); + mpackRequest.setMpackUri(getMpackUri(mpackRequest)); } //Directly download the mpack using the given url else @@ -168,6 +178,19 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider { } + /*** + * Uses the Registries functions to get the mpack uri. + * @param mpackRequest + * @return + * @throws AmbariException + */ + private String getMpackUri(MpackRequest mpackRequest) throws AmbariException { + Registry registry = getManagementController().getRegistry(mpackRequest.getRegistryId()); + RegistryMpack registryMpack = registry.getRegistryMpack(mpackRequest.getMpackName()); + RegistryMpackVersion registryMpackVersion = registryMpack.getMpackVersion(mpackRequest.getMpackVersion()); + return registryMpackVersion.getMpackUrl(); + } + @Override public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, @@ -216,7 +239,7 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider { if (propertyMap.containsKey(MPACK_ID)) { Object objMpackId = propertyMap.get(MPACK_ID); - if(objMpackId != null) + if (objMpackId != null) mpackId = Long.valueOf((String) objMpackId); MpackEntity entity = mpackDAO.findById(mpackId); http://git-wip-us.apache.org/repos/asf/ambari/blob/5465aaa5/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java index 2bae72d..0c05292 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java @@ -85,10 +85,12 @@ public class MpackManager { * @throws IOException */ private void parseMpackDirectories() { + try { for (final File dirEntry : mpacksStaging.listFiles()) { if (dirEntry.isDirectory()) { String mpackName = dirEntry.getName(); + if (!mpackName.equals(MPACK_TAR_LOCATION)) { for (final File file : dirEntry.listFiles()) { if (file.isDirectory()) { @@ -138,7 +140,6 @@ public class MpackManager { mpackVersion = mpackRequest.getMpackVersion(); mpack.setRegistryId(mpackRequest.getRegistryId()); - //Todo : Madhu implement GET /registries/{registryId}/mpacks mpackTarPath = downloadMpack(mpackRequest.getMpackUri()); if (createMpackDirectory(mpack, mpackTarPath)) { @@ -156,10 +157,10 @@ public class MpackManager { if (createMpackDirectory(mpack, mpackTarPath)) { mpackDirectory = mpacksStaging + File.separator + mpack.getName() + File.separator + mpack.getVersion(); - mpack.setMpackUri(mpackRequest.getMpackUri()); } } extractMpackTar(mpack, mpackTarPath, mpackDirectory); + mpack.setMpackUri(mpackRequest.getMpackUri()); mpackId = populateDB(mpack); if (mpackId != null) { @@ -176,16 +177,17 @@ public class MpackManager { /** * Mpack is downloaded as a tar.gz file. It is extracted into mpack-v2-staging/{mpack-name}/{mpack-version}/ directory * - * @param mpack Mpack to process - * @param mpackTarPath Path to mpack tarball + * @param mpack Mpack to process + * @param mpackTarPath Path to mpack tarball * @param mpackDirectory Mpack directory * @throws IOException */ private void extractMpackTar(Mpack mpack, Path mpackTarPath, String mpackDirectory) throws IOException { + TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(String.valueOf(mpackTarPath)))))); - // To read individual TAR file TarArchiveEntry entry = null; File outputFile = null; + //Create a loop to read every single entry in TAR file while ((entry = mpackTarFile.getNextTarEntry()) != null) { outputFile = new File(mpacksStaging, entry.getName()); @@ -204,7 +206,9 @@ public class MpackManager { outputFileStream.close(); } } + mpackTarFile.close(); + String mpackTarDirectory = mpackTarPath.toString(); Path extractedMpackDirectory = Files.move (Paths.get(mpacksStaging + File.separator + mpackTarDirectory.substring(mpackTarDirectory.lastIndexOf('/') + 1, mpackTarDirectory.indexOf(".tar")) + File.separator), @@ -217,30 +221,30 @@ public class MpackManager { * Reads the mpack.json file within the {mpack-name}.tar.gz file and populates Mpack object. * Extract the mpack-name and mpack-version from mpack.json to create the new mpack directory to hold the mpack files. * - * @param mpack Mpack to process + * @param mpack Mpack to process * @param mpackTarPath Path to mpack tarball * @return boolean * @throws IOException */ - private Boolean createMpackDirectory(Mpack mpack, Path mpackTarPath) throws IOException { + private Boolean createMpackDirectory(Mpack mpack, Path mpackTarPath) throws IOException, ResourceAlreadyExistsException { + TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(mpackTarPath.toString()))))); - // To read individual TAR file TarArchiveEntry entry = null; String individualFiles; int offset; + // Create a loop to read every single entry in TAR file while ((entry = mpackTarFile.getNextTarEntry()) != null) { // Get the name of the file individualFiles = entry.getName(); String[] dirFile = individualFiles.split(File.separator); + //Search for mpack.json String fileName = dirFile[dirFile.length - 1]; if (fileName.contains("mpack") && fileName.contains(".json")) { - // Get Size of the file and create a byte array for the size byte[] content = new byte[(int) entry.getSize()]; offset = 0; LOG.debug("Size of the File is: " + entry.getSize()); - // Read file from the archive into byte array mpackTarFile.read(content, offset, content.length - offset); //Read the mpack.json file into Mpack Object for further use. @@ -251,11 +255,20 @@ public class MpackManager { mpackTarFile.close(); - File mpackDirectory = new File(mpacksStaging + File.separator + mpack.getName()); - if (!mpackDirectory.exists()) - return mpackDirectory.mkdir(); - else - return true; + //Check if the mpack already exists + List<MpackEntity> mpackEntities = mpackDAO.findByNameVersion(mpack.getName(), mpack.getVersion()); + if (mpackEntities.size() == 0) { + File mpackDirectory = new File(mpacksStaging + File.separator + mpack.getName()); + + if (!mpackDirectory.exists()) { + return mpackDirectory.mkdir(); + } else { + return true; + } + } else { + String message = "Mpack :" + mpack.getName() + " version: " + mpack.getVersion() + " already exists in server"; + throw new ResourceAlreadyExistsException(message); + } } } @@ -270,18 +283,21 @@ public class MpackManager { * @throws IOException */ private void createSymLinks(Mpack mpack) throws IOException { + String stackId = mpack.getStackId(); String[] stackMetaData = stackId.split("-"); String stackName = stackMetaData[0]; String stackVersion = stackMetaData[1]; File stack = new File(stackRoot + "/" + stackName); - if(!stack.exists()) { - stack.mkdir(); - } Path stackPath = Paths.get(stackRoot + "/" + stackName + "/" + stackVersion); Path mpackPath = Paths.get(mpacksStaging + "/" + mpack.getName() + "/" + mpack.getVersion()); - if(Files.isSymbolicLink(stackPath)) + + if (!stack.exists()) { + stack.mkdir(); + } + if (Files.isSymbolicLink(stackPath)) { Files.delete(stackPath); + } Files.createSymbolicLink(stackPath, mpackPath); } @@ -292,15 +308,17 @@ public class MpackManager { * @return */ public Path downloadMpack(String mpackURI) throws IOException { + URL url = new URL(mpackURI); - String fileName = mpackURI.substring(mpackURI.lastIndexOf('/') + 1, mpackURI.length()); + String mpackTarFile = mpackURI.substring(mpackURI.lastIndexOf('/') + 1, mpackURI.length()); File stagingDir = new File(mpacksStaging.toString() + File.separator + MPACK_TAR_LOCATION); + Path targetPath = new File(stagingDir.getPath() + File.separator + mpackTarFile).toPath(); + if (!stagingDir.exists()) { stagingDir.mkdir(); } - Path targetPath = new File(stagingDir.getPath() + File.separator + fileName).toPath(); - Files.copy(url.openStream(), targetPath, StandardCopyOption.REPLACE_EXISTING); + Files.copy(url.openStream(), targetPath, StandardCopyOption.REPLACE_EXISTING); return targetPath; } @@ -315,11 +333,14 @@ public class MpackManager { * @return boolean */ protected boolean validateMpackInfo(String expectedMpackName, String expectedMpackVersion, String actualMpackName, String actualMpackVersion) { - if (expectedMpackName.equalsIgnoreCase(actualMpackName) && expectedMpackVersion.equalsIgnoreCase(actualMpackVersion)) + + if (expectedMpackName.equalsIgnoreCase(actualMpackName) && expectedMpackVersion.equalsIgnoreCase(actualMpackVersion)) { return true; - else + } + else { LOG.info("Incorrect information : Mismatch in - (" + expectedMpackName + "," + actualMpackName + ") or (" + expectedMpackVersion + "," + actualMpackVersion + ")"); - return false; + return false; + } } /** @@ -330,15 +351,19 @@ public class MpackManager { * @throws IOException */ protected Long populateDB(Mpack mpack) throws IOException { + String mpackName = mpack.getName(); String mpackVersion = mpack.getVersion(); List resultSet = mpackDAO.findByNameVersion(mpackName, mpackVersion); + if (resultSet.size() == 0) { LOG.info("Adding mpack {}-{} to the database", mpackName, mpackVersion); + MpackEntity mpackEntity = new MpackEntity(); mpackEntity.setMpackName(mpackName); mpackEntity.setMpackVersion(mpackVersion); mpackEntity.setMpackUri(mpack.getMpackUri()); + mpackEntity.setRegistryId(mpack.getRegistryId()); Long mpackId = mpackDAO.create(mpackEntity); return mpackId; @@ -363,12 +388,14 @@ public class MpackManager { if (stackEntity == null) { LOG.info("Adding stack {}-{} to the database", stackName, stackVersion); stackEntity = new StackEntity(); + stackEntity.setStackName(stackName); stackEntity.setStackVersion(stackVersion); stackEntity.setCurrentMpackId(mpack.getMpackId()); stackDAO.create(stackEntity); } else { LOG.info("Updating stack {}-{} to the database", stackName, stackVersion); + stackEntity.setCurrentMpackId(mpack.getMpackId()); stackDAO.merge(stackEntity); } @@ -380,6 +407,7 @@ public class MpackManager { * @return ArrayList */ public ArrayList<Packlet> getPacklets(Long mpackId) { + Mpack mpack = mpackMap.get(mpackId); if (mpack.getPacklets() != null) return mpack.getPacklets();