Repository: ambari Updated Branches: refs/heads/trunk 56526a84f -> 9cdd3e8a1
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cdd3e8a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java ---------------------------------------------------------------------- diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java index 1eae756..d1d1bb0 100644 --- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java +++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java @@ -19,19 +19,13 @@ package org.apache.ambari.view.utils.hdfs; import org.apache.ambari.view.ViewContext; -import org.apache.ambari.view.utils.ambari.AmbariApi; -import org.apache.ambari.view.utils.ambari.NoClusterAssociatedException; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.azure.NativeAzureFileSystem; -import org.apache.hadoop.fs.azure.Wasb; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem; -import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -47,27 +41,27 @@ public class ConfigurationBuilder { public static final String HDFS_SITE = "hdfs-site"; public static final String DEFAULT_FS_INSTANCE_PROPERTY = "webhdfs.url"; - public static final String DEFAULT_FS_CLUSTER_PROPERTY = "fs.defaultFS"; + public static final String DEFAULT_FS_CLUSTER_PROPERTY = "fs.defaultFS"; public static final String NAMESERVICES_INSTANCE_PROPERTY = "webhdfs.nameservices"; - public static final String NAMESERVICES_CLUSTER_PROPERTY = "dfs.nameservices"; + public static final String NAMESERVICES_CLUSTER_PROPERTY = "dfs.nameservices"; public static final String HA_NAMENODES_INSTANCE_PROPERTY = "webhdfs.ha.namenodes.list"; - public static final String HA_NAMENODES_CLUSTER_PROPERTY = "dfs.ha.namenodes.%s"; + public static final String HA_NAMENODES_CLUSTER_PROPERTY = "dfs.ha.namenodes.%s"; public static final String NAMENODE_RPC_NN1_INSTANCE_PROPERTY = "webhdfs.ha.namenode.rpc-address.nn1"; public static final String NAMENODE_RPC_NN2_INSTANCE_PROPERTY = "webhdfs.ha.namenode.rpc-address.nn2"; - public static final String NAMENODE_RPC_NN_CLUSTER_PROPERTY = "dfs.namenode.rpc-address.%s.%s"; + public static final String NAMENODE_RPC_NN_CLUSTER_PROPERTY = "dfs.namenode.rpc-address.%s.%s"; public static final String NAMENODE_HTTP_NN1_INSTANCE_PROPERTY = "webhdfs.ha.namenode.http-address.nn1"; public static final String NAMENODE_HTTP_NN2_INSTANCE_PROPERTY = "webhdfs.ha.namenode.http-address.nn2"; - public static final String NAMENODE_HTTP_NN_CLUSTER_PROPERTY = "dfs.namenode.http-address.%s.%s"; + public static final String NAMENODE_HTTP_NN_CLUSTER_PROPERTY = "dfs.namenode.http-address.%s.%s"; public static final String NAMENODE_HTTPS_NN1_INSTANCE_PROPERTY = "webhdfs.ha.namenode.https-address.nn1"; public static final String NAMENODE_HTTPS_NN2_INSTANCE_PROPERTY = "webhdfs.ha.namenode.https-address.nn2"; - public static final String NAMENODE_HTTPS_NN_CLUSTER_PROPERTY = "dfs.namenode.https-address.%s.%s"; + public static final String NAMENODE_HTTPS_NN_CLUSTER_PROPERTY = "dfs.namenode.https-address.%s.%s"; public static final String FAILOVER_PROXY_PROVIDER_INSTANCE_PROPERTY = "webhdfs.client.failover.proxy.provider"; - public static final String FAILOVER_PROXY_PROVIDER_CLUSTER_PROPERTY = "dfs.client.failover.proxy.provider.%s"; + public static final String FAILOVER_PROXY_PROVIDER_CLUSTER_PROPERTY = "dfs.client.failover.proxy.provider.%s"; public static final String UMASK_CLUSTER_PROPERTY = "fs.permissions.umask-mode"; public static final String UMASK_INSTANCE_PROPERTY = "hdfs.umask-mode"; @@ -85,6 +79,8 @@ public class ConfigurationBuilder { private AuthConfigurationBuilder authParamsBuilder; private Map<String, String> authParams; private URI defaultFsUri; + private Map<String, String> customProperties; + /** * Constructor of ConfigurationBuilder based on ViewContext * @param context ViewContext @@ -94,6 +90,17 @@ public class ConfigurationBuilder { this.authParamsBuilder = new AuthConfigurationBuilder(context); } + /** + * takes context and any extra custom properties that needs to be included into config + * @param context + * @param customProperties + */ + public ConfigurationBuilder(ViewContext context, Map<String, String> customProperties) { + this.context = context; + this.authParamsBuilder = new AuthConfigurationBuilder(context); + this.customProperties = customProperties; + } + private void parseProperties() throws HdfsApiException { String defaultFS = getDefaultFS(context); @@ -113,7 +120,7 @@ public class ConfigurationBuilder { } catch (URISyntaxException e) { throw new HdfsApiException("HDFS060 Invalid " + DEFAULT_FS_INSTANCE_PROPERTY + - "='" + defaultFS + "' URI", e); + "='" + defaultFS + "' URI", e); } conf.set("fs.defaultFS", defaultFS); @@ -142,30 +149,30 @@ public class ConfigurationBuilder { defaultFS = addProtocolIfMissing(defaultFS); - if(context.getCluster() != null){ + if (context.getCluster() != null) { try { URI fsUri = new URI(defaultFS); String protocol = fsUri.getScheme(); String hostWithPort = defaultFS.substring(protocol.length() + 3); - Boolean webHdfsEnabled = Boolean.valueOf(getProperty(HDFS_SITE,DFS_WEBHDFS_ENABLED)); - Boolean isHttps = DFS_HTTP_POLICY_HTTPS_ONLY.equals(getProperty(HDFS_SITE,DFS_HTTP_POLICY)); + Boolean webHdfsEnabled = Boolean.valueOf(getProperty(HDFS_SITE, DFS_WEBHDFS_ENABLED)); + Boolean isHttps = DFS_HTTP_POLICY_HTTPS_ONLY.equals(getProperty(HDFS_SITE, DFS_HTTP_POLICY)); boolean isHA = isHAEnabled(defaultFS); - if(webHdfsEnabled && isHttps){ + if (webHdfsEnabled && isHttps && "hdfs".equals(protocol)) { protocol = "swebhdfs"; - String httpAddr = getProperty(HDFS_SITE,DFS_NAMENODE_HTTPS_ADDERSS); - if(!isHA && httpAddr != null) hostWithPort = httpAddr ; - }else if(webHdfsEnabled){ + String httpAddr = getProperty(HDFS_SITE, DFS_NAMENODE_HTTPS_ADDERSS); + if (!isHA && httpAddr != null) hostWithPort = httpAddr; + } else if (webHdfsEnabled && "hdfs".equals(protocol)) { protocol = "webhdfs"; - String httpsAddr = getProperty(HDFS_SITE,DFS_NAMENODE_HTTP_ADDERSS); - if(!isHA) hostWithPort = httpsAddr; + String httpsAddr = getProperty(HDFS_SITE, DFS_NAMENODE_HTTP_ADDERSS); + if (!isHA) hostWithPort = httpsAddr; } - return protocol + "://" +hostWithPort; + return protocol + "://" + hostWithPort; } catch (URISyntaxException e) { - throw new HdfsApiException("Invalid URI format."+e.getMessage(),e); + throw new HdfsApiException("Invalid URI format." + e.getMessage(), e); } } return defaultFS; @@ -174,7 +181,7 @@ public class ConfigurationBuilder { private String getProperty(String type, String key, String instanceProperty) { String value; - if(context.getCluster() != null) { + if (context.getCluster() != null) { value = context.getCluster().getConfigurationValue(type, key); } else { value = context.getProperties().get(instanceProperty); @@ -182,9 +189,9 @@ public class ConfigurationBuilder { return value; } - private String getProperty(String type,String key){ - if(context.getCluster() != null){ - return context.getCluster().getConfigurationValue(type,key); + private String getProperty(String type, String key) { + if (context.getCluster() != null) { + return context.getCluster().getConfigurationValue(type, key); } return null; } @@ -192,7 +199,7 @@ public class ConfigurationBuilder { private void copyPropertyIfExists(String type, String key) { String value; - if(context.getCluster() != null) { + if (context.getCluster() != null) { value = context.getCluster().getConfigurationValue(type, key); if (value != null) { conf.set(key, value); @@ -205,13 +212,35 @@ public class ConfigurationBuilder { } } + private void copyPropertiesBySite(String type) { + if (context.getCluster() != null) { + Map<String, String> configs = context.getCluster().getConfigByType(type); + LOG.debug("configs from core-site : {}", configs); + copyProperties(configs); + } else { + LOG.error("Cannot find cluster."); + } + } + + private void copyProperties(Map<String, String> configs) { + if (null != configs) { + for(Map.Entry<String, String> entry : configs.entrySet()){ + String key = entry.getKey(); + String value = entry.getValue(); + conf.set(key, value); + } + } else { + LOG.error("configs were null."); + } + } + private void copyHAProperties(String defaultFS) throws URISyntaxException, HdfsApiException { URI uri = new URI(defaultFS); String nameservice = uri.getHost(); copyClusterProperty(NAMESERVICES_CLUSTER_PROPERTY, NAMESERVICES_INSTANCE_PROPERTY); String namenodeIDs = copyClusterProperty(String.format(HA_NAMENODES_CLUSTER_PROPERTY, nameservice), - HA_NAMENODES_INSTANCE_PROPERTY); + HA_NAMENODES_INSTANCE_PROPERTY); String[] namenodes = namenodeIDs.split(","); if (namenodes.length != 2) { @@ -219,22 +248,22 @@ public class ConfigurationBuilder { } //NN1 copyClusterProperty(String.format(NAMENODE_RPC_NN_CLUSTER_PROPERTY, nameservice, namenodes[0]), - NAMENODE_RPC_NN1_INSTANCE_PROPERTY); + NAMENODE_RPC_NN1_INSTANCE_PROPERTY); copyClusterProperty(String.format(NAMENODE_HTTP_NN_CLUSTER_PROPERTY, nameservice, namenodes[0]), - NAMENODE_HTTP_NN1_INSTANCE_PROPERTY); + NAMENODE_HTTP_NN1_INSTANCE_PROPERTY); copyClusterProperty(String.format(NAMENODE_HTTPS_NN_CLUSTER_PROPERTY, nameservice, namenodes[0]), NAMENODE_HTTPS_NN1_INSTANCE_PROPERTY); //NN2 copyClusterProperty(String.format(NAMENODE_RPC_NN_CLUSTER_PROPERTY, nameservice, namenodes[1]), - NAMENODE_RPC_NN2_INSTANCE_PROPERTY); + NAMENODE_RPC_NN2_INSTANCE_PROPERTY); copyClusterProperty(String.format(NAMENODE_HTTP_NN_CLUSTER_PROPERTY, nameservice, namenodes[1]), - NAMENODE_HTTP_NN2_INSTANCE_PROPERTY); + NAMENODE_HTTP_NN2_INSTANCE_PROPERTY); copyClusterProperty(String.format(NAMENODE_HTTPS_NN_CLUSTER_PROPERTY, nameservice, namenodes[1]), NAMENODE_HTTPS_NN2_INSTANCE_PROPERTY); copyClusterProperty(String.format(FAILOVER_PROXY_PROVIDER_CLUSTER_PROPERTY, nameservice), - FAILOVER_PROXY_PROVIDER_INSTANCE_PROPERTY); + FAILOVER_PROXY_PROVIDER_INSTANCE_PROPERTY); } private String copyClusterProperty(String propertyName, String instancePropertyName) { @@ -250,7 +279,7 @@ public class ConfigurationBuilder { URI uri = new URI(defaultFS); String nameservice = uri.getHost(); String namenodeIDs = getProperty(HDFS_SITE, String.format(HA_NAMENODES_CLUSTER_PROPERTY, nameservice), - HA_NAMENODES_INSTANCE_PROPERTY); + HA_NAMENODES_INSTANCE_PROPERTY); return namenodeIDs != null; } @@ -294,49 +323,35 @@ public class ConfigurationBuilder { public Configuration buildConfig() throws HdfsApiException { parseProperties(); setAuthParams(buildAuthenticationConfig()); + copyPropertiesBySite(CORE_SITE); + copyPropertiesBySite(HDFS_SITE); String umask = context.getProperties().get(UMASK_INSTANCE_PROPERTY); - if(umask != null && !umask.isEmpty()) conf.set(UMASK_CLUSTER_PROPERTY,umask); + if (umask != null && !umask.isEmpty()) conf.set(UMASK_CLUSTER_PROPERTY, umask); - conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName()); - conf.set("fs.webhdfs.impl", WebHdfsFileSystem.class.getName()); - conf.set("fs.file.impl", LocalFileSystem.class.getName()); - conf.set("fs.swebhdfs.impl", SWebHdfsFileSystem.class.getName()); - - configureWASB(); - configureADL(); + if(null != this.customProperties){ + copyProperties(this.customProperties); + } + + if(LOG.isDebugEnabled()){ + LOG.debug("final conf : {}", printConf()); + } return conf; } - /** - * Fill Azure Blob Storage properties if wasb:// scheme configured - */ - public void configureWASB() { - LOG.debug("defaultFsUri.getScheme() == " + defaultFsUri.getScheme()); - if (defaultFsUri.getScheme().equals("wasb")) { - conf.set("fs.AbstractFileSystem.wasb.impl", Wasb.class.getName()); - conf.set("fs.wasb.impl", NativeAzureFileSystem.class.getName()); - - String account = defaultFsUri.getHost(); - LOG.debug("WASB account == " + account); - copyPropertyIfExists(CORE_SITE, "fs.azure.account.key." + account); - copyPropertyIfExists(CORE_SITE, "fs.azure.account.keyprovider." + account); - copyPropertyIfExists(CORE_SITE, "fs.azure.shellkeyprovider.script"); + private String printConf() { + try { + StringWriter stringWriter = new StringWriter(); + conf.writeXml(stringWriter); + stringWriter.close(); + return stringWriter.toString().replace("\n", ""); + } catch (IOException e) { + LOG.error("error while converting conf to xml : ", e); + return ""; } } - /** - * Fill adl properties if adl:// scheme configured - */ - public void configureADL() { - if (defaultFsUri.getScheme().equals("adl")) { - conf.set("fs.adl.impl", "com.microsoft.azure.datalake.store.AdlFileSystem"); - copyPropertyIfExists(CORE_SITE,"dfs.webhdfs.oauth2.access.token.provider"); - copyPropertyIfExists(CORE_SITE,"fs.azure.datalake.token.provider.service.urls"); - copyPropertyIfExists(CORE_SITE,"fs.azure.datalake.token.provider.script"); - } - } /** * Builds the authentication configuration http://git-wip-us.apache.org/repos/asf/ambari/blob/9cdd3e8a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java ---------------------------------------------------------------------- diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java index c7ae952..66679e8 100644 --- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java +++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java @@ -19,26 +19,29 @@ package org.apache.ambari.view.utils.hdfs; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.PrivilegedExceptionAction; -import java.util.*; - -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; -import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.json.simple.JSONArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.security.auth.Subject; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** * Hdfs Business Delegate http://git-wip-us.apache.org/repos/asf/ambari/blob/9cdd3e8a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsUtil.java ---------------------------------------------------------------------- diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsUtil.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsUtil.java index 8bc3ca5..0670f1a 100644 --- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsUtil.java +++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsUtil.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.HashMap; import java.util.Map; public class HdfsUtil { @@ -108,40 +107,62 @@ public class HdfsUtil { } /** + * takes any custom properties that a view wants to be included into the config + * @param context + * @param customViewProperties + * @return + * @throws HdfsApiException + */ + public static synchronized HdfsApi connectToHDFSApi(ViewContext context, Map<String, String> customViewProperties) + throws HdfsApiException { + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(HdfsUtil.class.getClassLoader()); + try { + ConfigurationBuilder configurationBuilder = new ConfigurationBuilder(context, customViewProperties); + return getHdfsApi(context, configurationBuilder); + } finally { + Thread.currentThread().setContextClassLoader(currentClassLoader); + } + } + + /** * Factory of HdfsApi for specific ViewContext * @param context ViewContext that contains connection credentials * @return HdfsApi object */ public static synchronized HdfsApi connectToHDFSApi(ViewContext context) throws HdfsApiException { - HdfsApi api = null; ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(null); + Thread.currentThread().setContextClassLoader(HdfsUtil.class.getClassLoader()); try { ConfigurationBuilder configurationBuilder = new ConfigurationBuilder(context); - AuthConfigurationBuilder authConfigurationBuilder = new AuthConfigurationBuilder(context); - - Map<String, String> authParams = authConfigurationBuilder.build(); - configurationBuilder.setAuthParams(authParams); - - try { - api = new HdfsApi(configurationBuilder, getHdfsUsername(context)); - LOG.info("HdfsApi connected OK"); - } catch (IOException e) { - String message = "HDFS040 Couldn't open connection to HDFS"; - LOG.error(message); - throw new HdfsApiException(message, e); - } catch (InterruptedException e) { - String message = "HDFS041 Couldn't open connection to HDFS"; - LOG.error(message); - throw new HdfsApiException(message, e); - } - return api; - } - finally { + return getHdfsApi(context, configurationBuilder); + } finally { Thread.currentThread().setContextClassLoader(currentClassLoader); } } + private static HdfsApi getHdfsApi(ViewContext context, ConfigurationBuilder configurationBuilder) throws HdfsApiException { + HdfsApi api = null; + AuthConfigurationBuilder authConfigurationBuilder = new AuthConfigurationBuilder(context); + Map<String, String> authParams = authConfigurationBuilder.build(); + configurationBuilder.setAuthParams(authParams); + try { + api = new HdfsApi(configurationBuilder, getHdfsUsername(context)); + LOG.info("HdfsApi connected OK"); + } catch (IOException e) { + LOG.error("exception occurred while creating hdfsApi objcet : {}", e.getMessage(), e); + String message = "HDFS040 Couldn't open connection to HDFS"; + LOG.error(message); + throw new HdfsApiException(message, e); + } catch (InterruptedException e) { + LOG.error("exception occurred while creating hdfsApi objcet : {}", e.getMessage(), e); + String message = "HDFS041 Couldn't open connection to HDFS"; + LOG.error(message); + throw new HdfsApiException(message, e); + } + return api; + } + /** * Returns username for HdfsApi from "webhdfs.username" property if set, * if not set then current Ambari username http://git-wip-us.apache.org/repos/asf/ambari/blob/9cdd3e8a/contrib/views/wfmanager/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/views/wfmanager/pom.xml b/contrib/views/wfmanager/pom.xml index 53d26a2..ae01a4f 100644 --- a/contrib/views/wfmanager/pom.xml +++ b/contrib/views/wfmanager/pom.xml @@ -88,6 +88,18 @@ <groupId>tomcat</groupId> <artifactId>jasper-runtime</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -96,9 +108,17 @@ <version>${hadoop.version}</version> <exclusions> <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> <groupId>tomcat</groupId> <artifactId>jasper-runtime</artifactId> </exclusion> + <exclusion> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + </exclusion> </exclusions> </dependency> http://git-wip-us.apache.org/repos/asf/ambari/blob/9cdd3e8a/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/FileServices.java ---------------------------------------------------------------------- diff --git a/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/FileServices.java b/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/FileServices.java index ab286b2..591cd85 100644 --- a/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/FileServices.java +++ b/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/FileServices.java @@ -17,14 +17,19 @@ */ package org.apache.oozie.ambari.view; -import javax.ws.rs.Path; - +import com.google.common.base.Optional; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.commons.hdfs.FileOperationService; import org.apache.ambari.view.commons.hdfs.UploadService; import org.apache.ambari.view.commons.hdfs.UserService; +import org.apache.ambari.view.commons.hdfs.ViewPropertyHelper; + +import javax.ws.rs.Path; +import java.util.HashMap; +import java.util.Map; public class FileServices { + public static final String VIEW_CONF_KEYVALUES = "view.conf.keyvalues"; private ViewContext context; @@ -38,7 +43,7 @@ public class FileServices { */ @Path("/upload") public UploadService upload() { - return new UploadService(context); + return new UploadService(context, getViewConfigs()); } /** @@ -47,7 +52,7 @@ public class FileServices { */ @Path("/fileops") public FileOperationService fileOps() { - return new FileOperationService(context); + return new FileOperationService(context, getViewConfigs()); } /** @@ -56,6 +61,11 @@ public class FileServices { */ @Path("/user") public UserService userService() { - return new UserService(context); + return new UserService(context, getViewConfigs()); + } + + private Map<String,String> getViewConfigs() { + Optional<Map<String, String>> props = ViewPropertyHelper.getViewConfigs(context, VIEW_CONF_KEYVALUES); + return props.isPresent()? props.get() : new HashMap<String, String>(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9cdd3e8a/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/HDFSFileUtils.java ---------------------------------------------------------------------- diff --git a/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/HDFSFileUtils.java b/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/HDFSFileUtils.java index 327d8fc..53b43f9 100644 --- a/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/HDFSFileUtils.java +++ b/contrib/views/wfmanager/src/main/java/org/apache/oozie/ambari/view/HDFSFileUtils.java @@ -17,10 +17,9 @@ */ package org.apache.oozie.ambari.view; -import java.io.FileNotFoundException; -import java.io.IOException; - +import com.google.common.base.Optional; import org.apache.ambari.view.ViewContext; +import org.apache.ambari.view.commons.hdfs.ViewPropertyHelper; import org.apache.ambari.view.utils.hdfs.HdfsApi; import org.apache.ambari.view.utils.hdfs.HdfsUtil; import org.apache.hadoop.fs.FSDataInputStream; @@ -29,7 +28,13 @@ import org.apache.hadoop.fs.FileStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; + public class HDFSFileUtils { + public static final String VIEW_CONF_KEYVALUES = "view.conf.keyvalues"; + private final static Logger LOGGER = LoggerFactory .getLogger(HDFSFileUtils.class); private ViewContext viewContext; @@ -84,7 +89,15 @@ public class HDFSFileUtils { private HdfsApi getHdfsgetApi() { try { - return HdfsUtil.connectToHDFSApi(viewContext); + Optional<Map<String, String>> props = ViewPropertyHelper.getViewConfigs(viewContext, VIEW_CONF_KEYVALUES); + HdfsApi api; + if(props.isPresent()){ + api = HdfsUtil.connectToHDFSApi(viewContext, props.get()); + }else{ + api = HdfsUtil.connectToHDFSApi(viewContext); + } + + return api; } catch (Exception ex) { LOGGER.error("Error in getting HDFS Api", ex); throw new RuntimeException( http://git-wip-us.apache.org/repos/asf/ambari/blob/9cdd3e8a/contrib/views/wfmanager/src/main/resources/view.xml ---------------------------------------------------------------------- diff --git a/contrib/views/wfmanager/src/main/resources/view.xml b/contrib/views/wfmanager/src/main/resources/view.xml index c5d4a5b..f2f520d 100644 --- a/contrib/views/wfmanager/src/main/resources/view.xml +++ b/contrib/views/wfmanager/src/main/resources/view.xml @@ -137,6 +137,14 @@ <required>true</required> </parameter> + <parameter> + <name>view.conf.keyvalues</name> + <description>The key values that will be copied to hdfs connection configuration verbatim. Format : key1=value1; + key2=value2</description> + <label>View Configs</label> + <required>false</required> + </parameter> + <persistence> <!-- <entity>
