http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java deleted file mode 100644 index fdd049a..0000000 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java +++ /dev/null @@ -1,912 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.client.rest; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.client.HddsClientUtils; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.client.*; -import org.apache.hadoop.hdds.client.OzoneQuota; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.client.protocol.ClientProtocol; -import org.apache.hadoop.ozone.client.rest.headers.Header; -import org.apache.hadoop.ozone.client.rest.response.BucketInfo; -import org.apache.hadoop.ozone.client.rest.response.KeyInfoDetails; -import org.apache.hadoop.ozone.client.rest.response.VolumeInfo; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.helpers.ServiceInfo; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort; -import org.apache.hadoop.ozone.web.response.ListBuckets; -import org.apache.hadoop.ozone.web.response.ListKeys; -import org.apache.hadoop.ozone.web.response.ListVolumes; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.Time; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpResponse; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static java.net.HttpURLConnection.HTTP_CREATED; -import static java.net.HttpURLConnection.HTTP_OK; - -/** - * Ozone Client REST protocol implementation. It uses REST protocol to - * connect to Ozone Handler that executes client calls. - */ -public class RestClient implements ClientProtocol { - - private static final String PATH_SEPARATOR = "/"; - private static final Logger LOG = LoggerFactory.getLogger(RestClient.class); - - private final Configuration conf; - private final URI ozoneRestUri; - private final CloseableHttpClient httpClient; - private final UserGroupInformation ugi; - private final OzoneAcl.OzoneACLRights userRights; - - /** - * Creates RestClient instance with the given configuration. - * @param conf Configuration - * @throws IOException - */ - public RestClient(Configuration conf) - throws IOException { - try { - Preconditions.checkNotNull(conf); - this.conf = conf; - - long socketTimeout = conf.getTimeDuration( - OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT, - OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT, - TimeUnit.MILLISECONDS); - long connectionTimeout = conf.getTimeDuration( - OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT, - OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT, - TimeUnit.MILLISECONDS); - int maxConnection = conf.getInt( - OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_MAX, - OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_DEFAULT); - - int maxConnectionPerRoute = conf.getInt( - OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX, - OzoneConfigKeys - .OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX_DEFAULT - ); - - /* - To make RestClient Thread safe, creating the HttpClient with - ThreadSafeClientConnManager. - */ - PoolingHttpClientConnectionManager connManager = - new PoolingHttpClientConnectionManager(); - connManager.setMaxTotal(maxConnection); - connManager.setDefaultMaxPerRoute(maxConnectionPerRoute); - - this.httpClient = HttpClients.custom() - .setConnectionManager(connManager) - .setDefaultRequestConfig( - RequestConfig.custom() - .setSocketTimeout(Math.toIntExact(socketTimeout)) - .setConnectTimeout(Math.toIntExact(connectionTimeout)) - .build()) - .build(); - this.ugi = UserGroupInformation.getCurrentUser(); - this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS, - OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT); - - // TODO: Add new configuration parameter to configure RestServerSelector. - RestServerSelector defaultSelector = new DefaultRestServerSelector(); - InetSocketAddress restServer = getOzoneRestServerAddress(defaultSelector); - URIBuilder uriBuilder = new URIBuilder() - .setScheme("http") - .setHost(restServer.getHostName()) - .setPort(restServer.getPort()); - this.ozoneRestUri = uriBuilder.build(); - - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - private InetSocketAddress getOzoneRestServerAddress( - RestServerSelector selector) throws IOException { - String httpAddress = conf.get(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY); - - if (httpAddress == null) { - throw new IllegalArgumentException( - OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY + " must be defined. See" + - " https://wiki.apache.org/hadoop/Ozone#Configuration for" + - " details on configuring Ozone."); - } - - HttpGet httpGet = new HttpGet("http://"; + httpAddress + "/serviceList"); - HttpEntity entity = executeHttpRequest(httpGet); - try { - String serviceListJson = EntityUtils.toString(entity); - - ObjectMapper objectMapper = new ObjectMapper(); - TypeReference<List<ServiceInfo>> serviceInfoReference = - new TypeReference<List<ServiceInfo>>() { - }; - List<ServiceInfo> services = objectMapper.readValue( - serviceListJson, serviceInfoReference); - - List<ServiceInfo> dataNodeInfos = services.stream().filter( - a -> a.getNodeType().equals(HddsProtos.NodeType.DATANODE)) - .collect(Collectors.toList()); - - ServiceInfo restServer = selector.getRestServer(dataNodeInfos); - - return NetUtils.createSocketAddr( - NetUtils.normalizeHostName(restServer.getHostname()) + ":" - + restServer.getPort(ServicePort.Type.HTTP)); - } finally { - EntityUtils.consume(entity); - } - } - - @Override - public void createVolume(String volumeName) throws IOException { - createVolume(volumeName, VolumeArgs.newBuilder().build()); - } - - @Override - public void createVolume(String volumeName, VolumeArgs volArgs) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName); - Preconditions.checkNotNull(volArgs); - URIBuilder builder = new URIBuilder(ozoneRestUri); - String owner = volArgs.getOwner() == null ? - ugi.getUserName() : volArgs.getOwner(); - //TODO: support for ACLs has to be done in OzoneHandler (rest server) - /** - List<OzoneAcl> listOfAcls = new ArrayList<>(); - //User ACL - listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER, - owner, userRights)); - //ACLs from VolumeArgs - if(volArgs.getAcls() != null) { - listOfAcls.addAll(volArgs.getAcls()); - } - */ - builder.setPath(PATH_SEPARATOR + volumeName); - - String quota = volArgs.getQuota(); - if(quota != null) { - builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota); - } - - HttpPost httpPost = new HttpPost(builder.build()); - addOzoneHeaders(httpPost); - //use admin from VolumeArgs, if it's present - if(volArgs.getAdmin() != null) { - httpPost.removeHeaders(HttpHeaders.AUTHORIZATION); - httpPost.addHeader(HttpHeaders.AUTHORIZATION, - Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + - volArgs.getAdmin()); - } - httpPost.addHeader(Header.OZONE_USER, owner); - LOG.info("Creating Volume: {}, with {} as owner and quota set to {}.", - volumeName, owner, quota == null ? "default" : quota); - EntityUtils.consume(executeHttpRequest(httpPost)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - - @Override - public void setVolumeOwner(String volumeName, String owner) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName); - Preconditions.checkNotNull(owner); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName); - HttpPut httpPut = new HttpPut(builder.build()); - addOzoneHeaders(httpPut); - httpPut.addHeader(Header.OZONE_USER, owner); - EntityUtils.consume(executeHttpRequest(httpPut)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void setVolumeQuota(String volumeName, OzoneQuota quota) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName); - Preconditions.checkNotNull(quota); - String quotaString = quota.toString(); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName); - builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quotaString); - HttpPut httpPut = new HttpPut(builder.build()); - addOzoneHeaders(httpPut); - EntityUtils.consume(executeHttpRequest(httpPut)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public OzoneVolume getVolumeDetails(String volumeName) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName); - builder.setParameter(Header.OZONE_INFO_QUERY_TAG, - Header.OZONE_INFO_QUERY_VOLUME); - HttpGet httpGet = new HttpGet(builder.build()); - addOzoneHeaders(httpGet); - HttpEntity response = executeHttpRequest(httpGet); - VolumeInfo volInfo = - VolumeInfo.parse(EntityUtils.toString(response)); - //TODO: OzoneHandler in datanode has to be modified to send ACLs - OzoneVolume volume = new OzoneVolume(conf, - this, - volInfo.getVolumeName(), - volInfo.getCreatedBy(), - volInfo.getOwner().getName(), - volInfo.getQuota().sizeInBytes(), - HddsClientUtils.formatDateTime(volInfo.getCreatedOn()), - null); - EntityUtils.consume(response); - return volume; - } catch (URISyntaxException | ParseException e) { - throw new IOException(e); - } - } - - @Override - public boolean checkVolumeAccess(String volumeName, OzoneAcl acl) - throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); - } - - @Override - public void deleteVolume(String volumeName) throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName); - HttpDelete httpDelete = new HttpDelete(builder.build()); - addOzoneHeaders(httpDelete); - EntityUtils.consume(executeHttpRequest(httpDelete)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public List<OzoneVolume> listVolumes(String volumePrefix, String prevKey, - int maxListResult) - throws IOException { - return listVolumes(null, volumePrefix, prevKey, maxListResult); - } - - @Override - public List<OzoneVolume> listVolumes(String user, String volumePrefix, - String prevKey, int maxListResult) - throws IOException { - try { - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR); - builder.addParameter(Header.OZONE_INFO_QUERY_TAG, - Header.OZONE_LIST_QUERY_SERVICE); - builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, - String.valueOf(maxListResult)); - addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, volumePrefix, builder); - addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey, builder); - HttpGet httpGet = new HttpGet(builder.build()); - if (!Strings.isNullOrEmpty(user)) { - httpGet.addHeader(Header.OZONE_USER, user); - } - addOzoneHeaders(httpGet); - HttpEntity response = executeHttpRequest(httpGet); - ListVolumes volumeList = - ListVolumes.parse(EntityUtils.toString(response)); - EntityUtils.consume(response); - return volumeList.getVolumes().stream().map(volInfo -> { - long creationTime = 0; - try { - creationTime = HddsClientUtils.formatDateTime(volInfo.getCreatedOn()); - } catch (ParseException e) { - LOG.warn("Parse exception in getting creation time for volume", e); - } - return new OzoneVolume(conf, this, volInfo.getVolumeName(), - volInfo.getCreatedBy(), volInfo.getOwner().getName(), - volInfo.getQuota().sizeInBytes(), creationTime, null); - }).collect(Collectors.toList()); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void createBucket(String volumeName, String bucketName) - throws IOException { - createBucket(volumeName, bucketName, BucketArgs.newBuilder().build()); - } - - @Override - public void createBucket( - String volumeName, String bucketName, BucketArgs bucketArgs) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(bucketArgs); - URIBuilder builder = new URIBuilder(ozoneRestUri); - OzoneConsts.Versioning versioning = OzoneConsts.Versioning.DISABLED; - if(bucketArgs.getVersioning() != null && - bucketArgs.getVersioning()) { - versioning = OzoneConsts.Versioning.ENABLED; - } - StorageType storageType = bucketArgs.getStorageType() == null ? - StorageType.DEFAULT : bucketArgs.getStorageType(); - - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName); - HttpPost httpPost = new HttpPost(builder.build()); - addOzoneHeaders(httpPost); - - //ACLs from BucketArgs - if(bucketArgs.getAcls() != null) { - for (OzoneAcl acl : bucketArgs.getAcls()) { - httpPost.addHeader( - Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString()); - } - } - httpPost.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString()); - httpPost.addHeader(Header.OZONE_BUCKET_VERSIONING, - versioning.toString()); - LOG.info("Creating Bucket: {}/{}, with Versioning {} and Storage Type" + - " set to {}", volumeName, bucketName, versioning, - storageType); - - EntityUtils.consume(executeHttpRequest(httpPost)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void addBucketAcls( - String volumeName, String bucketName, List<OzoneAcl> addAcls) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(addAcls); - URIBuilder builder = new URIBuilder(ozoneRestUri); - - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName); - HttpPut httpPut = new HttpPut(builder.build()); - addOzoneHeaders(httpPut); - - for (OzoneAcl acl : addAcls) { - httpPut.addHeader( - Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString()); - } - EntityUtils.consume(executeHttpRequest(httpPut)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void removeBucketAcls( - String volumeName, String bucketName, List<OzoneAcl> removeAcls) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(removeAcls); - URIBuilder builder = new URIBuilder(ozoneRestUri); - - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName); - HttpPut httpPut = new HttpPut(builder.build()); - addOzoneHeaders(httpPut); - - for (OzoneAcl acl : removeAcls) { - httpPut.addHeader( - Header.OZONE_ACLS, Header.OZONE_ACL_REMOVE + " " + acl.toString()); - } - EntityUtils.consume(executeHttpRequest(httpPut)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void setBucketVersioning( - String volumeName, String bucketName, Boolean versioning) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(versioning); - URIBuilder builder = new URIBuilder(ozoneRestUri); - - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName); - HttpPut httpPut = new HttpPut(builder.build()); - addOzoneHeaders(httpPut); - - httpPut.addHeader(Header.OZONE_BUCKET_VERSIONING, - getBucketVersioning(versioning).toString()); - EntityUtils.consume(executeHttpRequest(httpPut)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void setBucketStorageType( - String volumeName, String bucketName, StorageType storageType) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(storageType); - URIBuilder builder = new URIBuilder(ozoneRestUri); - - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName); - HttpPut httpPut = new HttpPut(builder.build()); - addOzoneHeaders(httpPut); - - httpPut.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString()); - EntityUtils.consume(executeHttpRequest(httpPut)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void deleteBucket(String volumeName, String bucketName) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName); - HttpDelete httpDelete = new HttpDelete(builder.build()); - addOzoneHeaders(httpDelete); - EntityUtils.consume(executeHttpRequest(httpDelete)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void checkBucketAccess(String volumeName, String bucketName) - throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); - } - - @Override - public OzoneBucket getBucketDetails(String volumeName, String bucketName) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName); - builder.setParameter(Header.OZONE_INFO_QUERY_TAG, - Header.OZONE_INFO_QUERY_BUCKET); - HttpGet httpGet = new HttpGet(builder.build()); - addOzoneHeaders(httpGet); - HttpEntity response = executeHttpRequest(httpGet); - BucketInfo bucketInfo = - BucketInfo.parse(EntityUtils.toString(response)); - OzoneBucket bucket = new OzoneBucket(conf, - this, - bucketInfo.getVolumeName(), - bucketInfo.getBucketName(), - bucketInfo.getAcls(), - bucketInfo.getStorageType(), - getBucketVersioningFlag(bucketInfo.getVersioning()), - HddsClientUtils.formatDateTime(bucketInfo.getCreatedOn())); - EntityUtils.consume(response); - return bucket; - } catch (URISyntaxException | ParseException e) { - throw new IOException(e); - } - } - - @Override - public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix, - String prevBucket, int maxListResult) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName); - builder.addParameter(Header.OZONE_INFO_QUERY_TAG, - Header.OZONE_INFO_QUERY_BUCKET); - builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, - String.valueOf(maxListResult)); - addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, bucketPrefix, builder); - addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevBucket, builder); - HttpGet httpGet = new HttpGet(builder.build()); - addOzoneHeaders(httpGet); - HttpEntity response = executeHttpRequest(httpGet); - ListBuckets bucketList = - ListBuckets.parse(EntityUtils.toString(response)); - EntityUtils.consume(response); - return bucketList.getBuckets().stream().map(bucketInfo -> { - long creationTime = 0; - try { - creationTime = - HddsClientUtils.formatDateTime(bucketInfo.getCreatedOn()); - } catch (ParseException e) { - LOG.warn("Parse exception in getting creation time for volume", e); - } - return new OzoneBucket(conf, this, volumeName, - bucketInfo.getBucketName(), bucketInfo.getAcls(), - bucketInfo.getStorageType(), - getBucketVersioningFlag(bucketInfo.getVersioning()), creationTime); - }).collect(Collectors.toList()); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - /** - * Writes a key in an existing bucket. - * - * @param volumeName Name of the Volume - * @param bucketName Name of the Bucket - * @param keyName Name of the Key - * @param size Size of the data - * @param type - * @param factor @return {@link OzoneOutputStream} - */ - @Override - public OzoneOutputStream createKey( - String volumeName, String bucketName, String keyName, long size, - ReplicationType type, ReplicationFactor factor) - throws IOException { - // TODO: Once ReplicationType and ReplicationFactor are supported in - // OzoneHandler (in Datanode), set them in header. - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - HddsClientUtils.checkNotNull(keyName, type, factor); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName + - PATH_SEPARATOR + keyName); - HttpPut putRequest = new HttpPut(builder.build()); - addOzoneHeaders(putRequest); - PipedInputStream in = new PipedInputStream(); - OutputStream out = new PipedOutputStream(in); - putRequest.setEntity(new InputStreamEntity(in, size)); - FutureTask<HttpEntity> futureTask = - new FutureTask<>(() -> executeHttpRequest(putRequest)); - new Thread(futureTask).start(); - OzoneOutputStream outputStream = new OzoneOutputStream( - new OutputStream() { - @Override - public void write(int b) throws IOException { - out.write(b); - } - - @Override - public void close() throws IOException { - try { - out.close(); - EntityUtils.consume(futureTask.get()); - } catch (ExecutionException | InterruptedException e) { - throw new IOException(e); - } - } - }); - - return outputStream; - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public OzoneInputStream getKey( - String volumeName, String bucketName, String keyName) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(keyName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName + - PATH_SEPARATOR + keyName); - HttpGet getRequest = new HttpGet(builder.build()); - addOzoneHeaders(getRequest); - HttpEntity entity = executeHttpRequest(getRequest); - PipedInputStream in = new PipedInputStream(); - OutputStream out = new PipedOutputStream(in); - FutureTask<Void> futureTask = - new FutureTask<>(() -> { - entity.writeTo(out); - out.close(); - return null; - }); - new Thread(futureTask).start(); - OzoneInputStream inputStream = new OzoneInputStream( - new InputStream() { - - @Override - public int read() throws IOException { - return in.read(); - } - - @Override - public void close() throws IOException { - in.close(); - EntityUtils.consume(entity); - } - }); - - return inputStream; - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void deleteKey(String volumeName, String bucketName, String keyName) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(keyName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName); - HttpDelete httpDelete = new HttpDelete(builder.build()); - addOzoneHeaders(httpDelete); - EntityUtils.consume(executeHttpRequest(httpDelete)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public void renameKey(String volumeName, String bucketName, - String fromKeyName, String toKeyName) throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - HddsClientUtils.checkNotNull(fromKeyName, toKeyName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName + PATH_SEPARATOR + bucketName - + PATH_SEPARATOR + fromKeyName); - builder.addParameter(Header.OZONE_RENAME_TO_KEY_PARAM_NAME, toKeyName); - HttpPost httpPost = new HttpPost(builder.build()); - addOzoneHeaders(httpPost); - EntityUtils.consume(executeHttpRequest(httpPost)); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public List<OzoneKey> listKeys(String volumeName, String bucketName, - String keyPrefix, String prevKey, - int maxListResult) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder - .setPath(PATH_SEPARATOR + volumeName + PATH_SEPARATOR + bucketName); - builder.addParameter(Header.OZONE_INFO_QUERY_TAG, - Header.OZONE_INFO_QUERY_KEY); - builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, - String.valueOf(maxListResult)); - addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, keyPrefix, builder); - addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey, builder); - HttpGet httpGet = new HttpGet(builder.build()); - addOzoneHeaders(httpGet); - HttpEntity response = executeHttpRequest(httpGet); - ListKeys keyList = ListKeys.parse(EntityUtils.toString(response)); - EntityUtils.consume(response); - return keyList.getKeyList().stream().map(keyInfo -> { - long creationTime = 0, modificationTime = 0; - try { - creationTime = HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()); - modificationTime = - HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()); - } catch (ParseException e) { - LOG.warn("Parse exception in getting creation time for volume", e); - } - return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(), - keyInfo.getSize(), creationTime, modificationTime); - }).collect(Collectors.toList()); - } catch (URISyntaxException e) { - throw new IOException(e); - } - } - - @Override - public OzoneKeyDetails getKeyDetails( - String volumeName, String bucketName, String keyName) - throws IOException { - try { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(keyName); - URIBuilder builder = new URIBuilder(ozoneRestUri); - builder.setPath(PATH_SEPARATOR + volumeName + - PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName); - builder.setParameter(Header.OZONE_INFO_QUERY_TAG, - Header.OZONE_INFO_QUERY_KEY_DETAIL); - HttpGet httpGet = new HttpGet(builder.build()); - addOzoneHeaders(httpGet); - HttpEntity response = executeHttpRequest(httpGet); - KeyInfoDetails keyInfo = - KeyInfoDetails.parse(EntityUtils.toString(response)); - - List<OzoneKeyLocation> ozoneKeyLocations = new ArrayList<>(); - keyInfo.getKeyLocations().forEach((a) -> ozoneKeyLocations.add( - new OzoneKeyLocation(a.getContainerID(), a.getLocalID(), - a.getLength(), a.getOffset()))); - OzoneKeyDetails key = new OzoneKeyDetails(volumeName, - bucketName, - keyInfo.getKeyName(), - keyInfo.getSize(), - HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()), - HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()), - ozoneKeyLocations); - EntityUtils.consume(response); - return key; - } catch (URISyntaxException | ParseException e) { - throw new IOException(e); - } - } - - /** - * Adds Ozone headers to http request. - * - * @param httpRequest Http Request - */ - private void addOzoneHeaders(HttpUriRequest httpRequest) { - httpRequest.addHeader(HttpHeaders.AUTHORIZATION, - Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " + - ugi.getUserName()); - httpRequest.addHeader(HttpHeaders.DATE, - HddsClientUtils.formatDateTime(Time.monotonicNow())); - httpRequest.addHeader(Header.OZONE_VERSION_HEADER, - Header.OZONE_V1_VERSION_HEADER); - } - - /** - * Sends the http request to server and returns the response HttpEntity. - * It's responsibility of the caller to consume and close response HttpEntity - * by calling {@code EntityUtils.consume} - * - * @param httpUriRequest http request - * @throws IOException - */ - private HttpEntity executeHttpRequest(HttpUriRequest httpUriRequest) - throws IOException { - HttpResponse response = httpClient.execute(httpUriRequest); - int errorCode = response.getStatusLine().getStatusCode(); - HttpEntity entity = response.getEntity(); - if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) { - return entity; - } - if (entity != null) { - throw new IOException( - OzoneException.parse(EntityUtils.toString(entity))); - } else { - throw new IOException("Unexpected null in http payload," + - " while processing request"); - } - } - - /** - * Converts OzoneConts.Versioning to boolean. - * - * @param version - * @return corresponding boolean value - */ - private Boolean getBucketVersioningFlag( - OzoneConsts.Versioning version) { - if(version != null) { - switch(version) { - case ENABLED: - return true; - case NOT_DEFINED: - case DISABLED: - default: - return false; - } - } - return false; - } - - /** - * Converts Bucket versioning flag into OzoneConts.Versioning. - * - * @param flag versioning flag - * @return corresponding OzoneConts.Versionin - */ - private OzoneConsts.Versioning getBucketVersioning(Boolean flag) { - if(flag != null) { - if(flag) { - return OzoneConsts.Versioning.ENABLED; - } else { - return OzoneConsts.Versioning.DISABLED; - } - } - return OzoneConsts.Versioning.NOT_DEFINED; - } - - @Override - public void close() throws IOException { - httpClient.close(); - } - - private void addQueryParamter(String param, String value, - URIBuilder builder) { - if (!Strings.isNullOrEmpty(value)) { - builder.addParameter(param, value); - } - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java deleted file mode 100644 index fbd6eb8..0000000 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.client.rest; - -import org.apache.hadoop.ozone.om.helpers.ServiceInfo; - -import java.util.List; - -/** - * The implementor of this interface should select the REST server which will - * be used by the client to connect to Ozone Cluster, given list of - * REST Servers/DataNodes (DataNodes are the ones which hosts REST Service). - */ -public interface RestServerSelector { - - /** - * Returns the REST Service which will be used by the client for connection. - * - * @param restServices list of available REST servers - * @return ServiceInfo - */ - ServiceInfo getRestServer(List<ServiceInfo> restServices); - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java deleted file mode 100644 index 233e788..0000000 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.client.rest.exceptions; - -/** - * This package contains ozone rest client libraries. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java deleted file mode 100644 index 340709f..0000000 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - [email protected] -package org.apache.hadoop.ozone.client.rest.headers; - -import org.apache.hadoop.classification.InterfaceAudience; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java deleted file mode 100644 index ebcc104..0000000 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.client.rest; - -/** - * This package contains Ozone rest client library classes. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java deleted file mode 100644 index 330eba8..0000000 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ /dev/null @@ -1,576 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.client.rpc; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdds.scm.client.HddsClientUtils; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.retry.RetryPolicy; -import org.apache.hadoop.ipc.Client; -import org.apache.hadoop.ipc.ProtobufRpcEngine; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ozone.OmUtils; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.client.*; -import org.apache.hadoop.hdds.client.OzoneQuota; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.ozone.client.VolumeArgs; -import org.apache.hadoop.ozone.client.OzoneClientUtils; -import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream; -import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream; -import org.apache.hadoop.ozone.client.io.LengthInputStream; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.client.protocol.ClientProtocol; -import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.OpenKeySession; -import org.apache.hadoop.ozone.om.helpers.ServiceInfo; -import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; -import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.protocol.proto - .OzoneManagerProtocolProtos.ServicePort; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.protocolPB.OMPBHelper; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.protocolPB - .StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.protocolPB - .StorageContainerLocationProtocolPB; -import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.*; -import java.util.stream.Collectors; - -/** - * Ozone RPC Client Implementation, it connects to OM, SCM and DataNode - * to execute client calls. This uses RPC protocol for communication - * with the servers. - */ -public class RpcClient implements ClientProtocol { - - private static final Logger LOG = - LoggerFactory.getLogger(RpcClient.class); - - private final OzoneConfiguration conf; - private final StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient; - private final OzoneManagerProtocolClientSideTranslatorPB - ozoneManagerClient; - private final XceiverClientManager xceiverClientManager; - private final int chunkSize; - private final UserGroupInformation ugi; - private final OzoneAcl.OzoneACLRights userRights; - private final OzoneAcl.OzoneACLRights groupRights; - private final RetryPolicy retryPolicy; - - /** - * Creates RpcClient instance with the given configuration. - * @param conf - * @throws IOException - */ - public RpcClient(Configuration conf) throws IOException { - Preconditions.checkNotNull(conf); - this.conf = new OzoneConfiguration(conf); - this.ugi = UserGroupInformation.getCurrentUser(); - this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS, - OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT); - this.groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS, - OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT); - long omVersion = - RPC.getProtocolVersion(OzoneManagerProtocolPB.class); - InetSocketAddress omAddress = OmUtils - .getOmAddressForClients(conf); - RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class, - ProtobufRpcEngine.class); - this.ozoneManagerClient = - new OzoneManagerProtocolClientSideTranslatorPB( - RPC.getProxy(OzoneManagerProtocolPB.class, omVersion, - omAddress, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), - Client.getRpcTimeout(conf))); - - long scmVersion = - RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class); - InetSocketAddress scmAddress = getScmAddressForClient(); - RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class, - ProtobufRpcEngine.class); - this.storageContainerLocationClient = - new StorageContainerLocationProtocolClientSideTranslatorPB( - RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion, - scmAddress, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), - Client.getRpcTimeout(conf))); - - this.xceiverClientManager = new XceiverClientManager(conf); - retryPolicy = OzoneClientUtils.createRetryPolicy(conf); - - int configuredChunkSize = conf.getInt( - ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, - ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT); - if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) { - LOG.warn("The chunk size ({}) is not allowed to be more than" - + " the maximum size ({})," - + " resetting to the maximum size.", - configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE); - chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE; - } else { - chunkSize = configuredChunkSize; - } - } - - private InetSocketAddress getScmAddressForClient() throws IOException { - List<ServiceInfo> services = ozoneManagerClient.getServiceList(); - ServiceInfo scmInfo = services.stream().filter( - a -> a.getNodeType().equals(HddsProtos.NodeType.SCM)) - .collect(Collectors.toList()).get(0); - return NetUtils.createSocketAddr(scmInfo.getHostname()+ ":" + - scmInfo.getPort(ServicePort.Type.RPC)); - } - - @Override - public void createVolume(String volumeName) throws IOException { - createVolume(volumeName, VolumeArgs.newBuilder().build()); - } - - @Override - public void createVolume(String volumeName, VolumeArgs volArgs) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName); - Preconditions.checkNotNull(volArgs); - - String admin = volArgs.getAdmin() == null ? - ugi.getUserName() : volArgs.getAdmin(); - String owner = volArgs.getOwner() == null ? - ugi.getUserName() : volArgs.getOwner(); - long quota = volArgs.getQuota() == null ? - OzoneConsts.MAX_QUOTA_IN_BYTES : - OzoneQuota.parseQuota(volArgs.getQuota()).sizeInBytes(); - List<OzoneAcl> listOfAcls = new ArrayList<>(); - //User ACL - listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER, - owner, userRights)); - //Group ACLs of the User - List<String> userGroups = Arrays.asList(UserGroupInformation - .createRemoteUser(owner).getGroupNames()); - userGroups.stream().forEach((group) -> listOfAcls.add( - new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights))); - //ACLs from VolumeArgs - if(volArgs.getAcls() != null) { - listOfAcls.addAll(volArgs.getAcls()); - } - - OmVolumeArgs.Builder builder = OmVolumeArgs.newBuilder(); - builder.setVolume(volumeName); - builder.setAdminName(admin); - builder.setOwnerName(owner); - builder.setQuotaInBytes(quota); - - //Remove duplicates and add ACLs - for (OzoneAcl ozoneAcl : - listOfAcls.stream().distinct().collect(Collectors.toList())) { - builder.addOzoneAcls(OMPBHelper.convertOzoneAcl(ozoneAcl)); - } - - LOG.info("Creating Volume: {}, with {} as owner and quota set to {} bytes.", - volumeName, owner, quota); - ozoneManagerClient.createVolume(builder.build()); - } - - @Override - public void setVolumeOwner(String volumeName, String owner) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName); - Preconditions.checkNotNull(owner); - ozoneManagerClient.setOwner(volumeName, owner); - } - - @Override - public void setVolumeQuota(String volumeName, OzoneQuota quota) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName); - Preconditions.checkNotNull(quota); - long quotaInBytes = quota.sizeInBytes(); - ozoneManagerClient.setQuota(volumeName, quotaInBytes); - } - - @Override - public OzoneVolume getVolumeDetails(String volumeName) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName); - OmVolumeArgs volume = ozoneManagerClient.getVolumeInfo(volumeName); - return new OzoneVolume( - conf, - this, - volume.getVolume(), - volume.getAdminName(), - volume.getOwnerName(), - volume.getQuotaInBytes(), - volume.getCreationTime(), - volume.getAclMap().ozoneAclGetProtobuf().stream(). - map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList())); - } - - @Override - public boolean checkVolumeAccess(String volumeName, OzoneAcl acl) - throws IOException { - throw new UnsupportedOperationException("Not yet implemented."); - } - - @Override - public void deleteVolume(String volumeName) throws IOException { - HddsClientUtils.verifyResourceName(volumeName); - ozoneManagerClient.deleteVolume(volumeName); - } - - @Override - public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume, - int maxListResult) - throws IOException { - List<OmVolumeArgs> volumes = ozoneManagerClient.listAllVolumes( - volumePrefix, prevVolume, maxListResult); - - return volumes.stream().map(volume -> new OzoneVolume( - conf, - this, - volume.getVolume(), - volume.getAdminName(), - volume.getOwnerName(), - volume.getQuotaInBytes(), - volume.getCreationTime(), - volume.getAclMap().ozoneAclGetProtobuf().stream(). - map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList()))) - .collect(Collectors.toList()); - } - - @Override - public List<OzoneVolume> listVolumes(String user, String volumePrefix, - String prevVolume, int maxListResult) - throws IOException { - List<OmVolumeArgs> volumes = ozoneManagerClient.listVolumeByUser( - user, volumePrefix, prevVolume, maxListResult); - - return volumes.stream().map(volume -> new OzoneVolume( - conf, - this, - volume.getVolume(), - volume.getAdminName(), - volume.getOwnerName(), - volume.getQuotaInBytes(), - volume.getCreationTime(), - volume.getAclMap().ozoneAclGetProtobuf().stream(). - map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList()))) - .collect(Collectors.toList()); - } - - @Override - public void createBucket(String volumeName, String bucketName) - throws IOException { - createBucket(volumeName, bucketName, BucketArgs.newBuilder().build()); - } - - @Override - public void createBucket( - String volumeName, String bucketName, BucketArgs bucketArgs) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(bucketArgs); - - Boolean isVersionEnabled = bucketArgs.getVersioning() == null ? - Boolean.FALSE : bucketArgs.getVersioning(); - StorageType storageType = bucketArgs.getStorageType() == null ? - StorageType.DEFAULT : bucketArgs.getStorageType(); - List<OzoneAcl> listOfAcls = new ArrayList<>(); - //User ACL - listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER, - ugi.getUserName(), userRights)); - //Group ACLs of the User - List<String> userGroups = Arrays.asList(UserGroupInformation - .createRemoteUser(ugi.getUserName()).getGroupNames()); - userGroups.stream().forEach((group) -> listOfAcls.add( - new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights))); - //ACLs from BucketArgs - if(bucketArgs.getAcls() != null) { - listOfAcls.addAll(bucketArgs.getAcls()); - } - - OmBucketInfo.Builder builder = OmBucketInfo.newBuilder(); - builder.setVolumeName(volumeName) - .setBucketName(bucketName) - .setIsVersionEnabled(isVersionEnabled) - .setStorageType(storageType) - .setAcls(listOfAcls.stream().distinct().collect(Collectors.toList())); - - LOG.info("Creating Bucket: {}/{}, with Versioning {} and " + - "Storage Type set to {}", volumeName, bucketName, isVersionEnabled, - storageType); - ozoneManagerClient.createBucket(builder.build()); - } - - @Override - public void addBucketAcls( - String volumeName, String bucketName, List<OzoneAcl> addAcls) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(addAcls); - OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); - builder.setVolumeName(volumeName) - .setBucketName(bucketName) - .setAddAcls(addAcls); - ozoneManagerClient.setBucketProperty(builder.build()); - } - - @Override - public void removeBucketAcls( - String volumeName, String bucketName, List<OzoneAcl> removeAcls) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(removeAcls); - OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); - builder.setVolumeName(volumeName) - .setBucketName(bucketName) - .setRemoveAcls(removeAcls); - ozoneManagerClient.setBucketProperty(builder.build()); - } - - @Override - public void setBucketVersioning( - String volumeName, String bucketName, Boolean versioning) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(versioning); - OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); - builder.setVolumeName(volumeName) - .setBucketName(bucketName) - .setIsVersionEnabled(versioning); - ozoneManagerClient.setBucketProperty(builder.build()); - } - - @Override - public void setBucketStorageType( - String volumeName, String bucketName, StorageType storageType) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(storageType); - OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); - builder.setVolumeName(volumeName) - .setBucketName(bucketName) - .setStorageType(storageType); - ozoneManagerClient.setBucketProperty(builder.build()); - } - - @Override - public void deleteBucket( - String volumeName, String bucketName) throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - ozoneManagerClient.deleteBucket(volumeName, bucketName); - } - - @Override - public void checkBucketAccess( - String volumeName, String bucketName) throws IOException { - - } - - @Override - public OzoneBucket getBucketDetails( - String volumeName, String bucketName) throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - OmBucketInfo bucketArgs = - ozoneManagerClient.getBucketInfo(volumeName, bucketName); - return new OzoneBucket( - conf, - this, - bucketArgs.getVolumeName(), - bucketArgs.getBucketName(), - bucketArgs.getAcls(), - bucketArgs.getStorageType(), - bucketArgs.getIsVersionEnabled(), - bucketArgs.getCreationTime()); - } - - @Override - public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix, - String prevBucket, int maxListResult) - throws IOException { - List<OmBucketInfo> buckets = ozoneManagerClient.listBuckets( - volumeName, prevBucket, bucketPrefix, maxListResult); - - return buckets.stream().map(bucket -> new OzoneBucket( - conf, - this, - bucket.getVolumeName(), - bucket.getBucketName(), - bucket.getAcls(), - bucket.getStorageType(), - bucket.getIsVersionEnabled(), - bucket.getCreationTime())) - .collect(Collectors.toList()); - } - - @Override - public OzoneOutputStream createKey( - String volumeName, String bucketName, String keyName, long size, - ReplicationType type, ReplicationFactor factor) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - HddsClientUtils.checkNotNull(keyName, type, factor); - String requestId = UUID.randomUUID().toString(); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(size) - .setType(HddsProtos.ReplicationType.valueOf(type.toString())) - .setFactor(HddsProtos.ReplicationFactor.valueOf(factor.getValue())) - .build(); - - OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs); - ChunkGroupOutputStream groupOutputStream = - new ChunkGroupOutputStream.Builder() - .setHandler(openKey) - .setXceiverClientManager(xceiverClientManager) - .setScmClient(storageContainerLocationClient) - .setOmClient(ozoneManagerClient) - .setChunkSize(chunkSize) - .setRequestID(requestId) - .setType(HddsProtos.ReplicationType.valueOf(type.toString())) - .setFactor(HddsProtos.ReplicationFactor.valueOf(factor.getValue())) - .setRetryPolicy(retryPolicy) - .build(); - groupOutputStream.addPreallocateBlocks( - openKey.getKeyInfo().getLatestVersionLocations(), - openKey.getOpenVersion()); - return new OzoneOutputStream(groupOutputStream); - } - - @Override - public OzoneInputStream getKey( - String volumeName, String bucketName, String keyName) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(keyName); - String requestId = UUID.randomUUID().toString(); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .build(); - OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); - LengthInputStream lengthInputStream = - ChunkGroupInputStream.getFromOmKeyInfo( - keyInfo, xceiverClientManager, storageContainerLocationClient, - requestId); - return new OzoneInputStream(lengthInputStream.getWrappedStream()); - } - - @Override - public void deleteKey( - String volumeName, String bucketName, String keyName) - throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - Preconditions.checkNotNull(keyName); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .build(); - ozoneManagerClient.deleteKey(keyArgs); - } - - @Override - public void renameKey(String volumeName, String bucketName, - String fromKeyName, String toKeyName) throws IOException { - HddsClientUtils.verifyResourceName(volumeName, bucketName); - HddsClientUtils.checkNotNull(fromKeyName, toKeyName); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(fromKeyName) - .build(); - ozoneManagerClient.renameKey(keyArgs, toKeyName); - } - - @Override - public List<OzoneKey> listKeys(String volumeName, String bucketName, - String keyPrefix, String prevKey, - int maxListResult) - throws IOException { - List<OmKeyInfo> keys = ozoneManagerClient.listKeys( - volumeName, bucketName, prevKey, keyPrefix, maxListResult); - - return keys.stream().map(key -> new OzoneKey( - key.getVolumeName(), - key.getBucketName(), - key.getKeyName(), - key.getDataSize(), - key.getCreationTime(), - key.getModificationTime())) - .collect(Collectors.toList()); - } - - @Override - public OzoneKeyDetails getKeyDetails( - String volumeName, String bucketName, String keyName) - throws IOException { - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - Preconditions.checkNotNull(keyName); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .build(); - OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); - - List<OzoneKeyLocation> ozoneKeyLocations = new ArrayList<>(); - keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().forEach( - (a) -> ozoneKeyLocations.add(new OzoneKeyLocation(a.getContainerID(), - a.getLocalID(), a.getLength(), a.getOffset()))); - return new OzoneKeyDetails(keyInfo.getVolumeName(), - keyInfo.getBucketName(), - keyInfo.getKeyName(), - keyInfo.getDataSize(), - keyInfo.getCreationTime(), - keyInfo.getModificationTime(), - ozoneKeyLocations); - } - - @Override - public void close() throws IOException { - IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient); - IOUtils.cleanupWithLogger(LOG, ozoneManagerClient); - IOUtils.cleanupWithLogger(LOG, xceiverClientManager); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java deleted file mode 100644 index 0fcc3fc..0000000 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.client.rpc; - -/** - * This package contains Ozone rpc client library classes. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java deleted file mode 100644 index 3aefe8a..0000000 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.client; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.rules.Timeout; - -import java.net.InetSocketAddress; - -import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; -import static org.apache.hadoop.ozone.OmUtils.getOmAddress; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * This test class verifies the parsing of SCM endpoint config settings. The - * parsing logic is in {@link org.apache.hadoop.hdds.scm.client.HddsClientUtils}. - */ -public class TestHddsClientUtils { - @Rule - public Timeout timeout = new Timeout(300000); - - @Rule - public ExpectedException thrown= ExpectedException.none(); - - /** - * Verify client endpoint lookup failure if it is not configured. - */ - @Test - public void testMissingScmClientAddress() { - final Configuration conf = new OzoneConfiguration(); - thrown.expect(IllegalArgumentException.class); - getScmAddressForClients(conf); - } - - /** - * Verify that the client endpoint can be correctly parsed from - * configuration. - */ - @Test - public void testGetScmClientAddress() { - final Configuration conf = new OzoneConfiguration(); - - // First try a client address with just a host name. Verify it falls - // back to the default port. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = getScmAddressForClients(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); - - // Next try a client address with a host name and port. Verify both - // are used correctly. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - addr = getScmAddressForClients(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(100)); - } - - @Test - public void testGetOmAddress() { - final Configuration conf = new OzoneConfiguration(); - - // First try a client address with just a host name. Verify it falls - // back to the default port. - conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = getOmAddress(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT)); - - // Next try a client address with just a host name and port. Verify the port - // is ignored and the default OM port is used. - conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4:100"); - addr = getOmAddress(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(100)); - - // Assert the we are able to use default configs if no value is specified. - conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, ""); - addr = getOmAddress(conf); - assertThat(addr.getHostString(), is("0.0.0.0")); - assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java deleted file mode 100644 index be63eab..0000000 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.client; - -/** - * This package contains test classes for Ozone Client. - */ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml deleted file mode 100644 index df58f36..0000000 --- a/hadoop-ozone/common/dev-support/findbugsExcludeFile.xml +++ /dev/null @@ -1,22 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<FindBugsFilter> - <Match> - <Package name="org.apache.hadoop.ozone.protocol.proto"/> - </Match> -</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml deleted file mode 100644 index 942576b..0000000 --- a/hadoop-ozone/common/pom.xml +++ /dev/null @@ -1,115 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 -http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-ozone</artifactId> - <version>0.3.0-SNAPSHOT</version> - </parent> - <artifactId>hadoop-ozone-common</artifactId> - <version>0.3.0-SNAPSHOT</version> - <description>Apache Hadoop Ozone Common</description> - <name>Apache Hadoop Ozone Common</name> - <packaging>jar</packaging> - - <dependencies> - - </dependencies> - - <build> - <resources> - <resource> - <directory>${basedir}/src/main/resources</directory> - <excludes> - <exclude>ozone-version-info.properties</exclude> - </excludes> - <filtering>false</filtering> - </resource> - <resource> - <directory>${basedir}/src/main/resources</directory> - <includes> - <include>ozone-version-info.properties</include> - </includes> - <filtering>true</filtering> - </resource> - </resources> - <plugins> - <plugin> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-maven-plugins</artifactId> - <executions> - <execution> - <id>version-info</id> - <phase>generate-resources</phase> - <goals> - <goal>version-info</goal> - </goals> - <configuration> - <source> - <directory>${basedir}/../</directory> - <includes> - <include>*/src/main/java/**/*.java</include> - <include>*/src/main/proto/*.proto</include> - </includes> - </source> - </configuration> - </execution> - <execution> - <id>compile-protoc</id> - <goals> - <goal>protoc</goal> - </goals> - <configuration> - <protocVersion>${protobuf.version}</protocVersion> - <protocCommand>${protoc.path}</protocCommand> - <imports> - <param> - ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto - </param> - <param> - ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ - </param> - <param> - ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ - </param> - <param> - ${basedir}/../../hadoop-hdds/common/src/main/proto/ - </param> - <param>${basedir}/src/main/proto</param> - </imports> - <source> - <directory>${basedir}/src/main/proto</directory> - <includes> - <include>OzoneManagerProtocol.proto</include> - </includes> - </source> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>findbugs-maven-plugin</artifactId> - <configuration> - <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile> - </configuration> - </plugin> - </plugins> - </build> -</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-ozone/common/src/main/bin/ozone ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone deleted file mode 100755 index 4b50771..0000000 --- a/hadoop-ozone/common/src/main/bin/ozone +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The name of the script being executed. -HADOOP_SHELL_EXECNAME="ozone" -MYNAME="${BASH_SOURCE-$0}" - -## @description build up the hdfs command's usage text. -## @audience public -## @stability stable -## @replaceable no -function hadoop_usage -{ - hadoop_add_option "--buildpaths" "attempt to add class files from build tree" - hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon" - hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode" - hadoop_add_option "--hosts filename" "list of hosts to use in worker mode" - hadoop_add_option "--loglevel level" "set the log4j level for this command" - hadoop_add_option "--workers" "turn on worker mode" - - hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" - hadoop_add_subcommand "datanode" daemon "run a HDDS datanode" - hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables" - hadoop_add_subcommand "freon" client "runs an ozone data generator" - hadoop_add_subcommand "fs" client "run a filesystem command on Ozone file system. Equivalent to 'hadoop fs'" - hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path" - hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning." - hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration" - hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode." - hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data" - hadoop_add_subcommand "om" daemon "Ozone Manager" - hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service" - hadoop_add_subcommand "s3g" daemon "run the S3 compatible REST gateway" - hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager" - hadoop_add_subcommand "sh" client "command line interface for object store operations" - hadoop_add_subcommand "version" client "print the version" - - hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false -} - -## @description Default command handler for hadoop command -## @audience public -## @stability stable -## @replaceable no -## @param CLI arguments -function ozonecmd_case -{ - subcmd=$1 - shift - - case ${subcmd} in - classpath) - hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@" - ;; - datanode) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode" - ;; - envvars) - echo "JAVA_HOME='${JAVA_HOME}'" - echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'" - echo "HDFS_DIR='${HDFS_DIR}'" - echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'" - echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'" - echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'" - echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'" - echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" - if [[ -n "${QATESTMODE}" ]]; then - echo "MYNAME=${MYNAME}" - echo "HADOOP_SHELL_EXECNAME=${HADOOP_SHELL_EXECNAME}" - fi - exit 0 - ;; - freon) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.Freon - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" - ;; - genesis) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" - ;; - getozoneconf) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf; - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" - ;; - om) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager - HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties" - HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}" - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" - ;; - sh | shell) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" - ;; - scm) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManager' - hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}" - OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm" - ;; - s3g) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway' - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-s3gateway" - ;; - fs) - HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" - ;; - scmcli) - HADOOP_CLASSNAME=org.apache.hadoop.hdds.scm.cli.SCMCLI - OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-tools" - ;; - version) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.util.OzoneVersionInfo - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-common" - ;; - genconf) - HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations - OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" - ;; - *) - HADOOP_CLASSNAME="${subcmd}" - if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then - hadoop_exit_with_usage 1 - fi - ;; - esac -} - -# let's locate libexec... -if [[ -n "${HADOOP_HOME}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" -else - bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) - HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" -fi - -HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" -# shellcheck disable=SC2034 -HADOOP_NEW_CONFIG=true -if [[ -f "${HADOOP_LIBEXEC_DIR}/ozone-config.sh" ]]; then - # shellcheck source=./hadoop-ozone/common/src/main/bin/ozone-config.sh - . "${HADOOP_LIBEXEC_DIR}/ozone-config.sh" -else - echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/ozone-config.sh." 2>&1 - exit 1 -fi - -# now that we have support code, let's abs MYNAME so we can use it later -MYNAME=$(hadoop_abs "${MYNAME}") - -if [[ $# = 0 ]]; then - hadoop_exit_with_usage 1 -fi - -HADOOP_SUBCMD=$1 -shift - - -if hadoop_need_reexec ozone "${HADOOP_SUBCMD}"; then - hadoop_uservar_su ozone "${HADOOP_SUBCMD}" \ - "${MYNAME}" \ - "--reexec" \ - "${HADOOP_USER_PARAMS[@]}" - exit $? -fi - -hadoop_verify_user_perm "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" - -HADOOP_SUBCMD_ARGS=("$@") - -if declare -f ozone_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then - hadoop_debug "Calling dynamically: ozone_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}" - "ozone_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" -else - ozonecmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" -fi - - -# -# Setting up classpath based on the generate classpath descriptors -# -if [ ! "$OZONE_RUN_ARTIFACT_NAME" ]; then - echo "ERROR: Ozone components require to set OZONE_RUN_ARTIFACT_NAME to set the classpath" - exit -1 -fi -export HDDS_LIB_JARS_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib" -CLASSPATH_FILE="${HADOOP_HDFS_HOME}/share/ozone/classpath/${OZONE_RUN_ARTIFACT_NAME}.classpath" -if [ ! "$CLASSPATH_FILE" ]; then - echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing" - exit -1 -fi -# shellcheck disable=SC1090,SC2086 -source $CLASSPATH_FILE -OIFS=$IFS -IFS=':' -# shellcheck disable=SC2154 -for jar in $classpath; do - hadoop_add_classpath "$jar" -done -hadoop_add_classpath "${HADOOP_HDFS_HOME}/share/ozone/web" - -#We need to add the artifact manually as it's not part the generated classpath desciptor -ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib" -MAIN_ARTIFACT=$(find "$ARTIFACT_LIB_DIR" -name "${OZONE_RUN_ARTIFACT_NAME}-*.jar") -if [ ! "$MAIN_ARTIFACT" ]; then - echo "ERROR: Component jar file $MAIN_ARTIFACT is missing from ${HADOOP_HDFS_HOME}/share/ozone/lib" -fi -hadoop_add_classpath "${MAIN_ARTIFACT}" -IFS=$OIFS - - -hadoop_add_client_opts - -if [[ ${HADOOP_WORKER_MODE} = true ]]; then - hadoop_common_worker_mode_execute "${HADOOP_HDFS_HOME}/bin/ozone" "${HADOOP_USER_PARAMS[@]}" - exit $? -fi - -hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" - -# everything is in globals at this point, so call the generic handler -hadoop_generic_java_subcmd_handler --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
