YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af649d21 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af649d21 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af649d21 Branch: refs/heads/yarn-native-services Commit: af649d21fd66eacabdf288ba26b361be465437e5 Parents: 4ce02c3 Author: Billie Rinaldi <[email protected]> Authored: Mon Dec 19 12:09:53 2016 -0800 Committer: Billie Rinaldi <[email protected]> Committed: Mon Dec 19 12:09:53 2016 -0800 ---------------------------------------------------------------------- .../dev-support/findbugs-exclude.xml | 96 +++++ .../dev-support/findbugs-exclude.xml | 20 - .../hadoop-yarn-slider-core/pom.xml | 5 + .../apache/hadoop/security/KerberosDiags.java | 5 +- .../slider/api/proto/RestTypeMarshalling.java | 283 ------------- .../slider/api/types/ComponentInformation.java | 2 +- .../slider/api/types/RestTypeMarshalling.java | 284 +++++++++++++ .../org/apache/slider/client/SliderClient.java | 56 +-- .../apache/slider/client/TokensOperation.java | 9 +- .../client/ipc/SliderClusterOperations.java | 2 +- .../rest/SliderApplicationApiRestClient.java | 14 - .../org/apache/slider/common/SliderKeys.java | 8 +- .../common/params/AbstractActionArgs.java | 5 +- .../AbstractClusterBuildingActionArgs.java | 1 - .../apache/slider/common/params/CommonArgs.java | 9 +- .../apache/slider/common/tools/Comparators.java | 17 +- .../slider/common/tools/ConfigHelper.java | 2 +- .../slider/common/tools/CoreFileSystem.java | 31 -- .../apache/slider/common/tools/SliderUtils.java | 102 +---- .../slider/core/buildutils/InstanceBuilder.java | 1 - .../apache/slider/core/conf/AggregateConf.java | 2 +- .../org/apache/slider/core/conf/ConfTree.java | 15 +- .../core/persist/AppDefinitionPersister.java | 15 +- .../core/registry/retrieve/AMWebClient.java | 51 --- .../core/restclient/HttpOperationResponse.java | 34 -- .../restclient/UrlConnectionOperations.java | 120 ------ .../slider/core/zk/MiniZooKeeperCluster.java | 27 +- .../apache/slider/core/zk/ZKIntegration.java | 20 +- .../apache/slider/providers/ProviderUtils.java | 21 +- .../providers/docker/DockerProviderService.java | 12 +- .../server/appmaster/management/BoolMetric.java | 15 - .../management/MetricsAndMonitoring.java | 25 -- .../management/RangeLimitedCounter.java | 85 ---- .../appmaster/management/RecordedEvent.java | 58 --- .../server/appmaster/rpc/SliderIPCService.java | 2 +- .../server/appmaster/state/NodeInstance.java | 2 +- .../state/OutstandingRequestTracker.java | 2 +- .../server/appmaster/state/RoleStatus.java | 4 - .../appmaster/web/rest/InsecureAmFilter.java | 1 - .../web/rest/publisher/PublisherResource.java | 5 +- .../appmaster/web/view/ContainerStatsBlock.java | 3 +- .../server/servicemonitor/ProbeStatus.java | 1 + .../utility/AbstractSliderLaunchedService.java | 2 - .../services/workflow/LongLivedProcess.java | 5 +- .../slider/common/tools/TestSliderUtils.java | 2 +- .../slider/test/MiniZooKeeperCluster.java | 395 ------------------- 46 files changed, 501 insertions(+), 1375 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml new file mode 100644 index 0000000..d253762 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml @@ -0,0 +1,96 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<FindBugsFilter> + <Match> + <Package name="org.apache.slider.api.proto" /> + </Match> + <Match> + <class name="~org\.apache\.slider\.*" /> + <Bug pattern="IS2_INCONSISTENT_SYNC" /> + </Match> + <Match> + <Class name="org.apache.slider.core.zk.BlockingZKWatcher" /> + <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" /> + </Match> + <Match> + <Class name="org.apache.slider.server.appmaster.state.ProviderAppState" /> + <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" /> + </Match> + <Match> + <Class name="org.apache.slider.providers.ProviderUtils" /> + <Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" /> + </Match> + <Match> + <Class name="org.apache.slider.server.appmaster.SliderAppMaster" /> + <Bug pattern="WA_AWAIT_NOT_IN_LOOP" /> + </Match> + <Match> + <Class name="org.apache.slider.core.zk.ZKIntegration" /> + <Bug pattern="SBSC_USE_STRINGBUFFER_CONCATENATION" /> + </Match> + <Match> + <Bug pattern="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE" /> + </Match> + <Match> + <Class name="org.apache.slider.server.servicemonitor.ProbeWorker" /> + <Bug pattern="SF_SWITCH_FALLTHROUGH" /> + </Match> + <Match> + <Class name="org.apache.slider.core.persist.JsonSerDeser" /> + <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" /> + </Match> + <Match> + <Class name="org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider" /> + <Bug pattern="EI_EXPOSE_REP" /> + </Match> + <Match> + <Class name="org.apache.slider.server.appmaster.state.OutstandingRequest" /> + <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS" /> + </Match> + <Match> + <Class name="org.apache.slider.server.appmaster.web.SliderAMWebApp" /> + <Bug pattern="LG_LOST_LOGGER_DUE_TO_WEAK_REFERENCE" /> + </Match> + <Match> + <Class name="org.apache.slider.server.servicemonitor.Probe" /> + <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" /> + </Match> + <Match> + <Class name="org.apache.slider.server.services.workflow.ForkedProcessService" /> + <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" /> + </Match> + <Match> + <Class name="org.apache.slider.server.appmaster.state.RoleInstance"/> + <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/> + </Match> + <Match> + <Class name="org.apache.slider.client.SliderClient" /> + <Method name="actionRegistryListConfigsYarn" /> + <Bug pattern="OS_OPEN_STREAM" /> + </Match> + <Match> + <Class name="org.apache.slider.client.SliderClient" /> + <Method name="actionRegistryListExports" /> + <Bug pattern="OS_OPEN_STREAM" /> + </Match> + <Match> + <Class name="org.apache.slider.common.tools.SliderUtils" /> + <Method name="getApplicationResourceInputStream" /> + <Bug pattern="OS_OPEN_STREAM" /> + </Match> +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml deleted file mode 100644 index b89146a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml +++ /dev/null @@ -1,20 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<FindBugsFilter> - -</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index 7453d12..81e607b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -26,6 +26,11 @@ <packaging>jar</packaging> <name>Apache Hadoop YARN Slider Core</name> + <properties> + <!-- Needed for generating FindBugs warnings using parent pom --> + <yarn.basedir>${project.parent.basedir}</yarn.basedir> + </properties> + <build> <!-- resources are filtered for dynamic updates. This gets build info in--> <resources> http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java index 8c572b3..905d4b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java @@ -35,6 +35,7 @@ import java.io.Closeable; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.PrintStream; import java.io.PrintWriter; import java.lang.reflect.InvocationTargetException; import java.net.InetAddress; @@ -82,7 +83,7 @@ public class KerberosDiags implements Closeable { private final Configuration conf; private final List<String> services; - private final PrintWriter out; + private final PrintStream out; private final File keytab; private final String principal; private final long minKeyLength; @@ -97,7 +98,7 @@ public class KerberosDiags implements Closeable { @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") public KerberosDiags(Configuration conf, - PrintWriter out, + PrintStream out, List<String> services, File keytab, String principal, http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java deleted file mode 100644 index ec35028..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.api.proto; - -import org.apache.slider.api.types.ApplicationLivenessInformation; -import org.apache.slider.api.types.ComponentInformation; -import org.apache.slider.api.types.ContainerInformation; -import org.apache.slider.api.types.NodeEntryInformation; -import org.apache.slider.api.types.NodeInformation; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.persist.AggregateConfSerDeser; -import org.apache.slider.core.persist.ConfTreeSerDeser; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Class to handle marshalling of REST - * types to/from Protobuf records. - */ -public class RestTypeMarshalling { - - public static Messages.ApplicationLivenessInformationProto - marshall(ApplicationLivenessInformation info) { - - Messages.ApplicationLivenessInformationProto.Builder builder = - Messages.ApplicationLivenessInformationProto.newBuilder(); - builder.setAllRequestsSatisfied(info.allRequestsSatisfied); - builder.setRequestsOutstanding(info.requestsOutstanding); - return builder.build(); - } - - public static ApplicationLivenessInformation - unmarshall(Messages.ApplicationLivenessInformationProto wire) { - ApplicationLivenessInformation info = new ApplicationLivenessInformation(); - info.allRequestsSatisfied = wire.getAllRequestsSatisfied(); - info.requestsOutstanding = wire.getRequestsOutstanding(); - return info; - } - - public static ComponentInformation - unmarshall(Messages.ComponentInformationProto wire) { - ComponentInformation info = new ComponentInformation(); - info.name = wire.getName(); - info.priority = wire.getPriority(); - info.placementPolicy = wire.getPlacementPolicy(); - - info.actual = wire.getActual(); - info.completed = wire.getCompleted(); - info.desired = wire.getDesired(); - info.failed = wire.getFailed(); - info.releasing = wire.getReleasing(); - info.requested = wire.getRequested(); - info.started = wire.getStarted(); - info.startFailed = wire.getStartFailed(); - info.totalRequested = wire.getTotalRequested(); - info.containers = new ArrayList<>(wire.getContainersList()); - if (wire.hasFailureMessage()) { - info.failureMessage = wire.getFailureMessage(); - } - if (wire.hasPendingAntiAffineRequestCount()) { - info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount(); - } - if (wire.hasIsAARequestOutstanding()) { - info.isAARequestOutstanding = wire.getIsAARequestOutstanding(); - } - return info; - } - public static Messages.ComponentInformationProto marshall(ComponentInformation info) { - - Messages.ComponentInformationProto.Builder builder = - Messages.ComponentInformationProto.newBuilder(); - builder.setName(info.name); - builder.setPriority(info.priority); - builder.setPlacementPolicy(info.placementPolicy); - - builder.setActual(info.actual); - builder.setCompleted(info.completed); - builder.setDesired(info.desired); - builder.setFailed(info.failed); - builder.setReleasing(info.releasing); - builder.setRequested(info.requested); - builder.setStarted(info.started); - builder.setStartFailed(info.startFailed); - builder.setTotalRequested(info.totalRequested); - builder.setNodeFailed(info.nodeFailed); - builder.setPreempted(info.preempted); - builder.setFailedRecently(info.failedRecently); - if (info.failureMessage != null) { - builder.setFailureMessage(info.failureMessage); - } - if (info.containers != null) { - builder.addAllContainers(info.containers); - } - builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount); - builder.setIsAARequestOutstanding(info.isAARequestOutstanding); - return builder.build(); - } - - public static Messages.NodeInformationProto marshall(NodeInformation info) { - - Messages.NodeInformationProto.Builder builder = - Messages.NodeInformationProto.newBuilder(); - builder.setHostname(info.hostname); - builder.setLastUpdated(info.lastUpdated); - builder.setState(info.state != null? info.state : "unknown"); - builder.setRackName(info.rackName != null ? info.rackName : ""); - builder.setHealthReport(info.healthReport != null ? info.healthReport : ""); - builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : ""); - builder.setLabels(info.labels != null ? info.labels: ""); - - - if (info.entries != null) { - for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) { - NodeEntryInformation entry = elt.getValue(); - Messages.NodeEntryInformationProto.Builder node = - Messages.NodeEntryInformationProto.newBuilder(); - node.setPriority(entry.priority); - node.setName(elt.getKey()); - node.setFailed(entry.failed); - node.setFailedRecently(entry.failedRecently); - node.setLive(entry.live); - node.setLastUsed(entry.lastUsed); - node.setPreempted(entry.preempted); - node.setRequested(entry.requested); - node.setReleasing(entry.releasing); - node.setStartFailed(entry.startFailed); - node.setStarting(entry.starting); - builder.addEntries(node.build()); - } - } - return builder.build(); - } - - public static NodeInformation unmarshall(Messages.NodeInformationProto wire) { - NodeInformation info = new NodeInformation(); - info.healthReport = wire.getHealthReport(); - info.hostname = wire.getHostname(); - info.httpAddress = wire.getHttpAddress(); - info.labels = wire.getLabels(); - info.lastUpdated = wire.getLastUpdated(); - info.rackName = wire.getRackName(); - info.state = wire.getState(); - List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList(); - if (entriesList != null) { - info.entries = new HashMap<>(entriesList.size()); - for (Messages.NodeEntryInformationProto entry : entriesList) { - NodeEntryInformation nei = new NodeEntryInformation(); - nei.failed = entry.getFailed(); - nei.failedRecently = entry.getFailedRecently(); - nei.lastUsed = entry.getLastUsed(); - nei.live = entry.getLive(); - nei.preempted = entry.getPreempted(); - nei.priority = entry.getPriority(); - nei.requested = entry.getRequested(); - nei.releasing = entry.getReleasing(); - nei.startFailed = entry.getStartFailed(); - nei.starting = entry.getStarting(); - info.entries.put(entry.getName(), nei); - } - } - return info; - } - - public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) { - ContainerInformation info = new ContainerInformation(); - info.containerId = wire.getContainerId(); - info.component = wire.getComponent(); - info.appVersion = wire.getAppVersion(); - info.state = wire.getState(); - if (wire.hasReleased()) { - info.released = wire.getReleased(); - } - if (wire.hasExitCode()) { - info.exitCode = wire.getExitCode(); - } - if (wire.hasDiagnostics()) { - info.diagnostics = wire.getDiagnostics(); - } - if (wire.hasHost()) { - info.host = wire.getHost(); - } - if (wire.hasHostURL()) { - info.host = wire.getHostURL(); - } - info.createTime = wire.getCreateTime(); - info.startTime = wire.getStartTime(); - info.output = wire.getOutputList().toArray( - new String[wire.getOutputCount()] - ); - if (wire.hasPlacement()) { - info.placement = wire.getPlacement(); - } - return info; - } - - public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) { - List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size()); - for (Messages.ContainerInformationProto container : wire.getContainersList()) { - infoList.add(unmarshall(container)); - } - return infoList; - } - - public static Messages.ContainerInformationProto marshall(ContainerInformation info) { - - Messages.ContainerInformationProto.Builder builder = - Messages.ContainerInformationProto.newBuilder(); - if (info.containerId != null) { - builder.setContainerId(info.containerId); - } - if (info.component != null) { - builder.setComponent(info.component); - } - if (info.appVersion != null) { - builder.setAppVersion(info.appVersion); - } - builder.setCreateTime(info.createTime); - if (info.diagnostics != null) { - builder.setDiagnostics(info.diagnostics); - } - if (info.host != null) { - builder.setHost(info.host); - } - if (info.hostURL != null) { - builder.setHostURL(info.hostURL); - } - if (info.output != null) { - builder.addAllOutput(Arrays.asList(info.output)); - } - if (info.released != null) { - builder.setReleased(info.released); - } - if (info.placement != null) { - builder.setPlacement(info.placement); - } - builder.setStartTime(info.startTime); - builder.setState(info.state); - return builder.build(); - } - - public static String unmarshall(Messages.WrappedJsonProto wire) { - return wire.getJson(); - } - - public static ConfTree unmarshallToConfTree(Messages.WrappedJsonProto wire) throws - IOException { - return new ConfTreeSerDeser().fromJson(wire.getJson()); - } - - public static ConfTreeOperations unmarshallToCTO(Messages.WrappedJsonProto wire) throws - IOException { - return new ConfTreeOperations(new ConfTreeSerDeser().fromJson(wire.getJson())); - } - - public static AggregateConf unmarshallToAggregateConf(Messages.WrappedJsonProto wire) throws - IOException { - return new AggregateConfSerDeser().fromJson(wire.getJson()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java index c46a59f..d2fdd62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java @@ -31,7 +31,7 @@ import java.util.Map; * Serializable version of component data. * <p> * This is sent in REST calls as a JSON object âbut is also marshalled into - * a protobuf structure. Look at {@link org.apache.slider.api.proto.RestTypeMarshalling} + * a protobuf structure. Look at {@link RestTypeMarshalling} * for the specifics there. * <p> * This means that if any fields are added here. they must be added to http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java new file mode 100644 index 0000000..713cffd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.types.ApplicationLivenessInformation; +import org.apache.slider.api.types.ComponentInformation; +import org.apache.slider.api.types.ContainerInformation; +import org.apache.slider.api.types.NodeEntryInformation; +import org.apache.slider.api.types.NodeInformation; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.core.conf.ConfTreeOperations; +import org.apache.slider.core.persist.AggregateConfSerDeser; +import org.apache.slider.core.persist.ConfTreeSerDeser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Class to handle marshalling of REST + * types to/from Protobuf records. + */ +public class RestTypeMarshalling { + + public static Messages.ApplicationLivenessInformationProto + marshall(ApplicationLivenessInformation info) { + + Messages.ApplicationLivenessInformationProto.Builder builder = + Messages.ApplicationLivenessInformationProto.newBuilder(); + builder.setAllRequestsSatisfied(info.allRequestsSatisfied); + builder.setRequestsOutstanding(info.requestsOutstanding); + return builder.build(); + } + + public static ApplicationLivenessInformation + unmarshall(Messages.ApplicationLivenessInformationProto wire) { + ApplicationLivenessInformation info = new ApplicationLivenessInformation(); + info.allRequestsSatisfied = wire.getAllRequestsSatisfied(); + info.requestsOutstanding = wire.getRequestsOutstanding(); + return info; + } + + public static ComponentInformation + unmarshall(Messages.ComponentInformationProto wire) { + ComponentInformation info = new ComponentInformation(); + info.name = wire.getName(); + info.priority = wire.getPriority(); + info.placementPolicy = wire.getPlacementPolicy(); + + info.actual = wire.getActual(); + info.completed = wire.getCompleted(); + info.desired = wire.getDesired(); + info.failed = wire.getFailed(); + info.releasing = wire.getReleasing(); + info.requested = wire.getRequested(); + info.started = wire.getStarted(); + info.startFailed = wire.getStartFailed(); + info.totalRequested = wire.getTotalRequested(); + info.containers = new ArrayList<>(wire.getContainersList()); + if (wire.hasFailureMessage()) { + info.failureMessage = wire.getFailureMessage(); + } + if (wire.hasPendingAntiAffineRequestCount()) { + info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount(); + } + if (wire.hasIsAARequestOutstanding()) { + info.isAARequestOutstanding = wire.getIsAARequestOutstanding(); + } + return info; + } + public static Messages.ComponentInformationProto marshall(ComponentInformation info) { + + Messages.ComponentInformationProto.Builder builder = + Messages.ComponentInformationProto.newBuilder(); + builder.setName(info.name); + builder.setPriority(info.priority); + builder.setPlacementPolicy(info.placementPolicy); + + builder.setActual(info.actual); + builder.setCompleted(info.completed); + builder.setDesired(info.desired); + builder.setFailed(info.failed); + builder.setReleasing(info.releasing); + builder.setRequested(info.requested); + builder.setStarted(info.started); + builder.setStartFailed(info.startFailed); + builder.setTotalRequested(info.totalRequested); + builder.setNodeFailed(info.nodeFailed); + builder.setPreempted(info.preempted); + builder.setFailedRecently(info.failedRecently); + if (info.failureMessage != null) { + builder.setFailureMessage(info.failureMessage); + } + if (info.containers != null) { + builder.addAllContainers(info.containers); + } + builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount); + builder.setIsAARequestOutstanding(info.isAARequestOutstanding); + return builder.build(); + } + + public static Messages.NodeInformationProto marshall(NodeInformation info) { + + Messages.NodeInformationProto.Builder builder = + Messages.NodeInformationProto.newBuilder(); + builder.setHostname(info.hostname); + builder.setLastUpdated(info.lastUpdated); + builder.setState(info.state != null? info.state : "unknown"); + builder.setRackName(info.rackName != null ? info.rackName : ""); + builder.setHealthReport(info.healthReport != null ? info.healthReport : ""); + builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : ""); + builder.setLabels(info.labels != null ? info.labels: ""); + + + if (info.entries != null) { + for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) { + NodeEntryInformation entry = elt.getValue(); + Messages.NodeEntryInformationProto.Builder node = + Messages.NodeEntryInformationProto.newBuilder(); + node.setPriority(entry.priority); + node.setName(elt.getKey()); + node.setFailed(entry.failed); + node.setFailedRecently(entry.failedRecently); + node.setLive(entry.live); + node.setLastUsed(entry.lastUsed); + node.setPreempted(entry.preempted); + node.setRequested(entry.requested); + node.setReleasing(entry.releasing); + node.setStartFailed(entry.startFailed); + node.setStarting(entry.starting); + builder.addEntries(node.build()); + } + } + return builder.build(); + } + + public static NodeInformation unmarshall(Messages.NodeInformationProto wire) { + NodeInformation info = new NodeInformation(); + info.healthReport = wire.getHealthReport(); + info.hostname = wire.getHostname(); + info.httpAddress = wire.getHttpAddress(); + info.labels = wire.getLabels(); + info.lastUpdated = wire.getLastUpdated(); + info.rackName = wire.getRackName(); + info.state = wire.getState(); + List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList(); + if (entriesList != null) { + info.entries = new HashMap<>(entriesList.size()); + for (Messages.NodeEntryInformationProto entry : entriesList) { + NodeEntryInformation nei = new NodeEntryInformation(); + nei.failed = entry.getFailed(); + nei.failedRecently = entry.getFailedRecently(); + nei.lastUsed = entry.getLastUsed(); + nei.live = entry.getLive(); + nei.preempted = entry.getPreempted(); + nei.priority = entry.getPriority(); + nei.requested = entry.getRequested(); + nei.releasing = entry.getReleasing(); + nei.startFailed = entry.getStartFailed(); + nei.starting = entry.getStarting(); + info.entries.put(entry.getName(), nei); + } + } + return info; + } + + public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) { + ContainerInformation info = new ContainerInformation(); + info.containerId = wire.getContainerId(); + info.component = wire.getComponent(); + info.appVersion = wire.getAppVersion(); + info.state = wire.getState(); + if (wire.hasReleased()) { + info.released = wire.getReleased(); + } + if (wire.hasExitCode()) { + info.exitCode = wire.getExitCode(); + } + if (wire.hasDiagnostics()) { + info.diagnostics = wire.getDiagnostics(); + } + if (wire.hasHost()) { + info.host = wire.getHost(); + } + if (wire.hasHostURL()) { + info.host = wire.getHostURL(); + } + info.createTime = wire.getCreateTime(); + info.startTime = wire.getStartTime(); + info.output = wire.getOutputList().toArray( + new String[wire.getOutputCount()] + ); + if (wire.hasPlacement()) { + info.placement = wire.getPlacement(); + } + return info; + } + + public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) { + List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size()); + for (Messages.ContainerInformationProto container : wire.getContainersList()) { + infoList.add(unmarshall(container)); + } + return infoList; + } + + public static Messages.ContainerInformationProto marshall(ContainerInformation info) { + + Messages.ContainerInformationProto.Builder builder = + Messages.ContainerInformationProto.newBuilder(); + if (info.containerId != null) { + builder.setContainerId(info.containerId); + } + if (info.component != null) { + builder.setComponent(info.component); + } + if (info.appVersion != null) { + builder.setAppVersion(info.appVersion); + } + builder.setCreateTime(info.createTime); + if (info.diagnostics != null) { + builder.setDiagnostics(info.diagnostics); + } + if (info.host != null) { + builder.setHost(info.host); + } + if (info.hostURL != null) { + builder.setHostURL(info.hostURL); + } + if (info.output != null) { + builder.addAllOutput(Arrays.asList(info.output)); + } + if (info.released != null) { + builder.setReleased(info.released); + } + if (info.placement != null) { + builder.setPlacement(info.placement); + } + builder.setStartTime(info.startTime); + builder.setState(info.state); + return builder.build(); + } + + public static String unmarshall(Messages.WrappedJsonProto wire) { + return wire.getJson(); + } + + public static ConfTree unmarshallToConfTree(Messages.WrappedJsonProto wire) throws + IOException { + return new ConfTreeSerDeser().fromJson(wire.getJson()); + } + + public static ConfTreeOperations unmarshallToCTO(Messages.WrappedJsonProto wire) throws + IOException { + return new ConfTreeOperations(new ConfTreeSerDeser().fromJson(wire.getJson())); + } + + public static AggregateConf unmarshallToAggregateConf(Messages.WrappedJsonProto wire) throws + IOException { + return new AggregateConfSerDeser().fromJson(wire.getJson()); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 1c126ac..1a959d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -192,10 +192,10 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1322,9 +1322,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } private void initializeOutputStream(String outFile) - throws FileNotFoundException { + throws IOException { if (outFile != null) { - clientOutputStream = new PrintStream(new FileOutputStream(outFile)); + clientOutputStream = new PrintStream(outFile, "UTF-8"); } else { clientOutputStream = System.out; } @@ -3299,7 +3299,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe int updateCount = Integer.parseInt(updateCountStr); // if component was specified before, get the current count if (component.get(COMPONENT_INSTANCES) != null) { - currentCount = Integer.valueOf(component.get(COMPONENT_INSTANCES)); + currentCount = Integer.parseInt(component.get(COMPONENT_INSTANCES)); if (currentCount + updateCount < 0) { throw new BadCommandArgumentsException("The requested count " + "of \"%s\" for role %s makes the total number of " + @@ -3610,16 +3610,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // as this is an API entry point, validate // the arguments args.validate(); - RegistryOperations operations = getRegistryOperations(); String path = SliderRegistryUtils.resolvePath(args.path); ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal(); try { if (args.list) { File destDir = args.destdir; - if (destDir != null) { - destDir.mkdirs(); + if (destDir != null && !destDir.exists() && !destDir.mkdirs()) { + throw new IOException("Failed to create directory: " + destDir); } + Map<String, ServiceRecord> recordMap; Map<String, RegistryPathStatus> znodes; try { @@ -3656,9 +3656,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else { String filename = RegistryPathUtils.lastPathEntry(name) + ".json"; File jsonFile = new File(destDir, filename); - write(jsonFile, - serviceRecordMarshal.toBytes(instance), - true); + write(jsonFile, serviceRecordMarshal.toBytes(instance)); } } } else { @@ -3669,7 +3667,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path)); } if (outFile != null) { - write(outFile, serviceRecordMarshal.toBytes(instance), true); + write(outFile, serviceRecordMarshal.toBytes(instance)); } else { println(serviceRecordMarshal.toJson(instance)); } @@ -4062,11 +4060,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") private int actionKDiag(ActionKDiagArgs args) throws Exception { - PrintWriter out = new PrintWriter(System.err); + PrintStream out; boolean closeStream = false; if (args.out != null) { - out = new PrintWriter(new FileOutputStream(args.out)); + out = new PrintStream(args.out, "UTF-8"); closeStream = true; + } else { + out = System.err; } try { KerberosDiags kdiags = new KerberosDiags(getConfig(), @@ -4137,7 +4137,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe PrintStream out = null; try { if (registryArgs.out != null) { - out = new PrintStream(new FileOutputStream(registryArgs.out)); + out = new PrintStream(registryArgs.out, "UTF-8"); } else { out = System.out; } @@ -4145,11 +4145,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (!registryArgs.verbose) { out.println(configName); } else { - PublishedConfiguration published = - configurations.get(configName); - out.printf("%s: %s\n", - configName, - published.description); + PublishedConfiguration published = configurations.get(configName); + out.printf("%s: %s%n", configName, published.description); } } } finally { @@ -4178,7 +4175,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe boolean streaming = false; try { if (registryArgs.out != null) { - out = new PrintStream(new FileOutputStream(registryArgs.out)); + out = new PrintStream(registryArgs.out, "UTF-8"); streaming = true; log.debug("Saving output to {}", registryArgs.out); } else { @@ -4193,9 +4190,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe out.println(exportName); } else { PublishedExports published = exports.get(exportName); - out.printf("%s: %s\n", - exportName, - published.description); + out.printf("%s: %s%n", exportName, published.description); } } } finally { @@ -4401,9 +4396,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * Output to standard out/stderr (implementation specific detail) * @param src source */ - @SuppressWarnings("UseOfSystemOutOrSystemErr") private static void print(CharSequence src) { - clientOutputStream.append(src); + clientOutputStream.print(src); } /** @@ -4411,8 +4405,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @param message message */ private static void println(String message) { - print(message); - print("\n"); + clientOutputStream.println(message); } /** * Output to standard out/stderr with a newline after, formatted @@ -4420,8 +4413,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @param args arguments for string formatting */ private static void println(String message, Object ... args) { - print(String.format(message, args)); - print("\n"); + clientOutputStream.println(String.format(message, args)); } /** @@ -4497,12 +4489,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throw new UsageException(CommonArgs.usage(serviceArgs, actionName)); } - private int actionHelp(String errMsg, String actionName) - throws YarnException, IOException { - throw new UsageException("%s %s", errMsg, CommonArgs.usage(serviceArgs, - actionName)); - } - /** * List the nodes in the cluster, possibly filtering by node state or label. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java index 9b9c141..84c65b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java @@ -75,11 +75,10 @@ public class TokensOperation { } saveTokens(output, credentials); String filename = output.getCanonicalPath(); - footnote = String.format("%d tokens saved to %s\n" + - "To use these in the environment:\n" + - "export %s=%s", - credentials.numberOfTokens(), - filename, UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename); + footnote = String.format( + "%d tokens saved to %s%n" + "To use these in the environment:%n" + + "export %s=%s", credentials.numberOfTokens(), filename, + UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename); } else if (args.source != null) { File source = args.source; log.info("Reading credentials from file {}", source); http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java index eaf15e6..702233a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java @@ -51,7 +51,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.slider.api.proto.RestTypeMarshalling.*; +import static org.apache.slider.api.types.RestTypeMarshalling.*; /** * Cluster operations at a slightly higher level than the RPC code http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java index 4283ee8..573ef64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java @@ -297,20 +297,6 @@ public class SliderApplicationApiRestClient extends BaseRestClient .type(MediaType.APPLICATION_JSON_TYPE) .post(PingInformation.class, f); } - - /** - * Ping as a POST - * @param text text to include - * @return the response - * @throws IOException on any failure - */ - public PingInformation pingPut(String text) throws IOException { - WebResource pingResource = applicationResource(ACTION_PING); - Form f = new Form(); - return pingResource - .type(MediaType.TEXT_PLAIN) - .put(PingInformation.class, text); - } @Override public void stop(String text) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index adf40ce..4bf1b5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -18,6 +18,9 @@ package org.apache.slider.common; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; /** * Keys and various constants for Slider @@ -79,8 +82,9 @@ public interface SliderKeys extends SliderXmlConfKeys { */ String COMPONENT_TYPE_EXTERNAL_APP = "external_app"; String COMPONENT_SEPARATOR = "-"; - String[] COMPONENT_KEYS_TO_SKIP = {"zookeeper.", "env.MALLOC_ARENA_MAX", - "site.fs.", "site.dfs."}; + List<String> COMPONENT_KEYS_TO_SKIP = Collections.unmodifiableList(Arrays + .asList("zookeeper.", "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs.")); + /** * A component type for a client component */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java index e3cb288..63ccff8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java @@ -142,11 +142,12 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments { log.error(message); int index = 1; + StringBuilder buf = new StringBuilder(message); for (String actionArg : parameters) { log.error("[{}] \"{}\"", index++, actionArg); - message += " \"" + actionArg + "\" "; + buf.append(" \"" + actionArg + "\" "); } - throw new BadCommandArgumentsException(message); + throw new BadCommandArgumentsException(buf.toString()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index 20a9989..1d28c78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -214,7 +214,6 @@ public abstract class AbstractClusterBuildingActionArgs extends protected ConfTree buildConfTree(Map<String, String> optionsMap) throws BadCommandArgumentsException { ConfTree confTree = new ConfTree(); - ConfTreeOperations ops = new ConfTreeOperations(confTree); confTree.global.putAll(optionsMap); return confTree; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java index 162a87d..9c8e65d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java @@ -154,12 +154,12 @@ public abstract class CommonArgs extends ArgOps implements SliderActions, public void parse() throws SliderException { addActionArguments(); try { - commander.parse(getArgs()); + commander.parse(args); } catch (ParameterException e) { throw new BadCommandArgumentsException(e, "%s in %s", e.toString(), - (getArgs() != null - ? (SliderUtils.join(getArgs(), + (args != null + ? (SliderUtils.join(args, " ", false)) : "[]")); } @@ -297,7 +297,4 @@ public abstract class CommonArgs extends ArgOps implements SliderActions, return coreAction.parameters; } - public String[] getArgs() { - return args; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java index 6380d0c..a83901b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java @@ -29,25 +29,18 @@ public class Comparators { public static class LongComparator implements Comparator<Long>, Serializable { @Override public int compare(Long o1, Long o2) { - long result = o1 - o2; - // need to comparisons with a diff greater than integer size - if (result < 0 ) { - return -1; - } else if (result > 0) { - return 1; - } - return 0; + return o1.compareTo(o2); } } -public static class InvertedLongComparator implements Comparator<Long>, Serializable { - private static final LongComparator inner = new LongComparator(); + + public static class InvertedLongComparator + implements Comparator<Long>, Serializable { @Override public int compare(Long o1, Long o2) { - return -inner.compare(o1, o2); + return o2.compareTo(o1); } } - /** * Little template class to reverse any comparitor * @param <CompareType> the type that is being compared http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java index 9db241d..02eba49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -194,7 +194,7 @@ public class ConfigHelper { byte[] data = loadBytes(fs, path); //this is here to track down a parse issue //related to configurations - String s = new String(data, 0, data.length); + String s = new String(data, 0, data.length, "UTF-8"); log.debug("XML resource {} is \"{}\"", path, s); /* JDK7 try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index aa5edf1..c3d6d98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -464,37 +464,6 @@ public class CoreFileSystem { } /** - * Verify that a file exists in the zip file given by path - * @param path path to zip file - * @param file file expected to be in zip - * @throws FileNotFoundException file not found or is not a zip file - * @throws IOException trouble with FS - */ - public void verifyFileExistsInZip(Path path, String file) throws IOException { - fileSystem.copyToLocalFile(path, new Path("/tmp")); - File dst = new File((new Path("/tmp", path.getName())).toString()); - Enumeration<? extends ZipEntry> entries; - ZipFile zipFile = new ZipFile(dst); - boolean found = false; - - try { - entries = zipFile.entries(); - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - String nm = entry.getName(); - if (nm.endsWith(file)) { - found = true; - break; - } - } - } finally { - zipFile.close(); - } - dst.delete(); - if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path); - log.info("Verification of " + path + " passed"); - } - /** * Create the application-instance specific temporary directory * in the DFS * http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 4457957..a107dfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -36,7 +36,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.GlobFilter; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; @@ -310,10 +309,6 @@ public final class SliderUtils { } String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; Enumeration<URL> urlEnumeration = loader.getResources(class_file); - if (urlEnumeration == null) { - throw new IOException("Unable to find resources for class " + my_class); - } - for (; urlEnumeration.hasMoreElements(); ) { URL url = urlEnumeration.nextElement(); if ("jar".equals(url.getProtocol())) { @@ -756,10 +751,10 @@ public final class SliderUtils { public static String containersToString( List<ContainerInformation> containers, String version, Set<String> components) { - String containerf = " %-28s %30s %45s %s\n"; + String containerf = " %-28s %30s %45s %s%n"; StringBuilder builder = new StringBuilder(512); - builder.append("Containers:\n"); - builder.append(String.format(" %-28s %30s %45s %s\n", "Component Name", + builder.append("Containers:%n"); + builder.append(String.format(" %-28s %30s %45s %s%n", "Component Name", "App Version", "Container Id", "Container Info/Logs")); for (ContainerInformation container : containers) { if (filter(container.appVersion, version) @@ -969,7 +964,7 @@ public final class SliderUtils { */ public static Map<String, String> mergeMapsIgnoreDuplicateKeysAndPrefixes( Map<String, String> first, Map<String, String> second, - String... prefixes) { + List<String> prefixes) { Preconditions.checkArgument(first != null, "Null 'first' value"); Preconditions.checkArgument(second != null, "Null 'second' value"); Preconditions.checkArgument(prefixes != null, "Null 'prefixes' value"); @@ -2119,15 +2114,16 @@ public final class SliderUtils { is = new ByteArrayInputStream(content); } else { log.debug("Size unknown. Reading {}", zipEntry.getName()); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - while (true) { - int byteRead = zis.read(); - if (byteRead == -1) { - break; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + while (true) { + int byteRead = zis.read(); + if (byteRead == -1) { + break; + } + baos.write(byteRead); } - baos.write(byteRead); + is = new ByteArrayInputStream(baos.toByteArray()); } - is = new ByteArrayInputStream(baos.toByteArray()); } done = true; } @@ -2205,90 +2201,26 @@ public final class SliderUtils { } /** - * Look for the windows executable and check it has the right headers. - * <code>File.canRead()</code> doesn't work on windows, so the reading - * is mandatory. - * - * @param program program name for errors - * @param exe executable - * @throws IOException IOE - */ - public static void verifyWindowsExe(String program, File exe) - throws IOException { - verifyIsFile(program, exe); - - verifyFileSize(program, exe, 0x100); - - // now read two bytes and verify the header. - try(FileReader reader = new FileReader(exe)) { - int[] header = new int[2]; - header[0] = reader.read(); - header[1] = reader.read(); - if ((header[0] != 'M' || header[1] != 'Z')) { - throw new FileNotFoundException(program - + " at " + exe - + " is not a windows executable file"); - } - } - } - - /** - * Verify that a Unix exe works - * @param program program name for errors - * @param exe executable - * @throws IOException IOE - - */ - public static void verifyUnixExe(String program, File exe) - throws IOException { - verifyIsFile(program, exe); - - // read flag - if (!exe.canRead()) { - throw new IOException("Cannot read " + program + " at " + exe); - } - // exe flag - if (!exe.canExecute()) { - throw new IOException("Cannot execute " + program + " at " + exe); - } - } - - /** - * Validate an executable - * @param program program name for errors - * @param exe program to look at - * @throws IOException - */ - public static void validateExe(String program, File exe) throws IOException { - if (!Shell.WINDOWS) { - verifyWindowsExe(program, exe); - } else { - verifyUnixExe(program, exe); - } - } - - /** * Write bytes to a file * @param outfile output file * @param data data to write - * @param createParent flag to indicate that the parent dir should - * be created * @throws IOException on any IO problem */ - public static void write(File outfile, byte[] data, boolean createParent) + public static void write(File outfile, byte[] data) throws IOException { File parentDir = outfile.getCanonicalFile().getParentFile(); if (parentDir == null) { throw new IOException(outfile.getPath() + " has no parent dir"); } - if (createParent) { - parentDir.mkdirs(); + if (!parentDir.exists()) { + if(!parentDir.mkdirs()) { + throw new IOException("Failed to create parent directory " + parentDir); + } } SliderUtils.verifyIsDir(parentDir, log); try(FileOutputStream out = new FileOutputStream(outfile)) { out.write(data); } - } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java index f0686af..f14a07a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java @@ -142,7 +142,6 @@ public class InstanceBuilder { md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time)); md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time)); - MapOperations globalOptions = internalOps.getGlobalOptions(); BuildHelper.addBuildMetadata(md, "create"); SliderUtils.setInfoTime(md, StatusKeys.INFO_CREATE_TIME_HUMAN, http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java index 18c3156..a272420 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java @@ -163,7 +163,7 @@ public final class AggregateConf { public String getPassphrase() { if (passphrase == null) { passphrase = RandomStringUtils.randomAlphanumeric( - Integer.valueOf(SliderKeys.PASS_LEN)); + Integer.parseInt(SliderKeys.PASS_LEN)); } return passphrase; http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java index be7c56f..74b6abb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java @@ -79,17 +79,6 @@ public final class ConfTree { public Map<String, Map<String, String>> components = new HashMap<>(INITAL_MAP_CAPACITY); - - /** - * Shallow clone - * @return a shallow clone - * @throws CloneNotSupportedException - */ - @Override - public Object clone() throws CloneNotSupportedException { - return super.clone(); - } - @Override public String toString() { try { @@ -105,9 +94,7 @@ public final class ConfTree { * @return a JSON string description * @throws IOException Problems mapping/writing the object */ - public String toJson() throws IOException, - JsonGenerationException, - JsonMappingException { + public String toJson() throws IOException { return ConfTreeSerDeser.toString(this); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java index 7fb3158..9eb7d5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java @@ -147,7 +147,9 @@ public class AppDefinitionPersister { File tempDir = Files.createTempDir(); File pkgSrcDir = new File(tempDir, "default"); - pkgSrcDir.mkdirs(); + if (!pkgSrcDir.exists() && !pkgSrcDir.mkdirs()) { + throw new IOException("Failed to create directory: " + pkgSrcDir); + } File destMetaInfo = new File(pkgSrcDir, "metainfo.json"); if (isFileUsed) { if (buildInfo.appMetaInfo.getName().endsWith(".xml")) { @@ -194,12 +196,13 @@ public class AppDefinitionPersister { List<String> addons = new ArrayList<String>(); Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap(); - for (String key : addonMap.keySet()) { - File defPath = new File(addonMap.get(key)); - if (SliderUtils.isUnset(addonMap.get(key))) { + for (Map.Entry<String, String > entry : addonMap.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (SliderUtils.isUnset(value)) { throw new BadConfigException("Invalid path for addon package " + key); } - + File defPath = new File(value); if (!defPath.exists()) { throw new BadConfigException("addon folder or package path is not valid."); } @@ -234,7 +237,7 @@ public class AppDefinitionPersister { } // Helper class to hold details for the app and addon packages - public class AppDefinition { + static class AppDefinition { // The target folder where the package will be stored public Path targetFolderInFs; // The on disk location of the app def package or folder http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java index 40fa217..e204178 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java @@ -62,61 +62,10 @@ public class AMWebClient { restClient = new BaseRestClient(binding.createJerseyClient()); } - - - private static URLConnectionClientHandler getUrlConnectionClientHandler() { - return new URLConnectionClientHandler(new HttpURLConnectionFactory() { - @Override - public HttpURLConnection getHttpURLConnection(URL url) - throws IOException { - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) { - // is a redirect - are we changing schemes? - String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION); - String originalScheme = url.getProtocol(); - String redirectScheme = URI.create(redirectLocation).getScheme(); - if (!originalScheme.equals(redirectScheme)) { - // need to fake it out by doing redirect ourselves - log.info("Protocol change during redirect. Redirecting {} to URL {}", - url, redirectLocation); - URL redirectURL = new URL(redirectLocation); - connection = (HttpURLConnection) redirectURL.openConnection(); - } - } - if (connection instanceof HttpsURLConnection) { - log.debug("Attempting to configure HTTPS connection using client " - + "configuration"); - final SSLFactory factory; - final SSLSocketFactory sf; - final HostnameVerifier hv; - - try { - HttpsURLConnection c = (HttpsURLConnection) connection; - factory = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration()); - factory.init(); - sf = factory.createSSLSocketFactory(); - hv = factory.getHostnameVerifier(); - c.setSSLSocketFactory(sf); - c.setHostnameVerifier(hv); - } catch (Exception e) { - log.info("Unable to configure HTTPS connection from " - + "configuration. Using JDK properties."); - } - - } - return connection; - } - }); - } - public WebResource resource(String url) { return restClient.resource(url); } - public BaseRestClient getRestClient() { - return restClient; - } - /** * Execute the operation. Failures are raised as IOException subclasses * @param method method to execute http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java deleted file mode 100644 index 0266223..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.restclient; - -import java.util.List; -import java.util.Map; - -/** - * A response for use as a return value from operations - */ -public class HttpOperationResponse { - - public int responseCode; - public long lastModified; - public String contentType; - public byte[] data; - public Map<String, List<String>> headers; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/af649d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java index 20ef198..46f0d02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java @@ -87,124 +87,4 @@ public class UrlConnectionOperations extends Configured { Preconditions.checkArgument(url.getPort() != 0, "no port"); return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego); } - - public HttpOperationResponse execGet(URL url) throws - IOException, - AuthenticationException { - return execHttpOperation(HttpVerb.GET, url, null, ""); - } - - public HttpOperationResponse execHttpOperation(HttpVerb verb, - URL url, - byte[] payload, - String contentType) - throws IOException, AuthenticationException { - HttpURLConnection conn = null; - HttpOperationResponse outcome = new HttpOperationResponse(); - int resultCode; - byte[] body = null; - log.debug("{} {} spnego={}", verb, url, useSpnego); - - boolean doOutput = verb.hasUploadBody(); - if (doOutput) { - Preconditions.checkArgument(payload !=null, - "Null payload on a verb which expects one"); - } - try { - conn = openConnection(url); - conn.setRequestMethod(verb.getVerb()); - conn.setDoOutput(doOutput); - if (doOutput) { - conn.setRequestProperty("Content-Type", contentType); - } - - // now do the connection - conn.connect(); - - if (doOutput) { - OutputStream output = conn.getOutputStream(); - IOUtils.write(payload, output); - output.close(); - } - - resultCode = conn.getResponseCode(); - outcome.lastModified = conn.getLastModified(); - outcome.contentType = conn.getContentType(); - outcome.headers = conn.getHeaderFields(); - InputStream stream = conn.getErrorStream(); - if (stream == null) { - stream = conn.getInputStream(); - } - if (stream != null) { - // read into a buffer. - body = IOUtils.toByteArray(stream); - } else { - // no body: - log.debug("No body in response"); - - } - } catch (SSLException e) { - throw e; - } catch (IOException e) { - throw NetUtils.wrapException(url.toString(), - url.getPort(), "localhost", 0, e); - - } catch (AuthenticationException e) { - throw new AuthenticationException("From " + url + ": " + e, e); - - } finally { - if (conn != null) { - conn.disconnect(); - } - } - uprateFaults(HttpVerb.GET, url.toString(), resultCode, "", body); - outcome.responseCode = resultCode; - outcome.data = body; - return outcome; - } - - /** - * Uprate error codes 400 and up into faults; - * 404 is converted to a {@link NotFoundException}, - * 401 to {@link ForbiddenException} - * - * @param verb HTTP Verb used - * @param url URL as string - * @param resultCode response from the request - * @param bodyAsString - *@param body optional body of the request @throws IOException if the result was considered a failure - */ - public static void uprateFaults(HttpVerb verb, String url, - int resultCode, String bodyAsString, byte[] body) - throws IOException { - - if (resultCode < 400) { - //success - return; - } - String msg = verb.toString() +" "+ url; - if (resultCode == 404) { - throw new NotFoundException(msg); - } - if (resultCode == 401) { - throw new ForbiddenException(msg); - } - // all other error codes - - // get a string respnse - if (bodyAsString == null) { - if (body != null && body.length > 0) { - bodyAsString = new String(body); - } else { - bodyAsString = ""; - } - } - String message = msg + - " failed with exit code " + resultCode - + ", body length " + bodyAsString.length() - + ":\n" + bodyAsString; - log.error(message); - throw new IOException(message); - } - } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
