http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java deleted file mode 100644 index 3e020cf..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.tarball; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.util.RestApiErrorMessages; - -import java.io.IOException; -import java.nio.file.Paths; - -public class TarballClientProvider extends AbstractClientProvider - implements SliderKeys { - - public TarballClientProvider() { - } - - @Override - public void validateArtifact(Artifact artifact, FileSystem fs) - throws IOException { - if (artifact == null) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_INVALID); - } - if (StringUtils.isEmpty(artifact.getId())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); - } - Path p = new Path(artifact.getId()); - if (!fs.exists(p)) { - throw new IllegalArgumentException( "Artifact tarball does not exist " - + artifact.getId()); - } - } - - @Override - protected void validateConfigFile(ConfigFile configFile, FileSystem - fileSystem) throws IOException { - // validate dest_file is not absolute - if (Paths.get(configFile.getDestFile()).isAbsolute()) { - throw new IllegalArgumentException( - "Dest_file must not be absolute path: " + configFile.getDestFile()); - } - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java deleted file mode 100644 index d9b7450..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.tarball; - -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.SliderProviderFactory; - -public class TarballProviderFactory extends SliderProviderFactory { - private static final SliderProviderFactory FACTORY = new - TarballProviderFactory(); - - private TarballProviderFactory() { - } - - private static class Client { - static final AbstractClientProvider PROVIDER = new TarballClientProvider(); - } - - private static class Server { - static final ProviderService PROVIDER = new TarballProviderService(); - } - - @Override - public AbstractClientProvider createClientProvider() { - return Client.PROVIDER; - } - - @Override - public ProviderService createServerProvider() { - return Server.PROVIDER; - } - - public static SliderProviderFactory getInstance() { - return FACTORY; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java deleted file mode 100644 index 8be63fd..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.tarball; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.providers.AbstractProviderService; -import org.apache.slider.server.appmaster.state.RoleInstance; - -import java.io.IOException; - -public class TarballProviderService extends AbstractProviderService { - - protected TarballProviderService() { - super(TarballProviderService.class.getSimpleName()); - } - - @Override - public void processArtifact(ContainerLauncher launcher, Application - application, RoleInstance roleInstance, SliderFileSystem fileSystem) - throws IOException { - Path artifact = new Path(roleInstance.providerRole.component - .getArtifact().getId()); - if (!fileSystem.isFile(artifact)) { - throw new IOException("Package doesn't exist as a resource: " + - artifact.toString()); - } - log.info("Adding resource {}", artifact.toString()); - LocalResourceType type = LocalResourceType.ARCHIVE; - LocalResource packageResource = fileSystem.createAmResource( - artifact, type); - launcher.addLocalResource(APP_LIB_DIR, packageResource); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java index f1b07f0..a03d4ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -23,27 +23,25 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.SliderProviderFactory; -import org.apache.slider.server.appmaster.actions.ActionStartContainer; +import org.apache.hadoop.yarn.service.provider.ProviderService; +import org.apache.hadoop.yarn.service.provider.ProviderFactory; import org.apache.slider.server.appmaster.actions.QueueAccess; +import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.services.workflow.ServiceThreadFactory; import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import static org.apache.slider.common.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY; +import static org.apache.hadoop.yarn.service.conf.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY; /** * A service for launching containers @@ -59,15 +57,15 @@ public class RoleLaunchService /** * Queue submission API */ - private final QueueAccess actionQueue; + private QueueAccess actionQueue; /** * Filesystem to use for the launch */ - private final SliderFileSystem fs; + private SliderFileSystem fs; - private Map<String, String> envVars; + private Map<String, String> envVars = new HashMap<>(); /** * Construct an instance of the launcher @@ -83,6 +81,11 @@ public class RoleLaunchService this.envVars = envVars; } + public RoleLaunchService(SliderFileSystem fs) { + super(ROLE_LAUNCH_SERVICE); + this.fs = fs; + } + @Override public void init(Configuration conf) { super.init(conf); @@ -97,9 +100,13 @@ public class RoleLaunchService */ public void launchRole(ContainerAssignment assignment, Application application, Credentials credentials) { + } + + public void launchComponent(Application application, + ComponentInstance instance, Container container) { RoleLaunchService.RoleLauncher launcher = - new RoleLaunchService.RoleLauncher(assignment, application, - credentials); + new RoleLaunchService.RoleLauncher(application, instance, + container); execute(launcher); } @@ -107,67 +114,34 @@ public class RoleLaunchService * Thread that runs on the AM to launch a container */ private class RoleLauncher implements Runnable { - - private final ContainerAssignment assignment; // Allocated container public final Container container; public final Application application; - public final ProviderRole role; - private final Credentials credentials; + public ComponentInstance instance; - public RoleLauncher(ContainerAssignment assignment, + public RoleLauncher( Application application, - Credentials credentials) { - this.assignment = assignment; - this.credentials = credentials; - this.container = assignment.container; - RoleStatus roleStatus = assignment.role; - ProviderRole providerRole = roleStatus.getProviderRole(); - this.role = providerRole; + ComponentInstance instance, Container container) { + this.container = container; this.application = application; - - } - - @Override - public String toString() { - return "RoleLauncher{" + - "container=" + container.getId() + - ", containerRole='" + role.name + '\'' + - '}'; + this.instance = instance; } @Override public void run() { try { ContainerLauncher containerLauncher = - new ContainerLauncher(getConfig(), fs, container, credentials); - containerLauncher.setupUGI(); + new ContainerLauncher(null, fs, container, null); containerLauncher.putEnv(envVars); - RoleInstance failedInstance = role.failedInstances.poll(); - RoleInstance instance; - if (failedInstance != null) { - instance = new RoleInstance(container, failedInstance); - } else { - instance = new RoleInstance(container, role); - } - String[] envDescription = containerLauncher.dumpEnvToString(); - String commandsAsString = containerLauncher.getCommandsAsString(); - log.info("Launching container {} for component instance = {}", - container.getId(), instance.getCompInstanceName()); - log.info("Starting container with command: {}", commandsAsString); - instance.command = commandsAsString; - instance.role = role.name; - instance.roleId = role.id; - instance.environment = envDescription; - - ProviderService provider = SliderProviderFactory.getProviderService( - role.component.getArtifact()); + Component compSpec = instance.getCompSpec(); + ProviderService provider = ProviderFactory.getProviderService( + compSpec.getArtifact()); provider.buildContainerLaunchContext(containerLauncher, application, - container, role, fs, instance); + instance, fs); - long delay = role.component.getConfiguration() - .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0); + long delay = compSpec.getConfiguration() + .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0); long maxDelay = getConfig() .getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); @@ -177,14 +151,15 @@ public class RoleLaunchService delay, maxDelay/1000); delay = 0; } - log.info("Container launch delay for {} set to {} seconds", role.name, - delay); - actionQueue.schedule( - new ActionStartContainer("starting " + role.name, container, - containerLauncher.completeContainerLaunch(), instance, delay, - TimeUnit.SECONDS)); + if (delay > 0) { + Thread.sleep(delay * 1000); + } + instance.getComponent().getScheduler().getNmClient() + .startContainerAsync(container, + containerLauncher.completeContainerLaunch()); } catch (Exception e) { - log.error("Exception thrown while trying to start " + role.name + log.error("Exception thrown while trying to start " + instance + .getCompInstanceName() + " container = " + container.getId() + " on host " + container .getNodeId(), e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index e565849..06dde67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; +import org.apache.hadoop.yarn.service.provider.ProviderService; +import org.apache.hadoop.yarn.service.provider.ProviderFactory; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.WebApps; @@ -87,12 +89,12 @@ import org.apache.slider.api.proto.Messages; import org.apache.slider.api.proto.SliderClusterAPI; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.params.AbstractActionArgs; -import org.apache.slider.common.params.SliderAMArgs; -import org.apache.slider.common.params.SliderAMCreateAction; -import org.apache.slider.common.params.SliderActions; +import org.apache.hadoop.yarn.service.conf.SliderExitCodes; +import org.apache.hadoop.yarn.service.conf.SliderKeys; +import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs; +import org.apache.hadoop.yarn.service.client.params.SliderAMArgs; +import org.apache.hadoop.yarn.service.client.params.SliderAMCreateAction; +import org.apache.hadoop.yarn.service.client.params.SliderActions; import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.common.tools.PortScanner; import org.apache.slider.common.tools.SliderFileSystem; @@ -109,8 +111,6 @@ import org.apache.slider.core.main.RunService; import org.apache.slider.core.main.ServiceLauncher; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.ProviderCompleted; -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.SliderProviderFactory; import org.apache.slider.server.appmaster.actions.ActionHalt; import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance; import org.apache.slider.server.appmaster.actions.ActionStopSlider; @@ -142,8 +142,8 @@ import org.apache.slider.server.appmaster.state.ContainerAssignment; import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; -import org.apache.slider.server.appmaster.timelineservice.SliderMetricsSink; +import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; +import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink; import org.apache.slider.server.appmaster.web.SliderAMWebApp; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.WebAppApiImpl; @@ -157,7 +157,7 @@ import org.apache.slider.server.services.workflow.ServiceThreadFactory; import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.apache.slider.server.services.workflow.WorkflowRpcService; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; -import org.apache.slider.util.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -584,7 +584,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * @throws Throwable on a failure */ private int createAndRunCluster(String appName) throws Throwable { - Path appDir = new Path((serviceArgs.getAppDefDir())); + Path appDir = new Path((serviceArgs.getAppDefPath())); SliderFileSystem fs = getClusterFS(); fs.setAppDir(appDir); application = ServiceApiUtil.loadApplication(fs, appName); @@ -597,11 +597,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // initialize our providers for (Component component : application.getComponents()) { - SliderProviderFactory factory = SliderProviderFactory + ProviderFactory factory = ProviderFactory .createSliderProviderFactory(component.getArtifact()); ProviderService providerService = factory.createServerProvider(); // init the provider BUT DO NOT START IT YET - initAndAddService(providerService); +// initAndAddService(providerService); providers.add(providerService); } @@ -663,7 +663,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService serviceTimelinePublisher.start(); for (ProviderService providerService : providers) { - providerService.setServiceTimelinePublisher(serviceTimelinePublisher); } appState.setServiceTimelinePublisher(serviceTimelinePublisher); log.info("ServiceTimelinePublisher started."); @@ -798,10 +797,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService liveContainers = amRegistrationData.getContainersFromPreviousAttempts(); DefaultMetricsSystem.initialize("SliderAppMaster"); if (timelineServiceEnabled) { - DefaultMetricsSystem.instance().register("SliderMetricsSink", + DefaultMetricsSystem.instance().register("ServiceMetricsSink", "For processing metrics to ATS", - new SliderMetricsSink(serviceTimelinePublisher)); - log.info("SliderMetricsSink registered."); + new ServiceMetricsSink(serviceTimelinePublisher)); + log.info("ServiceMetricsSink registered."); } //determine the location for the role history data @@ -809,7 +808,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //build the instance AppStateBindingInfo binding = new AppStateBindingInfo(); - binding.serviceConfig = serviceConf; + binding.serviceConfig = null; binding.fs = fs.getFileSystem(); binding.historyPath = historyDir; binding.liveContainers = liveContainers; @@ -847,7 +846,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //Give the provider access to the state, and AM for (ProviderService providerService : providers) { - providerService.setAMState(stateForProviders); +// providerService.setAMState(stateForProviders); } // chaos monkey @@ -1115,7 +1114,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService instanceName, appAttemptID); for (ProviderService providerService : providers) { - providerService.bindToYarnRegistry(yarnRegistryOperations); +// providerService.bindToYarnRegistry(yarnRegistryOperations); } // Yarn registry @@ -1150,7 +1149,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService true); } if (timelineServiceEnabled) { - serviceTimelinePublisher.serviceAttemptRegistered(appState); + serviceTimelinePublisher.serviceAttemptRegistered(application); } } @@ -1185,15 +1184,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService new org.apache.slider.api.resource.Container(); container.setId(id.toString()); container.setLaunchTime(new Date()); - container.setState(org.apache.slider.api.resource.ContainerState.INIT); + container.setState(org.apache.slider.api.resource.ContainerState.RUNNING_BUT_UNREADY); container.setBareHost(instance.host); // TODO differentiate component name and component instance name ? container.setComponentName(roleInstance.getCompInstanceName()); instance.providerRole.component.addContainer(container); if (timelineServiceEnabled) { - serviceTimelinePublisher.componentInstanceStarted(container, - instance.providerRole.component.getName()); + serviceTimelinePublisher.componentInstanceStarted(container, null); } return true; } @@ -1226,11 +1224,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService return; } String cid = RegistryPathUtils.encodeYarnID(containerId.toString()); - try { - yarnRegistryOperations.deleteComponent(cid); - } catch (IOException e) { - log.warn("Failed to delete container {} : {}", containerId, e, e); - } +// try { +// yarnRegistryOperations.deleteComponent(cid); +// } catch (IOException e) { +// log.warn("Failed to delete container {} : {}", containerId, e, e); +// } // remove component instance dir try { @@ -1879,7 +1877,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService throws IOException, SliderException { // didn't start, so don't register for (ProviderService providerService : providers) { - providerService.start(); +// providerService.start(); } // and send the started event ourselves eventCallbackEvent(null); @@ -1902,27 +1900,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } } - /** - * report container loss. If this isn't already known about, react - * - * @param containerId id of the container which has failed - * @throws SliderException - */ - public synchronized void providerLostContainer( - ContainerId containerId) - throws SliderException { - log.info("containerLostContactWithProvider: container {} lost", - containerId); - RoleInstance activeContainer = appState.getOwnedContainer(containerId); - if (activeContainer != null) { - execute(appState.releaseContainer(containerId)); - // ask for more containers if needed - log.info("Container released; triggering review"); - reviewRequestAndReleaseNodes("Loss of container"); - } else { - log.info("Container not in active set - ignoring"); - } - } /** * Async start container request @@ -1985,30 +1962,29 @@ public class SliderAppMaster extends AbstractSliderLaunchedService LOG_YARN.error("Owned container not found for {}", containerId); return; } - ProviderService providerService = SliderProviderFactory + ProviderService providerService = ProviderFactory .getProviderService(cinfo.providerRole.component.getArtifact()); - if (providerService.processContainerStatus(containerId, containerStatus)) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - LOG_YARN.info("Re-requesting status for role {}, {}", - cinfo.role, containerId); - //trigger another async container status - nmClientAsync.getContainerStatusAsync(containerId, - cinfo.container.getNodeId()); - } else if (timelineServiceEnabled) { - RoleInstance instance = appState.getOwnedContainer(containerId); - if (instance != null) { - org.apache.slider.api.resource.Container container = - instance.providerRole.component - .getContainer(containerId.toString()); - if (container != null) { - serviceTimelinePublisher.componentInstanceUpdated(container, - instance.providerRole.component.getName()); - } - } - } +// if (providerService.processContainerStatus(containerId, containerStatus)) { +// try { +// Thread.sleep(1000); +// } catch (InterruptedException e) { +// } +// LOG_YARN.info("Re-requesting status for role {}, {}", +// cinfo.role, containerId); +// //trigger another async container status +// nmClientAsync.getContainerStatusAsync(containerId, +// cinfo.container.getNodeId()); +// } else if (timelineServiceEnabled) { +// RoleInstance instance = appState.getOwnedContainer(containerId); +// if (instance != null) { +// org.apache.slider.api.resource.Container container = +// instance.providerRole.component +// .getContainer(containerId.toString()); +// if (container != null) { +// serviceTimelinePublisher.componentInstanceUpdated(container); +// } +// } +// } } @Override // NMClientAsync.CallbackHandler http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java deleted file mode 100644 index 41fe494..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.actions; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.slider.server.appmaster.SliderAppMaster; -import org.apache.slider.server.appmaster.state.AppState; - -import java.util.concurrent.TimeUnit; - -/** - * Report container loss to the AM - * {@link SliderAppMaster#providerLostContainer(ContainerId)} - */ -public class ProviderReportedContainerLoss extends AsyncAction { - - private final ContainerId containerId; - - public ProviderReportedContainerLoss(ContainerId containerId) { - this("lost container", 0, TimeUnit.MILLISECONDS, containerId); - } - - public ProviderReportedContainerLoss(String name, - long delay, - TimeUnit timeUnit, - ContainerId containerId) { - super(name, delay, timeUnit); - this.containerId = containerId; - } - - @Override - public void execute(SliderAppMaster appMaster, - QueueAccess queueService, - AppState appState) throws Exception { - appMaster.providerLostContainer(containerId); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java deleted file mode 100644 index cf607a0..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.metrics; - -import com.codahale.metrics.Counter; -import org.apache.hadoop.metrics2.MetricsCollector; -import org.apache.hadoop.metrics2.MetricsInfo; -import org.apache.hadoop.metrics2.MetricsSource; -import org.apache.hadoop.metrics2.MetricsSystem; -import org.apache.hadoop.metrics2.annotation.Metric; -import org.apache.hadoop.metrics2.annotation.Metrics; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.metrics2.lib.MetricsRegistry; -import org.apache.hadoop.metrics2.lib.MutableGaugeInt; - -import java.util.HashMap; -import java.util.Map; - -import static org.apache.hadoop.metrics2.lib.Interns.info; - -@Metrics(context = "yarn-native-service") -public class SliderMetrics implements MetricsSource { - - @Metric("containers requested") - public MutableGaugeInt containersRequested; - - @Metric("anti-affinity containers pending") - public MutableGaugeInt pendingAAContainers; - - @Metric("containers running") - public MutableGaugeInt containersRunning; - - @Metric("containers desired") - public MutableGaugeInt containersDesired; - - @Metric("containers completed") - public MutableGaugeInt containersCompleted; - - @Metric("containers failed") - public MutableGaugeInt containersFailed; - - @Metric("containers failed since last threshold") - public MutableGaugeInt failedSinceLastThreshold; - - @Metric("containers preempted") - public MutableGaugeInt containersPreempted; - - @Metric("containers exceeded limits") - public MutableGaugeInt containersLimitsExceeded; - - @Metric("containers surplus") - public MutableGaugeInt surplusContainers; - - @Metric("containers failed due to disk failure") - public MutableGaugeInt containersDiskFailure; - - protected final MetricsRegistry registry; - - public SliderMetrics(MetricsInfo metricsInfo) { - registry = new MetricsRegistry(metricsInfo); - } - - @Override - public void getMetrics(MetricsCollector collector, boolean all) { - registry.snapshot(collector.addRecord(registry.info()), all); - } - - public static SliderMetrics register(String name, String description) { - SliderMetrics metrics = new SliderMetrics(info(name, description)); - DefaultMetricsSystem.instance().register(name, description, metrics); - return metrics; - } - - public void tag(String name, String description, String value) { - registry.tag(name, description, value); - } - - @Override public String toString() { - return "SliderMetrics{" - + "containersRequested=" + containersRequested.value() - + ", pendingAAContainers=" + pendingAAContainers.value() - + ", containersRunning=" + containersRunning.value() - + ", containersDesired=" + containersDesired.value() - + ", containersCompleted=" + containersCompleted.value() - + ", containersFailed=" + containersFailed.value() - + ", failedSinceLastThreshold=" + failedSinceLastThreshold.value() - + ", containersPreempted=" + containersPreempted.value() - + ", surplusContainers=" + surplusContainers.value() + '}'; - } -} - http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java index 1406fbe..022312c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java @@ -19,7 +19,7 @@ package org.apache.slider.server.appmaster.monkey; import com.google.common.base.Preconditions; -import org.apache.slider.common.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderKeys; import org.apache.slider.server.appmaster.actions.ActionKillContainer; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.operations.RMOperationHandler; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java index dd4785d..2df9472 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java @@ -42,16 +42,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.slider.api.SliderClusterProtocol; -import org.apache.slider.common.SliderExitCodes; +import org.apache.hadoop.yarn.service.conf.SliderExitCodes; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.ErrorStrings; -import org.apache.slider.core.exceptions.ServiceNotReadyException; import org.apache.slider.core.exceptions.SliderException; -import static org.apache.slider.common.SliderXmlConfKeys.*; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java index a40078a..37c0a70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java @@ -19,7 +19,7 @@ package org.apache.slider.server.appmaster.rpc; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.Service; -import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; /** * {@link PolicyProvider} for Slider protocols. http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java index 4fd4910..5b127b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java @@ -25,7 +25,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.TokenInfo; import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector; -import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; import java.lang.annotation.Annotation; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java index cc19eee..75eccd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java @@ -23,8 +23,8 @@ import org.apache.hadoop.security.UserGroupInformation; import static org.apache.slider.core.main.LauncherExitCodes.EXIT_UNAUTHORIZED; import org.apache.slider.api.resource.Application; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.conf.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.SliderException; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 52d67f8..ba923bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -53,8 +53,8 @@ import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.RoleStatistics; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderExitCodes; +import org.apache.hadoop.yarn.service.conf.SliderKeys; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadConfigException; @@ -67,13 +67,13 @@ import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.management.MetricsConstants; -import org.apache.slider.server.appmaster.metrics.SliderMetrics; +import org.apache.hadoop.yarn.service.metrics.ServiceMetrics; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation; -import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; -import org.apache.slider.util.ServiceApiUtil; +import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -216,7 +216,7 @@ public class AppState { private Resource minResource; private Resource maxResource; - private SliderMetrics appMetrics; + private ServiceMetrics appMetrics; private ServiceTimelinePublisher serviceTimelinePublisher; @@ -330,7 +330,7 @@ public class AppState { // set the cluster specification (once its dependency the client properties // is out the way this.app = binding.application; - appMetrics = SliderMetrics.register(app.getName(), + appMetrics = ServiceMetrics.register(app.getName(), "Metrics for service"); appMetrics.tag("type", "Metrics type [component or service]", "service"); appMetrics.tag("appId", "Application id for service", app.getId()); @@ -1013,8 +1013,8 @@ public class AppState { } private void incCompletedContainers(RoleStatus role) { - role.getComponentMetrics().containersCompleted.incr(); - appMetrics.containersCompleted.incr(); + role.getComponentMetrics().containersSucceeded.incr(); + appMetrics.containersSucceeded.incr(); } @VisibleForTesting @@ -1030,26 +1030,26 @@ public class AppState { role.getComponentMetrics().containersDiskFailure.incr(); role.getComponentMetrics().containersFailed.incr(); break; - case Failed: - appMetrics.failedSinceLastThreshold.incr(); - appMetrics.containersFailed.incr(); - role.getComponentMetrics().failedSinceLastThreshold.incr(); - role.getComponentMetrics().containersFailed.incr(); - break; - case Failed_limits_exceeded: - appMetrics.containersLimitsExceeded.incr(); - appMetrics.failedSinceLastThreshold.incr(); - appMetrics.containersFailed.incr(); - role.getComponentMetrics().containersLimitsExceeded.incr(); - role.getComponentMetrics().failedSinceLastThreshold.incr(); - role.getComponentMetrics().containersFailed.incr(); - break; - default: - appMetrics.failedSinceLastThreshold.incr(); - appMetrics.containersFailed.incr(); - role.getComponentMetrics().failedSinceLastThreshold.incr(); - role.getComponentMetrics().containersFailed.incr(); - break; +// case Failed: +// appMetrics.failedSinceLastThreshold.incr(); +// appMetrics.containersFailed.incr(); +// role.getComponentMetrics().failedSinceLastThreshold.incr(); +// role.getComponentMetrics().containersFailed.incr(); +// break; +// case Failed_limits_exceeded: +// appMetrics.containersLimitsExceeded.incr(); +// appMetrics.failedSinceLastThreshold.incr(); +// appMetrics.containersFailed.incr(); +// role.getComponentMetrics().containersLimitsExceeded.incr(); +// role.getComponentMetrics().failedSinceLastThreshold.incr(); +// role.getComponentMetrics().containersFailed.incr(); +// break; +// default: +// appMetrics.failedSinceLastThreshold.incr(); +// appMetrics.containersFailed.incr(); +// role.getComponentMetrics().failedSinceLastThreshold.incr(); +// role.getComponentMetrics().containersFailed.incr(); +// break; } } @@ -1308,7 +1308,7 @@ public class AppState { log.info("decrementing role count for role {} to {}; completed={}", roleStatus.getName(), roleStatus.getComponentMetrics().containersRunning.value(), - roleStatus.getComponentMetrics().containersCompleted.value()); + roleStatus.getComponentMetrics().containersSucceeded.value()); result.outcome = ContainerOutcome.Completed; roleHistory.onReleaseCompleted(container); @@ -1671,7 +1671,7 @@ public class AppState { if (instance.providerRole.probe == null) { continue; } - boolean ready = instance.providerRole.probe.ping(instance).isSuccess(); + boolean ready = instance.providerRole.probe.ping(null).isSuccess(); if (ready) { if (instance.state != STATE_READY) { instance.state = STATE_READY; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java index ac9b8eb..439a256 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java @@ -25,13 +25,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.slider.api.resource.Application; -import org.apache.slider.common.tools.CoreFileSystem; import org.apache.slider.providers.ProviderRole; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * Binding information for application states; designed to be extensible http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index b6c3675..15333e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -31,7 +31,6 @@ import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.BoolMetric; -import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.management.Timestamp; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index 5619492..070f2f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.proto.Messages; -import org.apache.slider.api.resource.ConfigFile; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.providers.ProviderRole; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 3d9a8f3..ec0ff25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -27,7 +27,7 @@ import org.apache.slider.api.types.RoleStatistics; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.BoolMetricPredicate; -import org.apache.slider.server.appmaster.metrics.SliderMetrics; +import org.apache.hadoop.yarn.service.metrics.ServiceMetrics; import java.io.Serializable; import java.util.Comparator; @@ -54,7 +54,7 @@ public final class RoleStatus implements MetricSet { /** resource requirements */ private Resource resourceRequirements; - private SliderMetrics componentMetrics; + private ServiceMetrics componentMetrics; /** any pending AA request */ private volatile OutstandingRequest outstandingAArequest = null; @@ -67,12 +67,12 @@ public final class RoleStatus implements MetricSet { this.name = providerRole.name; this.key = providerRole.id; componentMetrics = - SliderMetrics.register(this.name, "Metrics for component " + this.name); + ServiceMetrics.register(this.name, "Metrics for component " + this.name); componentMetrics .tag("type", "Metrics type [component or service]", "component"); } - public SliderMetrics getComponentMetrics() { + public ServiceMetrics getComponentMetrics() { return this.componentMetrics; } @@ -235,14 +235,15 @@ public final class RoleStatus implements MetricSet { } public long getFailedRecently() { - return componentMetrics.failedSinceLastThreshold.value(); +// return componentMetrics.failedSinceLastThreshold.value(); + return 0; } public long resetFailedRecently() { - long count = - componentMetrics.failedSinceLastThreshold.value(); - componentMetrics.failedSinceLastThreshold.set(0); - return count; +// long count = +// componentMetrics.failedSinceLastThreshold.value(); +// componentMetrics.failedSinceLastThreshold.set(0); + return 0; } public long getFailed() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java deleted file mode 100644 index 7504140..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.timelineservice; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hadoop.metrics2.AbstractMetric; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; -import org.apache.hadoop.yarn.client.api.TimelineV2Client; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.api.resource.Configuration; -import org.apache.slider.api.resource.Container; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.server.appmaster.actions.ActionStopSlider; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A single service that publishes all the Timeline Entities. - */ -public class ServiceTimelinePublisher extends CompositeService { - - // Number of bytes of config which can be published in one shot to ATSv2. - public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024; - - private TimelineV2Client timelineClient; - - private volatile boolean stopped = false; - - private static final Logger log = - LoggerFactory.getLogger(ServiceTimelinePublisher.class); - - @Override - protected void serviceStop() throws Exception { - stopped = true; - } - - public boolean isStopped() { - return stopped; - } - - public ServiceTimelinePublisher(TimelineV2Client client) { - super(ServiceTimelinePublisher.class.getName()); - timelineClient = client; - } - - public void serviceAttemptRegistered(AppState appState) { - Application application = appState.getClusterStatus(); - long currentTimeMillis = application.getLaunchTime() == null - ? System.currentTimeMillis() : application.getLaunchTime().getTime(); - - TimelineEntity entity = createServiceAttemptEntity(application.getId()); - entity.setCreatedTime(currentTimeMillis); - - // create info keys - Map<String, Object> entityInfos = new HashMap<String, Object>(); - entityInfos.put(SliderTimelineMetricsConstants.NAME, application.getName()); - entityInfos.put(SliderTimelineMetricsConstants.STATE, - application.getState().toString()); - entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_TIME, - currentTimeMillis); - entity.addInfo(entityInfos); - - // add an event - TimelineEvent startEvent = new TimelineEvent(); - startEvent.setId(SliderTimelineEvent.SERVICE_ATTEMPT_REGISTERED.toString()); - startEvent.setTimestamp(currentTimeMillis); - entity.addEvent(startEvent); - - // publish before configurations published - putEntity(entity); - - // publish application specific configurations - publishConfigurations(application.getConfiguration(), application.getId(), - SliderTimelineEntityType.SERVICE_ATTEMPT.toString(), true); - - // publish component as separate entity. - publishComponents(application.getComponents()); - } - - public void serviceAttemptUpdated(Application application) { - TimelineEntity entity = createServiceAttemptEntity(application.getId()); - entity.addInfo(SliderTimelineMetricsConstants.QUICK_LINKS, - application.getQuicklinks()); - putEntity(entity); - } - - public void serviceAttemptUnregistered(AppState appState, - ActionStopSlider stopAction) { - long currentTimeMillis = System.currentTimeMillis(); - - TimelineEntity entity = - createServiceAttemptEntity(appState.getClusterStatus().getId()); - - // add info - Map<String, Object> entityInfos = new HashMap<String, Object>(); - entityInfos.put(SliderTimelineMetricsConstants.EXIT_STATUS_CODE, - stopAction.getExitCode()); - entityInfos.put(SliderTimelineMetricsConstants.STATE, - stopAction.getFinalApplicationStatus().toString()); - if (stopAction.getMessage() != null) { - entityInfos.put(SliderTimelineMetricsConstants.EXIT_REASON, - stopAction.getMessage()); - } - if (stopAction.getEx() != null) { - entityInfos.put(SliderTimelineMetricsConstants.DIAGNOSTICS_INFO, - stopAction.getEx().toString()); - } - entity.addInfo(entityInfos); - - // add an event - TimelineEvent startEvent = new TimelineEvent(); - startEvent - .setId(SliderTimelineEvent.SERVICE_ATTEMPT_UNREGISTERED.toString()); - startEvent.setTimestamp(currentTimeMillis); - entity.addEvent(startEvent); - - putEntity(entity); - } - - public void componentInstanceStarted(Container container, - String componentName) { - - TimelineEntity entity = createComponentInstanceEntity(container.getId()); - entity.setCreatedTime(container.getLaunchTime().getTime()); - - // create info keys - Map<String, Object> entityInfos = new HashMap<String, Object>(); - entityInfos.put(SliderTimelineMetricsConstants.BARE_HOST, - container.getBareHost()); - entityInfos.put(SliderTimelineMetricsConstants.STATE, - container.getState().toString()); - entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_TIME, - container.getLaunchTime().getTime()); - entityInfos.put(SliderTimelineMetricsConstants.COMPONENT_NAME, - componentName); - entity.addInfo(entityInfos); - - // add an event - TimelineEvent startEvent = new TimelineEvent(); - startEvent - .setId(SliderTimelineEvent.COMPONENT_INSTANCE_REGISTERED.toString()); - startEvent.setTimestamp(container.getLaunchTime().getTime()); - entity.addEvent(startEvent); - - putEntity(entity); - } - - public void componentInstanceFinished(RoleInstance instance) { - TimelineEntity entity = createComponentInstanceEntity(instance.id); - - // create info keys - Map<String, Object> entityInfos = new HashMap<String, Object>(); - entityInfos.put(SliderTimelineMetricsConstants.EXIT_STATUS_CODE, - instance.exitCode); - entityInfos.put(SliderTimelineMetricsConstants.DIAGNOSTICS_INFO, - instance.diagnostics); - // TODO need to change the state based on enum value. - entityInfos.put(SliderTimelineMetricsConstants.STATE, "FINISHED"); - entity.addInfo(entityInfos); - - // add an event - TimelineEvent startEvent = new TimelineEvent(); - startEvent - .setId(SliderTimelineEvent.COMPONENT_INSTANCE_UNREGISTERED.toString()); - startEvent.setTimestamp(System.currentTimeMillis()); - entity.addEvent(startEvent); - - putEntity(entity); - } - - public void componentInstanceUpdated(Container container, - String componentName) { - TimelineEntity entity = createComponentInstanceEntity(container.getId()); - - // create info keys - Map<String, Object> entityInfos = new HashMap<String, Object>(); - entityInfos.put(SliderTimelineMetricsConstants.IP, container.getIp()); - entityInfos.put(SliderTimelineMetricsConstants.HOSTNAME, - container.getHostname()); - entityInfos.put(SliderTimelineMetricsConstants.STATE, - container.getState().toString()); - entity.addInfo(entityInfos); - - TimelineEvent updateEvent = new TimelineEvent(); - updateEvent - .setId(SliderTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString()); - updateEvent.setTimestamp(System.currentTimeMillis()); - entity.addEvent(updateEvent); - - putEntity(entity); - } - - private void publishComponents(List<Component> components) { - long currentTimeMillis = System.currentTimeMillis(); - for (Component component : components) { - TimelineEntity entity = createComponentEntity(component.getName()); - entity.setCreatedTime(currentTimeMillis); - - // create info keys - Map<String, Object> entityInfos = new HashMap<String, Object>(); - entityInfos.put(SliderTimelineMetricsConstants.ARTIFACT_ID, - component.getArtifact().getId()); - entityInfos.put(SliderTimelineMetricsConstants.ARTIFACT_TYPE, - component.getArtifact().getType().toString()); - if (component.getResource().getProfile() != null) { - entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_PROFILE, - component.getResource().getProfile()); - } - entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_CPU, - component.getResource().getCpus()); - entityInfos.put(SliderTimelineMetricsConstants.RESOURCE_MEMORY, - component.getResource().getMemory()); - - if (component.getLaunchCommand() != null) { - entityInfos.put(SliderTimelineMetricsConstants.LAUNCH_COMMAND, - component.getLaunchCommand()); - } - entityInfos.put(SliderTimelineMetricsConstants.UNIQUE_COMPONENT_SUPPORT, - component.getUniqueComponentSupport().toString()); - entityInfos.put(SliderTimelineMetricsConstants.RUN_PRIVILEGED_CONTAINER, - component.getRunPrivilegedContainer().toString()); - if (component.getPlacementPolicy() != null) { - entityInfos.put(SliderTimelineMetricsConstants.PLACEMENT_POLICY, - component.getPlacementPolicy().getLabel()); - } - entity.addInfo(entityInfos); - - putEntity(entity); - - // publish component specific configurations - publishConfigurations(component.getConfiguration(), component.getName(), - SliderTimelineEntityType.COMPONENT.toString(), false); - } - } - - private void publishConfigurations(Configuration configuration, - String entityId, String entityType, boolean isServiceAttemptEntity) { - if (isServiceAttemptEntity) { - // publish slider-client.xml properties at service level - publishConfigurations(SliderUtils.loadSliderClientXML().iterator(), - entityId, entityType); - } - publishConfigurations(configuration.getProperties().entrySet().iterator(), - entityId, entityType); - - publishConfigurations(configuration.getEnv().entrySet().iterator(), - entityId, entityType); - - for (ConfigFile configFile : configuration.getFiles()) { - publishConfigurations(configFile.getProps().entrySet().iterator(), - entityId, entityType); - } - } - - private void publishConfigurations(Iterator<Entry<String, String>> iterator, - String entityId, String entityType) { - int configSize = 0; - TimelineEntity entity = createTimelineEntity(entityId, entityType); - while (iterator.hasNext()) { - Entry<String, String> entry = iterator.next(); - int size = entry.getKey().length() + entry.getValue().length(); - configSize += size; - // Configs are split into multiple entities if they exceed 100kb in size. - if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) { - if (entity.getConfigs().size() > 0) { - putEntity(entity); - entity = createTimelineEntity(entityId, entityType); - } - configSize = size; - } - entity.addConfig(entry.getKey(), entry.getValue()); - } - if (configSize > 0) { - putEntity(entity); - } - } - - /** - * Called from SliderMetricsSink at regular interval of time. - * @param metrics of service or components - * @param entityId Id of entity - * @param entityType Type of entity - * @param timestamp - */ - public void publishMetrics(Iterable<AbstractMetric> metrics, String entityId, - String entityType, long timestamp) { - TimelineEntity entity = createTimelineEntity(entityId, entityType); - Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>(); - for (AbstractMetric metric : metrics) { - TimelineMetric timelineMetric = new TimelineMetric(); - timelineMetric.setId(metric.name()); - timelineMetric.addValue(timestamp, metric.value()); - entityMetrics.add(timelineMetric); - } - entity.setMetrics(entityMetrics); - putEntity(entity); - } - - private TimelineEntity createServiceAttemptEntity(String serviceId) { - TimelineEntity entity = createTimelineEntity(serviceId, - SliderTimelineEntityType.SERVICE_ATTEMPT.toString()); - return entity; - } - - private TimelineEntity createComponentInstanceEntity(String instanceId) { - TimelineEntity entity = createTimelineEntity(instanceId, - SliderTimelineEntityType.COMPONENT_INSTANCE.toString()); - return entity; - } - - private TimelineEntity createComponentEntity(String componentId) { - TimelineEntity entity = createTimelineEntity(componentId, - SliderTimelineEntityType.COMPONENT.toString()); - return entity; - } - - private TimelineEntity createTimelineEntity(String entityId, - String entityType) { - TimelineEntity entity = new TimelineEntity(); - entity.setId(entityId); - entity.setType(entityType); - return entity; - } - - private void putEntity(TimelineEntity entity) { - try { - if (log.isDebugEnabled()) { - log.debug("Publishing the entity " + entity + ", JSON-style content: " - + TimelineUtils.dumpTimelineRecordtoJSON(entity)); - } - if (timelineClient != null) { - timelineClient.putEntitiesAsync(entity); - } else { - log.error("Seems like client has been removed before the entity " - + "could be published for " + entity); - } - } catch (Exception e) { - log.error("Error when publishing entity " + entity, e); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java deleted file mode 100644 index 869ae26..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderMetricsSink.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.timelineservice; - -import org.apache.commons.configuration2.SubsetConfiguration; -import org.apache.hadoop.metrics2.MetricsRecord; -import org.apache.hadoop.metrics2.MetricsSink; -import org.apache.hadoop.metrics2.MetricsTag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Write the metrics to a ATSv2. Generally, this class is instantiated via - * hadoop-metrics2 property files. Specifically, you would create this class by - * adding the following to by This would actually be set as: <code> - * [prefix].sink.[some instance name].class - * =org.apache.slider.server.appmaster.timelineservice.SliderMetricsSink - * </code>, where <tt>prefix</tt> is "atsv2": and <tt>some instance name</tt> is - * just any unique name, so properties can be differentiated if there are - * multiple sinks of the same type created - */ -public class SliderMetricsSink implements MetricsSink { - - private static final Logger log = - LoggerFactory.getLogger(SliderMetricsSink.class); - - private ServiceTimelinePublisher serviceTimelinePublisher; - - public SliderMetricsSink() { - - } - - public SliderMetricsSink(ServiceTimelinePublisher publisher) { - serviceTimelinePublisher = publisher; - } - - /** - * Publishes service and component metrics to ATS. - */ - @Override - public void putMetrics(MetricsRecord record) { - if (serviceTimelinePublisher.isStopped()) { - log.warn("ServiceTimelinePublisher has stopped. " - + "Not publishing any more metrics to ATS."); - return; - } - - boolean isServiceMetrics = false; - boolean isComponentMetrics = false; - String appId = null; - for (MetricsTag tag : record.tags()) { - if (tag.name().equals("type") && tag.value().equals("service")) { - isServiceMetrics = true; - } else if (tag.name().equals("type") && tag.value().equals("component")) { - isComponentMetrics = true; - break; // if component metrics, no more information required from tag so - // break the loop - } else if (tag.name().equals("appId")) { - appId = tag.value(); - } - } - - if (isServiceMetrics && appId != null) { - if (log.isDebugEnabled()) { - log.debug("Publishing service metrics. " + record); - } - serviceTimelinePublisher.publishMetrics(record.metrics(), appId, - SliderTimelineEntityType.SERVICE_ATTEMPT.toString(), - record.timestamp()); - } else if (isComponentMetrics) { - if (log.isDebugEnabled()) { - log.debug("Publishing Component metrics. " + record); - } - serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(), - SliderTimelineEntityType.COMPONENT.toString(), record.timestamp()); - } - } - - @Override - public void init(SubsetConfiguration conf) { - } - - @Override - public void flush() { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java deleted file mode 100644 index 908754f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEntityType.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.timelineservice; - -/** - * Slider entities that are published to ATS. - */ -public enum SliderTimelineEntityType { - /** - * Used for publishing service entity information. - */ - SERVICE_ATTEMPT, - - /** - * Used for publishing component entity information. - */ - COMPONENT, - - /** - * Used for publishing component instance entity information. - */ - COMPONENT_INSTANCE -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java deleted file mode 100644 index 04f0219..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineEvent.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.timelineservice; - -/** - * Events that are used to store in ATS. - */ -public enum SliderTimelineEvent { - SERVICE_ATTEMPT_REGISTERED, - - SERVICE_ATTEMPT_UNREGISTERED, - - COMPONENT_INSTANCE_REGISTERED, - - COMPONENT_INSTANCE_UNREGISTERED, - - COMPONENT_INSTANCE_UPDATED -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java deleted file mode 100644 index 58d77ce..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.timelineservice; - -/** - * Constants which are stored as key in ATS - */ -public final class SliderTimelineMetricsConstants { - - public static final String URI = "URI"; - - public static final String NAME = "NAME"; - - public static final String STATE = "STATE"; - - public static final String EXIT_STATUS_CODE = "EXIT_STATUS_CODE"; - - public static final String EXIT_REASON = "EXIT_REASON"; - - public static final String DIAGNOSTICS_INFO = "DIAGNOSTICS_INFO"; - - public static final String LAUNCH_TIME = "LAUNCH_TIME"; - - public static final String QUICK_LINKS = "QUICK_LINKS"; - - public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND"; - - public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS"; - - public static final String RUNNING_CONTAINERS = - "NUMBER_OF_RUNNING_CONTAINERS"; - - /** - * Artifacts constants. - */ - public static final String ARTIFACT_ID = "ARTIFACT_ID"; - - public static final String ARTIFACT_TYPE = "ARTIFACT_TYPE"; - - public static final String ARTIFACT_URI = "ARTIFACT_URI"; - - /** - * Resource constants. - */ - public static final String RESOURCE_CPU = "RESOURCE_CPU"; - - public static final String RESOURCE_MEMORY = "RESOURCE_MEMORY"; - - public static final String RESOURCE_PROFILE = "RESOURCE_PROFILE"; - - /** - * component instance constants. - */ - public static final String IP = "IP"; - - public static final String HOSTNAME = "HOSTNAME"; - - public static final String BARE_HOST = "BARE_HOST"; - - public static final String COMPONENT_NAME = "COMPONENT_NAME"; - - /** - * component constants. - */ - public static final String DEPENDENCIES = "DEPENDENCIES"; - - public static final String DESCRIPTION = "DESCRIPTION"; - - public static final String UNIQUE_COMPONENT_SUPPORT = - "UNIQUE_COMPONENT_SUPPORT"; - - public static final String RUN_PRIVILEGED_CONTAINER = - "RUN_PRIVILEGED_CONTAINER"; - - public static final String PLACEMENT_POLICY = "PLACEMENT_POLICY"; - -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org