Although it was not mentioned in my most recent correspondence, I was using the
shade plugin to build an uber jar for slider in my most recent attempts. After
seeing some hints at
http://stackoverflow.com/questions/8594707/jersey-exception-severe-a-message-body-reader-for-java-class,
I looked at that interaction and disabled the uber jar creation. I'm not sure
why the uberjars failed here. Based on the test, I think that the json
deserialization issue is resolved in the AM, but the jmemcache now has
unresolved runtime dependencies (even when building jmemcached as an uberjar
using the assembly plugin) and is failing.
The jmemcached container reports in errors-2.txt (errors-1.txt is empty):
017-04-29 07:06:10,960 - Error while executing command 'start':
Traceback (most recent call last):
File
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/filecache/71/slider-agent.tar.gz/slider-agent/resource_management/libraries/script/script.py",
line 116, in execute
method(env)
File
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/filecache/108/jmemcached-1.0.1.zip/package/scripts/memcached.py",
line 43, in start
poll_after = 5
File
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/filecache/71/slider-agent.tar.gz/slider-agent/resource_management/core/base.py",
line 148, in __init__
self.env.run()
File
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/filecache/71/slider-agent.tar.gz/slider-agent/resource_management/core/environment.py",
line 149, in run
self.run_action(resource, action)
File
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/filecache/71/slider-agent.tar.gz/slider-agent/resource_management/core/environment.py",
line 115, in run_action
provider_action()
File
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/filecache/71/slider-agent.tar.gz/slider-agent/resource_management/core/providers/system.py",
line 260, in action_run
raise ex
Fail: Execution of '/usr/lib/jvm/java//bin/java -Xmx256m -Xms128m -classpath
/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/container_1493430076245_0007_01_000002/app/install/*:
com.thimbleware.jmemcached.Main --memory=200M --port=11211' returned 1. Error:
A JNI error has occurred, please check your installation and try again
Exception in thread "main" java.lang.NoClassDefFoundError:
org/apache/commons/cli/CommandLineParser
at java.lang.Class.getDeclaredMethods0(Native Method)
at java.lang.Class.privateGetDeclaredMethods(Class.java:2701)
at java.lang.Class.privateGetMethodRecursive(Class.java:3048)
at java.lang.Class.getMethod0(Class.java:3018)
at java.lang.Class.getMethod(Class.java:1784)
at
sun.launcher.LauncherHelper.validateMainClass(LauncherHelper.java:544)
at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:526)
Caused by: java.lang.ClassNotFoundException:
org.apache.commons.cli.CommandLineParser
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 7 more
command-1.txt reads:
{
"clusterName": "jmemcached",
"componentName": "MEMCACHED",
"hostname": "quickstart.cloudera",
"hostLevelParams": {
"java_home": "/usr/lib/jvm/java/",
"container_id": "container_1493430076245_0007_01_000002",
"package_list":
"[{\"type\":\"tarball\",\"name\":\"files/jmemcached-1.0.1.tar\"}]"
},
"commandType": "EXECUTION_COMMAND",
"roleCommand": "INSTALL",
"serviceName": "jmemcached",
"role": "MEMCACHED",
"commandParams": {
"record_config": "false",
"service_package_folder":
"${AGENT_WORK_ROOT}/work/app/definition/package",
"script": "scripts/memcached.py",
"schema_version": "2.0",
"command_timeout": "600",
"script_type": "PYTHON"
},
"taskId": 1,
"public_hostname": "quickstart.cloudera",
"appmaster_hostname": "quickstart.cloudera",
"commandId": "1-1",
"configurations": {
"global": {
"security_enabled": "false",
"additional_cp": "",
"app_container_id": "container_1493430076245_0007_01_000002",
"listen_port": "11211",
"xms_val": "128m",
"app_root":
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/container_1493430076245_0007_01_000002/app/install",
"app_log_dir":
"/var/log/hadoop-yarn/containers/application_1493430076245_0007/container_1493430076245_0007_01_000002",
"xmx_val": "256m",
"pid_file":
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/container_1493430076245_0007_01_000002/app/run/component.pid",
"memory_val": "200M",
"app_container_tag": "1",
"app_pid_dir":
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/container_1493430076245_0007_01_000002/app/run",
"app_install_dir":
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/container_1493430076245_0007_01_000002/app/install",
"app_input_conf_dir":
"/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493430076245_0007/container_1493430076245_0007_01_000002/propagatedconf"
}
}
}
With best regards:
Bill
________________________________
From: Foolish Ewe <[email protected]>
Sent: Friday, April 28, 2017 6:56 PM
To: [email protected]
Subject: Re: Slider fails to launch the memcached container in my version of
the example, Registration Response gets a Error 415 Unsupported Media Type
Thanks for the quick response Billie. I've done a fresh run, I'm still seeing
the same issue.
I suspect that the json deserialization is not occurring, and that the
deserializer is not being found at run time.
For the current run, I've made a small change to the appConfig.json to remove
the agent.ini, but didn't change the --manager lines.
[root@quickstart mybuild]# cat
./incubator-slider/app-packages/memcached/appConfig.json
{
"schema": "http://example.org/specification/v2.0.0";,
"metadata": {
},
"global": {
"application.def": ".slider/package/jmemcached/jmemcached-1.0.1.zip",
"java_home": "/usr/lib/jvm/java/",
"site.global.additional_cp": "",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
"site.global.listen_port": "11211"
},
"components": {
"slider-appmaster": {
"jvm.heapsize": "256M"
}
}
}
The python version is:
[root@quickstart mybuild]# python -V
Python 2.7.5
So that we can isolate relevant data based on log time stamps, the
slider-agent.log shows the first failure at:
INFO 2017-04-28 18:45:48,109 Controller.py:138 - Registering with the server at
https://quickstart.cloudera:36837/ws/v1/slider/agents/container_1493405038155_0001_01_000002___MEMCACHED/register
with data '{"actualState": 0, "logFolders": {}, "agentVersion": "1",
"allocatedPorts": {}, "timestamp": 1493405148109, "expectedState": 0, "tags":
"", "responseId": -1, "publicHostname": "quickstart.cloudera", "label":
"container_1493405038155_0001_01_000002___MEMCACHED"}'
INFO 2017-04-28 18:45:48,109 security.py:89 - SSL Connect being called..
connecting to the server
INFO 2017-04-28 18:45:48,181 security.py:51 - SSL connection established.
Two-way SSL authentication is turned off on the server.
INFO 2017-04-28 18:45:48,194 Controller.py:140 - Registration response =
'<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;
charset=ISO-8859-1"/>\n<title>Error 415 Unsupported Media
Type</title>\n</head>\n<body><h2>HTTP ERROR 415</h2>\n<p>Problem accessing
/ws/v1/slider/agents/container_1493405038155_0001_01_000002___MEMCACHED/register.
Reason:\n<pre> Unsupported Media Type</pre></p><hr /><i><small>Powered by
Jetty://</small></i><br/>
\n<br/> \n<br/>
\n<br/>
\n<br/> \n<br/>
\n<br/>
\n<br/> \n<br/>
\n<br/>
\n<br/> \n<br/>
\n<br/>
\n<br/>
\n<br/> \n<br/>
\n<br/>
\n<br/> \n<br/>
\n<br/>
\n\n</body>\n</html>\n'
The AM logs have slider have slider-err.txt which has repeated reports
corresponding to the SEVERE message at the end:
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in
[jar:file:/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493405038155_0001/filecache/74/slider.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in
[jar:file:/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493405038155_0001/filecache/20/slider-core-0.60.0-incubating.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in
[jar:file:/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/appcache/application_1493405038155_0001/filecache/27/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J Error Codes<http://www.slf4j.org/codes.html#multiple_bindings>
www.slf4j.org
SLF4J warning or error messages and their meanings No SLF4J providers were
found. This warning, i.e. not an error, message is reported when no SLF4J ...
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
Apr 28, 2017 6:45:39 PM com.sun.jersey.api.core.PackagesResourceConfig init
INFO: Scanning for root resource and provider classes in the packages:
org.apache.slider.server.appmaster.web.rest.agent
Apr 28, 2017 6:45:41 PM com.sun.jersey.api.core.ScanningResourceConfig
logClasses
INFO: Root resource classes found:
class org.apache.slider.server.appmaster.web.rest.agent.AgentWebServices
Apr 28, 2017 6:45:41 PM com.sun.jersey.api.core.ScanningResourceConfig init
INFO: No provider classes found.
Apr 28, 2017 6:45:42 PM
com.sun.jersey.server.impl.application.WebApplicationImpl _initiate
INFO: Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM'
Apr 28, 2017 6:45:48 PM com.sun.jersey.spi.inject.Errors processErrorMessages
WARNING: The following warnings have been detected with resource and/or
provider classes:
WARNING: A sub-resource method, public javax.ws.rs.core.Response
org.apache.slider.server.appmaster.web.rest.agent.AgentResource.endpointRoot(),
with URI template, "/", is treated as a resource method
Apr 28, 2017 6:45:48 PM com.sun.jersey.spi.container.ContainerRequest getEntity
SEVERE: A message body reader for Java class
org.apache.slider.server.appmaster.web.rest.agent.Register, and Java type class
org.apache.slider.server.appmaster.web.rest.agent.Register, and MIME media type
application/json was not found.
The registered message body readers compatible with the MIME media type are:
*/* ->
com.sun.jersey.core.impl.provider.entity.FormProvider
com.sun.jersey.core.impl.provider.entity.StringProvider
com.sun.jersey.core.impl.provider.entity.ByteArrayProvider
com.sun.jersey.core.impl.provider.entity.FileProvider
com.sun.jersey.core.impl.provider.entity.InputStreamProvider
com.sun.jersey.core.impl.provider.entity.DataSourceProvider
com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider$General
com.sun.jersey.core.impl.provider.entity.ReaderProvider
com.sun.jersey.core.impl.provider.entity.DocumentProvider
com.sun.jersey.core.impl.provider.entity.SourceProvider$StreamSourceReader
com.sun.jersey.core.impl.provider.entity.SourceProvider$SAXSourceReader
com.sun.jersey.core.impl.provider.entity.SourceProvider$DOMSourceReader
com.sun.jersey.core.impl.provider.entity.XMLRootElementProvider$General
com.sun.jersey.core.impl.provider.entity.XMLListElementProvider$General
com.sun.jersey.core.impl.provider.entity.XMLRootObjectProvider$General
com.sun.jersey.core.impl.provider.entity.EntityHolderReader
The AM Logs for slider.log reports:
2017-04-28 18:45:45,398 [main] INFO agent.AgentClientProvider - Validating app
definition .slider/package/jmemcached/jmemcached-1.0.1.zip
2017-04-28 18:45:45,416 [AmExecutor-006] INFO state.AppState - Reviewing
RoleStatus{name='MEMCACHED', key=1, minimum=0, maximum=1, desired=1, actual=0,
requested=0, releasing=0, failed=0, started=0, startFailed=0, completed=0,
failureMessage=''} : expected 1
2017-04-28 18:45:45,417 [AmExecutor-006] INFO state.AppState - MEMCACHED:
Asking for 1 more nodes(s) for a total of 1
2017-04-28 18:45:45,422 [AmExecutor-006] INFO state.AppState - Container ask
is Capability[<memory:256, vCores:1>]Priority[1073741825] and label = null
2017-04-28 18:45:46,805 [AMRM Heartbeater thread] INFO impl.AMRMClientImpl -
Received new token for : quickstart.cloudera:42111
2017-04-28 18:45:46,806 [AMRM Callback Handler Thread] INFO
appmaster.SliderAppMaster - onContainersAllocated(1)
2017-04-28 18:45:46,808 [AMRM Callback Handler Thread] INFO state.AppState -
Assigning role MEMCACHED to container container_1493405038155_0001_01_000002,
on quickstart.cloudera:42111,
2017-04-28 18:45:46,811 [AMRM Callback Handler Thread] INFO
appmaster.SliderAppMaster - Diagnostics: RoleStatus{name='slider-appmaster',
key=0, minimum=0, maximum=1, desired=1, actual=1, requested=0, releasing=0,
failed=0, started=1, startFailed=0, completed=0, failureMessage=''}
RoleStatus{name='MEMCACHED', key=1, minimum=0, maximum=1, desired=1, actual=1,
requested=0, releasing=0, failed=0, started=0, startFailed=0, completed=0,
failureMessage=''}
2017-04-28 18:45:46,826 [RoleLaunchService-007] INFO
agent.AgentProviderService - Build launch context for Agent
2017-04-28 18:45:46,830 [RoleLaunchService-007] INFO
agent.AgentProviderService - AGENT_WORK_ROOT set to $PWD
2017-04-28 18:45:46,830 [RoleLaunchService-007] INFO
agent.AgentProviderService - AGENT_LOG_ROOT set to <LOG_DIR>
2017-04-28 18:45:46,830 [RoleLaunchService-007] INFO
agent.AgentProviderService - PYTHONPATH set to ./infra/agent/slider-agent/
2017-04-28 18:45:46,846 [RoleLaunchService-007] INFO
agent.AgentProviderService - Using ./infra/agent/slider-agent/agent/main.py for
agent.
2017-04-28 18:45:46,852 [RoleLaunchService-007] INFO
appmaster.RoleLaunchService - Starting container with command: python
./infra/agent/slider-agent/agent/main.py --label
container_1493405038155_0001_01_000002___MEMCACHED --zk-quorum localhost:2181
--zk-reg-path /registry/users/root/services/org-apache-slider/jmemcached >
<LOG_DIR>/slider-agent.out 2>&1 ;
2017-04-28 18:45:46,852 [RoleLaunchService-007] INFO
appmaster.RoleLaunchService - Container launch delay for MEMCACHED set to 0
seconds
2017-04-28 18:45:46,906 [AmExecutor-006] WARN appmaster.SliderAppMaster - No
delegation tokens obtained and set for launch context
2017-04-28 18:45:46,984
[org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl #0] INFO
impl.NMClientAsyncImpl - Processing Event EventType: START_CONTAINER for
Container container_1493405038155_0001_01_000002
2017-04-28 18:45:46,987
[org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl #0] INFO
impl.ContainerManagementProtocolProxy - Opening proxy :
quickstart.cloudera:42111
2017-04-28 18:45:47,066
[org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl #0] INFO
appmaster.SliderAppMaster - Started Container
container_1493405038155_0001_01_000002
2017-04-28 18:45:47,228
[org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl #0] INFO
appmaster.SliderAppMaster - Deployed instance of role MEMCACHED onto
container_1493405038155_0001_01_000002
2017-04-28 18:45:47,229 [AmExecutor-006] INFO appmaster.SliderAppMaster -
Registering component container_1493405038155_0001_01_000002
2017-04-28 18:45:47,232
[org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl #1] INFO
impl.NMClientAsyncImpl - Processing Event EventType: QUERY_CONTAINER for
Container container_1493405038155_0001_01_000002
2017-04-28 18:45:47,232
[org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl #1] INFO
impl.ContainerManagementProtocolProxy - Opening proxy :
quickstart.cloudera:42111
2017-04-28 18:45:47,243 [AmExecutor-006] INFO zk.RegistryOperationsService -
Bound at
/users/root/services/org-apache-slider/jmemcached/components/container-1493405038155-0001-01-000002
: ServiceRecord{description='MEMCACHED'; external endpoints: {}; internal
endpoints: {}, attributes: {"yarn:id"="container-1493405038155-0001-01-000002"
"yarn:persistence"="container" }}
2017-04-28 18:47:44,870 [Thread-37] WARN agent.HeartbeatMonitor - Component
ComponentInstanceState{containerIdAsString='container_1493405038155_0001_01_000002',
state=INIT, failuresSeen=0, lastHeartbeat=1493405146852,
containerState=UNHEALTHY, componentName='MEMCACHED'} marked UNHEALTHY. Last
heartbeat received at 1493405146852 approx. 118017 ms. back.
2017-04-28 18:48:44,872 [Thread-37] WARN agent.HeartbeatMonitor - Component
ComponentInstanceState{containerIdAsString='container_1493405038155_0001_01_000002',
state=INIT, failuresSeen=0, lastHeartbeat=1493405146852,
containerState=HEARTBEAT_LOST, componentName='MEMCACHED'} marked
HEARTBEAT_LOST. Last heartbeat received at 1493405146852 approx. 178020 ms.
back.
2017-04-28 18:48:44,873 [AmExecutor-006] INFO appmaster.SliderAppMaster -
containerLostContactWithProvider: container
container_1493405038155_0001_01_000002 lost
2017-04-28 18:48:44,875 [AmExecutor-006] INFO appmaster.SliderAppMaster -
Container released; triggering review
2017-04-28 18:48:44,875 [AmExecutor-006] INFO state.AppState - Reviewing
RoleStatus{name='MEMCACHED', key=1, minimum=0, maximum=1, desired=1, actual=1,
requested=0, releasing=1, failed=0, started=1, startFailed=0, completed=0,
failureMessage=''} : expected 1
2017-04-28 18:48:45,282 [AMRM Callback Handler Thread] INFO
appmaster.SliderAppMaster - onContainersCompleted([1]
2017-04-28 18:48:45,282 [AMRM Callback Handler Thread] INFO
appmaster.SliderAppMaster - Container Completion for
containerID=container_1493405038155_0001_01_000002, state=COMPLETE,
exitStatus=-100, diagnostics=Container released by application
2017-04-28 18:48:45,283 [AMRM Callback Handler Thread] INFO state.AppState -
Container was queued for release : container_1493405038155_0001_01_000002
2017-04-28 18:48:45,283 [AMRM Callback Handler Thread] INFO state.AppState -
decrementing role count for role MEMCACHED to 0; releasing=0, completed=0
2017-04-28 18:48:45,737 [AMRM Callback Handler Thread] INFO state.AppState -
Removing node ID container_1493405038155_0001_01_000002
2017-04-28 18:48:45,738 [AmExecutor-006] INFO appmaster.SliderAppMaster -
Unregistering component container_1493405038155_0001_01_000002
________________________________
From: Billie Rinaldi <[email protected]>
Sent: Friday, April 28, 2017 3:23 PM
To: [email protected]
Subject: Re: Slider fails to launch the memcached container in my version of
the example, Registration Response gets a Error 415 Unsupported Media Type
I tried spinning up a memcached app and did not encounter this issue. I
don't really see anything wrong with your configuration. We do not
recommend trying to use a custom agent.ini, so I'd remove that. Also you
don't have to specify --manager localhost:8032 in the Slider create command
when Slider is reading that info from the yarn-site.xml. I don't expect
those changes to affect the issue you're seeing, though. What version of
Python are you using? What does the AM log say?
On Thu, Apr 27, 2017 at 5:47 PM, Foolish Ewe <[email protected]> wrote:
> Hello All:
>
>
> First, thanks to Billie Rinaldi for the previous help, however I am still
> trying to get the memcached example to work in my slider environment and I
> could use some guidance as to whether there is some configuration error or
> an actual slider issue.
>
> Informally it looks like the application master container is launched but
> the container that is supposed to launch memcached is failing at
> registration time, in particular, after adding some logging, I wee that the
> Registration Response is as follows:
>
> INFO 2017-04-27 21:58:40,722 Controller.py:140 - Registration response =
> '<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;
> charset=ISO-8859-1"/>\n<title>Error 415 Unsupported Media
> Type</title>\n</head>\n<body><h2>HTTP ERROR 415</h2>\n<p>Problem
> accessing
> /ws/v1/slider/agents/container_1493330089109_0001_01_000002___MEMCACHED/register.
> Reason:\n<pre> Unsupported Media Type</pre></p><hr /><i><small>Powered
> by Jetty://</small></i><br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n\n</body>\n</html>\n'
>
>
>
>
> Here is a more detailed description of what I did and my configuration. I
> run the following commands in a single node docker instance (effectively on
> a logical edge node):
>
> To install the package I run the command:
>
> /usr/lib/slider/bin/slider install-package --package
> /tmp/mybuild/incubator-slider/app-packages/memcached/jmemcached-1.0.1.zip
> --name jmemcached --debug --replacepkg
>
> To launch the package I run the command:
>
> /usr/lib/slider/bin/slider create jmemcached --template
> /tmp/mybuild/incubator-slider/app-packages/memcached/appConfig.json
> --resources
> /tmp/mybuild/incubator-slider/app-packages/memcached/resources-default.json
> --manager localhost:8032 --debug
>
> Where the configuration files read:
>
> [root@quickstart mybuild]# cat /tmp/mybuild/incubator-slider/
> app-packages/memcached/appConfig.json
>
> {
>
> "schema": "http://example.org/specification/v2.0.0";,
>
> "metadata": {
>
> },
>
> "global": {
>
> "agent.conf": "artifacts/agent/conf/agent.ini",
>
> "application.def": ".slider/package/jmemcached/jmemcached-1.0.1.zip",
>
> "java_home": "/usr/lib/jvm/java/",
>
> "site.global.additional_cp": "",
>
> "site.global.xmx_val": "256m",
>
> "site.global.xms_val": "128m",
>
> "site.global.memory_val": "200M",
>
> "site.global.listen_port": "11211"
>
> },
>
> "components": {
>
> "slider-appmaster": {
>
> "jvm.heapsize": "256M"
>
> }
>
> }
>
> }
>
> [root@quickstart mybuild]# cat /tmp/mybuild/incubator-slider/
> app-packages/memcached/resources-default.json
>
> {
>
> "schema" : "http://example.org/specification/v2.0.0";,
>
> "metadata" : {
>
> },
>
> "global" : {
>
> },
>
> "components": {
>
> "slider-appmaster": {
>
> },
>
> "MEMCACHED": {
>
> "yarn.role.priority": "1",
>
> "yarn.component.instances": "1",
>
> "yarn.memory": "256"
>
> }
>
> }
>
> }
>
> [root@quickstart mybuild]# cat ./incubator-slider/app-package
> s/memcached/metainfo.xml
>
> <?xml version="1.0"?>
>
> <!--
>
> Licensed to the Apache Software Foundation (ASF) under one or more
>
> contributor license agreements. See the NOTICE file distributed with
>
> this work for additional information regarding copyright ownership.
>
> The ASF licenses this file to You under the Apache License, Version 2.0
>
> (the "License"); you may not use this file except in compliance with
>
> the License. You may obtain a copy of the License at
>
>
> http://www.apache.org/licenses/LICENSE-2.0
Apache License 2.0 - Apache Software
Foundation<http://www.apache.org/licenses/LICENSE-2.0>
www.apache.org
Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS
AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions ...
Apache License 2.0 - Apache Software
Foundation<http://www.apache.org/licenses/LICENSE-2.0>
Apache License 2.0 - Apache Software
Foundation<http://www.apache.org/licenses/LICENSE-2.0>
www.apache.org
Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS
AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions ...
www.apache.org<http://www.apache.org>
Welcome to The Apache Software Foundation!<http://www.apache.org/>
www.apache.org
Open. The Apache Software Foundation. provides support for the Apache Community
of open-source software projects, which provide software products for the
public good.
Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS
AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions ...
Apache Licenses - Apache Software Foundation<http://www.apache.org/licenses/>
www.apache.org
The Apache Software Foundation uses various licenses to distribute software and
documentation, to accept regular contributions from individuals and
corporations, and ...
>
>
> Unless required by applicable law or agreed to in writing, software
>
> distributed under the License is distributed on an "AS IS" BASIS,
>
> WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>
> See the License for the specific language governing permissions and
>
> limitations under the License.
>
> -->
>
>
> <metainfo>
>
> <schemaVersion>2.0</schemaVersion>
>
> <application>
>
> <name>MEMCACHED</name>
>
> <comment>Memcache is a network accessible key/value storage system,
> often used as a distributed cache.</comment>
>
> <!-- Changed version to see if I can get a hand built fat jar to work,
> where I control dependencies
>
> <version>1.0.0</version>
>
> -->
>
> <version>1.0.1-SNAPSHOT-jar-with-dependencies</version>
>
> <exportedConfigs>None</exportedConfigs>
>
> <exportGroups>
>
> <exportGroup>
>
> <name>Servers</name>
>
> <exports>
>
> <export>
>
> <name>host_port</name>
>
> <value>${MEMCACHED_HOST}:${site.global.listen_port}</value>
>
> </export>
>
> </exports>
>
> </exportGroup>
>
> </exportGroups>
>
>
> <components>
>
> <component>
>
> <name>MEMCACHED</name>
>
> <category>MASTER</category>
>
> <compExports>Servers-host_port</compExports>
>
> <commandScript>
>
> <script>scripts/memcached.py</script>
>
> <scriptType>PYTHON</scriptType>
>
> </commandScript>
>
> </component>
>
> </components>
>
>
> <osSpecifics>
>
> <osSpecific>
>
> <osType>any</osType>
>
> <packages>
>
> <package>
>
> <type>tarball</type>
>
> <name>files/jmemcached-1.0.1.tar</name>
>
> </package>
>
> </packages>
>
> </osSpecific>
>
> </osSpecifics>
>
>
> </application>
>
> </metainfo>
>
> I did some code diving and enhance the logging, based on earlier
> observations:
> In Controller.py I added some logging, in particular look at the
> Registration Response and to determine if the registerUrl was well formed.
>
> self.registerUrl = self.server_url + SLIDER_PATH_AGENTS + self.label +
> SLIDER_REL_PATH_REGISTER
>
> #Some code omitted
>
> while not self.isRegistered:
> try:
> data = json.dumps(self.register.build(
> self.componentActualState,
> self.componentExpectedState,
> self.actionQueue.customServiceOrchestrator.allocated_ports,
> self.actionQueue.customServiceOrchestrator.log_folders,
> self.tags,
> id))
> logger.info("Registering with the server at " + self.registerUrl +
> " with data " + pprint.pformat(data))
> response = self.sendRequest(self.registerUrl, data)
> logger.info("Registration response = " + pprint.pformat(response))
> logger.info("Dumping dir(self) = " + pprint.pformat(dir(self)))
> logger.info("Dumping vars(self) = " + pprint.pformat(vars(self)))
> logger.info("Dumping vars(self.config" +
> pprint.pformat(vars(self.config)))
> regResp = json.loads(response)
>
> The logs indicated several HTTP 415 error responses, that seemed to
> prevent the launch.
>
> INFO 2017-04-27 21:58:40,606 Controller.py:138 - Registering with the
> server at https://quickstart.cloudera:42131/ws/v1/slider/agents/contai
> ner_1493330089109_0001_01_000002___MEMCACHED/register with data
> '{"actualState": 0, "logFolders": {}, "agentVersion": "1",
> "allocatedPorts": {}, "timestamp": 1493330320605, "expectedState": 0,
> "tags": "", "responseId": -1, "publicHostname": "quickstart.cloudera",
> "label": "container_1493330089109_0001_01_000002___MEMCACHED"}'
>
>
> [ Lines removed to reduce log noise]
>
> INFO 2017-04-27 21:58:40,722 Controller.py:140 - Registration response =
> '<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;
> charset=ISO-8859-1"/>\n<title>Error 415 Unsupported Media
> Type</title>\n</head>\n<body><h2>HTTP ERROR 415</h2>\n<p>Problem
> accessing
> /ws/v1/slider/agents/container_1493330089109_0001_01_000002___MEMCACHED/register.
> Reason:\n<pre> Unsupported Media Type</pre></p><hr /><i><small>Powered
> by Jetty://</small></i><br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n<br/>
> \n<br/> \n<br/>
> \n<br/>
> \n\n</body>\n</html>\n'
> [Lines removed here to reduce log noise]
> INFO 2017-04-27 21:58:40,724 Controller.py:143 - Dumping
> vars(self.config{'label': 'container_1493330089109_0001_
> 01_000002___MEMCACHED',
> 'logroot': '/var/log/hadoop-yarn/containers/application_1493330089109_
> 0001/container_1493330089109_0001_01_000002',
> 'workroot': '/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root
> /appcache/application_1493330089109_0001/container_149333008
> 9109_0001_01_000002'}
> INFO 2017-04-27 21:58:40,725 Controller.py:185 - Unable to connect to:
> https://quickstart.cloudera:42131/ws/v1/slider/agents/contai
> ner_1493330089109_0001_01_000002___MEMCACHED/register
> Traceback (most recent call last):
> File "/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root
> /appcache/application_1493330089109_0001/filecache/21/
> slider-agent.tar.gz/slider-agent/agent/Controller.py", line 144, in
> registerWithServer
> regResp = json.loads(response)
> File "/usr/lib64/python2.7/json/__init__.py", line 338, in loads
> return _default_decoder.decode(s)
> File "/usr/lib64/python2.7/json/decoder.py", line 366, in decode
> obj, end = self.raw_decode(s, idx=_w(s, 0).end())
> File "/usr/lib64/python2.7/json/decoder.py", line 384, in raw_decode
> raise ValueError("No JSON object could be decoded")
> ValueError: No JSON object could be decoded
>
>
> I did some code diving and enhance the logging, where I put strings in a
> python list and dump the information once the logging output is configured
> (hence the multiple logged messages at line 265).
> i.e, main.py has:
>
> def main():
> parser = OptionParser()
> parser.add_option("-v", "--verbose", dest="verbose", help="verbose log
> output", default=False)
> parser.add_option("-l", "--label", dest="label", help="label of the
> agent", default=None)
> parser.add_option("--zk-quorum", dest=Constants.ZK_QUORUM,
> help="Zookeeper Quorum", default=None)
> parser.add_option("--zk-reg-path", dest=Constants.ZK_REG_PATH,
> help="Zookeeper Registry Path", default=None)
> parser.add_option("--debug", dest="debug", help="Agent debug hint",
> default="")
> (options, args) = parser.parse_args()
> preLogConfigStrs = ["after parse_args, args = " + pprint.pformat(args),
> "after parse_args, options = " + pprint.pformat(options),
> "after parse_args, vars(options) = " + pprint.pformat(vars(options)),
> "after parse_args, sys.argv = " + pprint.pformat(sys.argv)]
>
> #some code omitted
>
> logger.info("Using AGENT_WORK_ROOT = " + options.root_folder)
> logger.info("Using AGENT_LOG_ROOT = " + options.log_folder)
> logger.info("logFile = " + repr(logFile))
> logger.info("Dumping data to log acquired before logging configuration
> was complete")
> for str in preLogConfigStrs:
> logger.info(str)
>
> Further inspection of the failed containers launching jmemcached, shows
> that the command line arguments have the
>
> INFO 2017-04-27 21:58:40,285 main.py:97 - Newloglevel=logging.DEBUG
> INFO 2017-04-27 21:58:40,285 main.py:260 - Using AGENT_WORK_ROOT =
> /var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/root/
> appcache/application_1493330089109_0001/container_1493330089
> 109_0001_01_000002
> INFO 2017-04-27 21:58:40,285 main.py:261 - Using AGENT_LOG_ROOT =
> /var/log/hadoop-yarn/containers/application_1493330089109_
> 0001/container_1493330089109_0001_01_000002
> INFO 2017-04-27 21:58:40,285 main.py:262 - logFile =
> '/var/log/hadoop-yarn/containers/application_1493330089109_
> 0001/container_1493330089109_0001_01_000002/./slider-agent.log'
> INFO 2017-04-27 21:58:40,285 main.py:263 - Dumping data to log acquired
> before logging configuration was complete
> INFO 2017-04-27 21:58:40,285 main.py:265 - after parse_args, args = []
> INFO 2017-04-27 21:58:40,285 main.py:265 - after parse_args, options =
> <Values at 0x2bde2d8: {'debug': '', 'zk_quorum': 'localhost:2181',
> 'zk_reg_path': '/registry/users/root/services
> /org-apache-slider/jmemcached', 'verbose': False, 'label':
> 'container_1493330089109_0001_01_000002___MEMCACHED'}>
> INFO 2017-04-27 21:58:40,285 main.py:265 - after parse_args, vars(options)
> = {'debug': '',
> 'label': 'container_1493330089109_0001_01_000002___MEMCACHED',
> 'verbose': False,
> 'zk_quorum': 'localhost:2181',
> 'zk_reg_path': '/registry/users/root/services
> /org-apache-slider/jmemcached'}
> INFO 2017-04-27 21:58:40,285 main.py:265 - after parse_args, sys.argv =
> ['./infra/agent/slider-agent/agent/main.py',
> '--label',
> 'container_1493330089109_0001_01_000002___MEMCACHED',
> '--zk-quorum',
> 'localhost:2181',
> '--zk-reg-path',
> '/registry/users/root/services/org-apache-slider/jmemcached']
>
>
> I'm still trying to get to the bottom of this, any ideas of what should be
> done next?
>
>
> With best regards:
>
>
> Bill
>
