[
https://issues.apache.org/jira/browse/METRON-1506?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
tantian updated METRON-1506:
----------------------------
Description:
...
18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType:
START_CONTAINER for Container container_e05_1521078534073_0005_01_000008
18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy :
node1:45454
18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType:
QUERY_CONTAINER for Container container_e05_1521078534073_0005_01_000008
18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy :
node1:45454
{color:#ff0000}18/03/26 17:16:32 ERROR service.ApplicationMaster: Received a
null request...{color}
18/03/26 17:17:19 INFO service.ApplicationMaster: [ADD]: Received request for
model ssss:1.0x1 containers of size 512M at path /user/root/maas/sample
18/03/26 17:17:19 INFO service.ApplicationMaster: Found container id of
5497558138889
18/03/26 17:17:19 INFO callback.LaunchContainer: Setting up container launch
container for containerid=container_e05_1521078534073_0005_01_000009
18/03/26 17:17:19 INFO callback.LaunchContainer: Local Directory Contents
...
So I read the codes in ModelSubmission.java, and I found the client
(maas_deploy.sh) only communicates to the zookeeper, and does not communicate
to the ApplicationMaster. But in the code, when the client list the queried
deployed models, it sends a null request to the ApplicationMaster, which I
thought is not necessary. The related codes are listed here (the red lines are
added by me):
{color:#333333}ModelRequest request = null;{color}
CuratorFramework client = null;
try {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
client =
CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli),
retryPolicy);
client.start();
MaaSConfig config = ConfigUtil.INSTANCE.read(client,
ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new
MaaSConfig(), MaaSConfig.class);
String mode = ModelSubmissionOptions.MODE.get(cli);
if ( mode.equalsIgnoreCase("ADD")) {
request = new ModelRequest() {{
setName(ModelSubmissionOptions.NAME.get(cli));
setAction(Action.ADD);
setVersion(ModelSubmissionOptions.VERSION.get(cli));
setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli)));
setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
}};
} else if(mode.equalsIgnoreCase("REMOVE")) {
request = new ModelRequest() {{
setName(ModelSubmissionOptions.NAME.get(cli));
setAction(Action.REMOVE);
setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
setVersion(ModelSubmissionOptions.VERSION.get(cli));
}};
}
else if(mode.equalsIgnoreCase("LIST")) {
String name = ModelSubmissionOptions.NAME.get(cli, null);
String version = ModelSubmissionOptions.VERSION.get(cli, null);
ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client,
config.getServiceRoot());
Model model = new Model(name, version);
Map<Model, List<ModelEndpoint>> endpoints =
serviceDiscoverer.listEndpoints(model);
for(Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) {
String modelTitle = "Model " + kv.getKey().getName() + " @ " +
kv.getKey().getVersion();
System.out.println(modelTitle);
for(ModelEndpoint endpoint : kv.getValue())
{ System.out.println(endpoint); }
}
}
if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli))
{ File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli));
Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
updateHDFS(fs, localDir, hdfsPath); }
{color:#ff0000}if (request != null) {{color}
Queue<ModelRequest> queue =
config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client));
queue.enqueue(request);
{color:#d04437}}{color}
} finally {
if (client != null)
{ client.close(); }
}
was:
...
18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType:
START_CONTAINER for Container container_e05_1521078534073_0005_01_000008
18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy :
node1:45454
18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType:
QUERY_CONTAINER for Container container_e05_1521078534073_0005_01_000008
18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy :
node1:45454
{color:#ff0000}18/03/26 17:16:32 ERROR service.ApplicationMaster: Received a
null request...{color}
18/03/26 17:17:19 INFO service.ApplicationMaster: [ADD]: Received request for
model ssss:1.0x1 containers of size 512M at path /user/root/maas/sample
18/03/26 17:17:19 INFO service.ApplicationMaster: Found container id of
5497558138889
18/03/26 17:17:19 INFO callback.LaunchContainer: Setting up container launch
container for containerid=container_e05_1521078534073_0005_01_000009
18/03/26 17:17:19 INFO callback.LaunchContainer: Local Directory Contents
...
So I read the codes in ModelSubmission.java, and I found the client
(maas_deploy.sh) only communicates to the zookeeper, and does not communicate
to the ApplicationMaster. But in the code, when the client list the queried
deployed models, it sends a null request to the ApplicationMaster, which I
thought is not necessary. The related codes are listed here (the red lines are
added by me):
{color:#333333}ModelRequest request = null;{color}
CuratorFramework client = null;
try {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
client =
CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli),
retryPolicy);
client.start();
MaaSConfig config = ConfigUtil.INSTANCE.read(client,
ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new
MaaSConfig(), MaaSConfig.class);
String mode = ModelSubmissionOptions.MODE.get(cli);
if ( mode.equalsIgnoreCase("ADD")) {
request = new ModelRequest() {{
setName(ModelSubmissionOptions.NAME.get(cli));
setAction(Action.ADD);
setVersion(ModelSubmissionOptions.VERSION.get(cli));
setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli)));
setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
}};
} else if(mode.equalsIgnoreCase("REMOVE")) {
request = new ModelRequest() {{
setName(ModelSubmissionOptions.NAME.get(cli));
setAction(Action.REMOVE);
setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
setVersion(ModelSubmissionOptions.VERSION.get(cli));
}};
}
else if(mode.equalsIgnoreCase("LIST")) {
String name = ModelSubmissionOptions.NAME.get(cli, null);
String version = ModelSubmissionOptions.VERSION.get(cli, null);
ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client,
config.getServiceRoot());
Model model = new Model(name, version);
Map<Model, List<ModelEndpoint>> endpoints =
serviceDiscoverer.listEndpoints(model);
for(Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) {
String modelTitle = "Model " + kv.getKey().getName() + " @ " +
kv.getKey().getVersion();
System.out.println(modelTitle);
for(ModelEndpoint endpoint : kv.getValue())
{ System.out.println(endpoint); }
}
}
if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli))
{ File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli));
Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
updateHDFS(fs, localDir, hdfsPath); }
{color:#ff0000}if (request != null) {{color}
Queue<ModelRequest> queue =
config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client));
queue.enqueue(request);
}
} finally {
if (client != null)
{ client.close(); }
}
> When using maas_deploy.sh to list the deployed models, the
> ApplicationMaster(MaaS) receives 'null request'
> ----------------------------------------------------------------------------------------------------------
>
> Key: METRON-1506
> URL: https://issues.apache.org/jira/browse/METRON-1506
> Project: Metron
> Issue Type: Bug
> Affects Versions: 0.4.1
> Reporter: tantian
> Priority: Minor
> Labels: easyfix
>
> ...
> 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType:
> START_CONTAINER for Container container_e05_1521078534073_0005_01_000008
> 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy
> : node1:45454
> 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType:
> QUERY_CONTAINER for Container container_e05_1521078534073_0005_01_000008
> 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy
> : node1:45454
> {color:#ff0000}18/03/26 17:16:32 ERROR service.ApplicationMaster: Received a
> null request...{color}
> 18/03/26 17:17:19 INFO service.ApplicationMaster: [ADD]: Received request
> for model ssss:1.0x1 containers of size 512M at path /user/root/maas/sample
> 18/03/26 17:17:19 INFO service.ApplicationMaster: Found container id of
> 5497558138889
> 18/03/26 17:17:19 INFO callback.LaunchContainer: Setting up container launch
> container for containerid=container_e05_1521078534073_0005_01_000009
> 18/03/26 17:17:19 INFO callback.LaunchContainer: Local Directory Contents
> ...
>
> So I read the codes in ModelSubmission.java, and I found the client
> (maas_deploy.sh) only communicates to the zookeeper, and does not communicate
> to the ApplicationMaster. But in the code, when the client list the queried
> deployed models, it sends a null request to the ApplicationMaster, which I
> thought is not necessary. The related codes are listed here (the red lines
> are added by me):
> {color:#333333}ModelRequest request = null;{color}
> CuratorFramework client = null;
> try {
> RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
> client =
> CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli),
> retryPolicy);
> client.start();
> MaaSConfig config = ConfigUtil.INSTANCE.read(client,
> ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new
> MaaSConfig(), MaaSConfig.class);
> String mode = ModelSubmissionOptions.MODE.get(cli);
> if ( mode.equalsIgnoreCase("ADD")) {
> request = new ModelRequest() {{
> setName(ModelSubmissionOptions.NAME.get(cli));
> setAction(Action.ADD);
> setVersion(ModelSubmissionOptions.VERSION.get(cli));
>
> setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
> setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli)));
> setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
> }};
> } else if(mode.equalsIgnoreCase("REMOVE")) {
> request = new ModelRequest() {{
> setName(ModelSubmissionOptions.NAME.get(cli));
> setAction(Action.REMOVE);
>
> setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli)));
> setVersion(ModelSubmissionOptions.VERSION.get(cli));
> }};
> }
> else if(mode.equalsIgnoreCase("LIST")) {
> String name = ModelSubmissionOptions.NAME.get(cli, null);
> String version = ModelSubmissionOptions.VERSION.get(cli, null);
> ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client,
> config.getServiceRoot());
> Model model = new Model(name, version);
> Map<Model, List<ModelEndpoint>> endpoints =
> serviceDiscoverer.listEndpoints(model);
> for(Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) {
> String modelTitle = "Model " + kv.getKey().getName() + " @ " +
> kv.getKey().getVersion();
> System.out.println(modelTitle);
> for(ModelEndpoint endpoint : kv.getValue())
> { System.out.println(endpoint); }
> }
> }
> if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli))
> { File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli));
> Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli));
> updateHDFS(fs, localDir, hdfsPath); }
> {color:#ff0000}if (request != null) {{color}
> Queue<ModelRequest> queue =
> config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client));
> queue.enqueue(request);
> {color:#d04437}}{color}
> } finally {
> if (client != null)
> { client.close(); }
> }
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
