[ https://issues.apache.org/jira/browse/METRON-1506?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
tantian updated METRON-1506: ---------------------------- Description: ... 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType: START_CONTAINER for Container container_e05_1521078534073_0005_01_000008 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy : node1:45454 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType: QUERY_CONTAINER for Container container_e05_1521078534073_0005_01_000008 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy : node1:45454 {color:#ff0000}18/03/26 17:16:32 ERROR service.ApplicationMaster: Received a null request...{color} 18/03/26 17:17:19 INFO service.ApplicationMaster: [ADD]: Received request for model ssss:1.0x1 containers of size 512M at path /user/root/maas/sample 18/03/26 17:17:19 INFO service.ApplicationMaster: Found container id of 5497558138889 18/03/26 17:17:19 INFO callback.LaunchContainer: Setting up container launch container for containerid=container_e05_1521078534073_0005_01_000009 18/03/26 17:17:19 INFO callback.LaunchContainer: Local Directory Contents ... So I read the codes in ModelSubmission.java, and I found the client (maas_deploy.sh) only communicates to the zookeeper, and does not communicate to the ApplicationMaster. But in the code, when the client list the queried deployed models, it sends a null request to the ApplicationMaster, which I thought is not necessary. The related codes are listed here (the red lines are added by me): {color:#333333}ModelRequest request = null;{color} CuratorFramework client = null; try { RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); client = CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli), retryPolicy); client.start(); MaaSConfig config = ConfigUtil.INSTANCE.read(client, ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new MaaSConfig(), MaaSConfig.class); String mode = ModelSubmissionOptions.MODE.get(cli); if ( mode.equalsIgnoreCase("ADD")) { request = new ModelRequest() {{ setName(ModelSubmissionOptions.NAME.get(cli)); setAction(Action.ADD); setVersion(ModelSubmissionOptions.VERSION.get(cli)); setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli))); setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli))); setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli)); }}; } else if(mode.equalsIgnoreCase("REMOVE")) { request = new ModelRequest() {{ setName(ModelSubmissionOptions.NAME.get(cli)); setAction(Action.REMOVE); setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli))); setVersion(ModelSubmissionOptions.VERSION.get(cli)); }}; } else if(mode.equalsIgnoreCase("LIST")) { String name = ModelSubmissionOptions.NAME.get(cli, null); String version = ModelSubmissionOptions.VERSION.get(cli, null); ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client, config.getServiceRoot()); Model model = new Model(name, version); Map<Model, List<ModelEndpoint>> endpoints = serviceDiscoverer.listEndpoints(model); for(Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) { String modelTitle = "Model " + kv.getKey().getName() + " @ " + kv.getKey().getVersion(); System.out.println(modelTitle); for(ModelEndpoint endpoint : kv.getValue()) { System.out.println(endpoint); } } } if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli)) { File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli)); Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli)); updateHDFS(fs, localDir, hdfsPath); } {color:#ff0000}if (request != null) {{color} Queue<ModelRequest> queue = config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client)); queue.enqueue(request); } } finally { if (client != null) { client.close(); } } was: ... 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType: START_CONTAINER for Container container_e05_1521078534073_0005_01_000008 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy : node1:45454 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType: QUERY_CONTAINER for Container container_e05_1521078534073_0005_01_000008 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy : node1:45454 {color:#ff0000}18/03/26 17:16:32 ERROR service.ApplicationMaster: Received a null request...{color} 18/03/26 17:17:19 INFO service.ApplicationMaster: [ADD]: Received request for model ssss:1.0x1 containers of size 512M at path /user/root/maas/sample 18/03/26 17:17:19 INFO service.ApplicationMaster: Found container id of 5497558138889 18/03/26 17:17:19 INFO callback.LaunchContainer: Setting up container launch container for containerid=container_e05_1521078534073_0005_01_000009 18/03/26 17:17:19 INFO callback.LaunchContainer: Local Directory Contents ... So I read the codes in ModelSubmission.java, and I found the client (maas_deploy.sh) only communicates to the zookeeper, and does not communicate to the ApplicationMaster. But in the code, when the client list the queried deployed models, it sends a null request to the ApplicationMaster, which I thought is not necessary. The related codes are listed here (the red lines are added by me): {color:#333333}ModelRequest request = null;{color} CuratorFramework client = null; try { RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); client = CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli), retryPolicy); client.start(); MaaSConfig config = ConfigUtil.INSTANCE.read(client, ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new MaaSConfig(), MaaSConfig.class); String mode = ModelSubmissionOptions.MODE.get(cli); if ( mode.equalsIgnoreCase("ADD")) { request = new ModelRequest() {{ setName(ModelSubmissionOptions.NAME.get(cli)); setAction(Action.ADD); setVersion(ModelSubmissionOptions.VERSION.get(cli)); setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli))); setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli))); setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli)); }}; } else if(mode.equalsIgnoreCase("REMOVE")) { request = new ModelRequest() {{ setName(ModelSubmissionOptions.NAME.get(cli)); setAction(Action.REMOVE); setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli))); setVersion(ModelSubmissionOptions.VERSION.get(cli)); }}; } else if(mode.equalsIgnoreCase("LIST")) { String name = ModelSubmissionOptions.NAME.get(cli, null); String version = ModelSubmissionOptions.VERSION.get(cli, null); ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client, config.getServiceRoot()); Model model = new Model(name, version); Map<Model, List<ModelEndpoint>> endpoints = serviceDiscoverer.listEndpoints(model); for(Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) { String modelTitle = "Model " + kv.getKey().getName() + " @ " + kv.getKey().getVersion(); System.out.println(modelTitle); for(ModelEndpoint endpoint : kv.getValue()) { System.out.println(endpoint); } } } if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli)) { File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli)); Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli)); updateHDFS(fs, localDir, hdfsPath); } {color:#ff0000}if (request != null) {{color} Queue<ModelRequest> queue = config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client)); queue.enqueue(request); {color:#d04437}}{color} } finally { if (client != null) { client.close(); } } > When using maas_deploy.sh to list the deployed models, the > ApplicationMaster(MaaS) receives 'null request' > ---------------------------------------------------------------------------------------------------------- > > Key: METRON-1506 > URL: https://issues.apache.org/jira/browse/METRON-1506 > Project: Metron > Issue Type: Bug > Affects Versions: 0.4.1 > Reporter: tantian > Priority: Minor > Labels: easyfix > > ... > 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType: > START_CONTAINER for Container container_e05_1521078534073_0005_01_000008 > 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy > : node1:45454 > 18/03/26 17:16:23 INFO impl.NMClientAsyncImpl: Processing Event EventType: > QUERY_CONTAINER for Container container_e05_1521078534073_0005_01_000008 > 18/03/26 17:16:23 INFO impl.ContainerManagementProtocolProxy: Opening proxy > : node1:45454 > {color:#ff0000}18/03/26 17:16:32 ERROR service.ApplicationMaster: Received a > null request...{color} > 18/03/26 17:17:19 INFO service.ApplicationMaster: [ADD]: Received request > for model ssss:1.0x1 containers of size 512M at path /user/root/maas/sample > 18/03/26 17:17:19 INFO service.ApplicationMaster: Found container id of > 5497558138889 > 18/03/26 17:17:19 INFO callback.LaunchContainer: Setting up container launch > container for containerid=container_e05_1521078534073_0005_01_000009 > 18/03/26 17:17:19 INFO callback.LaunchContainer: Local Directory Contents > ... > > So I read the codes in ModelSubmission.java, and I found the client > (maas_deploy.sh) only communicates to the zookeeper, and does not communicate > to the ApplicationMaster. But in the code, when the client list the queried > deployed models, it sends a null request to the ApplicationMaster, which I > thought is not necessary. The related codes are listed here (the red lines > are added by me): > {color:#333333}ModelRequest request = null;{color} > CuratorFramework client = null; > try { > RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); > client = > CuratorFrameworkFactory.newClient(ModelSubmissionOptions.ZK_QUORUM.get(cli), > retryPolicy); > client.start(); > MaaSConfig config = ConfigUtil.INSTANCE.read(client, > ModelSubmissionOptions.ZK_ROOT.get(cli, "/metron/maas/config"), new > MaaSConfig(), MaaSConfig.class); > String mode = ModelSubmissionOptions.MODE.get(cli); > if ( mode.equalsIgnoreCase("ADD")) { > request = new ModelRequest() {{ > setName(ModelSubmissionOptions.NAME.get(cli)); > setAction(Action.ADD); > setVersion(ModelSubmissionOptions.VERSION.get(cli)); > > setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli))); > setMemory(Integer.parseInt(ModelSubmissionOptions.MEMORY.get(cli))); > setPath(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli)); > }}; > } else if(mode.equalsIgnoreCase("REMOVE")) { > request = new ModelRequest() {{ > setName(ModelSubmissionOptions.NAME.get(cli)); > setAction(Action.REMOVE); > > setNumInstances(Integer.parseInt(ModelSubmissionOptions.NUM_INSTANCES.get(cli))); > setVersion(ModelSubmissionOptions.VERSION.get(cli)); > }}; > } > else if(mode.equalsIgnoreCase("LIST")) { > String name = ModelSubmissionOptions.NAME.get(cli, null); > String version = ModelSubmissionOptions.VERSION.get(cli, null); > ServiceDiscoverer serviceDiscoverer = new ServiceDiscoverer(client, > config.getServiceRoot()); > Model model = new Model(name, version); > Map<Model, List<ModelEndpoint>> endpoints = > serviceDiscoverer.listEndpoints(model); > for(Map.Entry<Model, List<ModelEndpoint>> kv : endpoints.entrySet()) { > String modelTitle = "Model " + kv.getKey().getName() + " @ " + > kv.getKey().getVersion(); > System.out.println(modelTitle); > for(ModelEndpoint endpoint : kv.getValue()) > { System.out.println(endpoint); } > } > } > if (ModelSubmissionOptions.LOCAL_MODEL_PATH.has(cli)) > { File localDir = new File(ModelSubmissionOptions.LOCAL_MODEL_PATH.get(cli)); > Path hdfsPath = new Path(ModelSubmissionOptions.HDFS_MODEL_PATH.get(cli)); > updateHDFS(fs, localDir, hdfsPath); } > {color:#ff0000}if (request != null) {{color} > Queue<ModelRequest> queue = > config.createQueue(ImmutableMap.of(ZKQueue.ZK_CLIENT, client)); > queue.enqueue(request); > } > } finally { > if (client != null) > { client.close(); } > } -- This message was sent by Atlassian JIRA (v7.6.3#76005)