Repository: eagle Updated Branches: refs/heads/master 20b4bea3e -> 7c0d83f3c
[EAGLE-999] JobConfigSerDeser fails to serialize/deserialize data with long string https://issues.apache.org/jira/browse/EAGLE-999 Author: Zhao, Qingwen <[email protected]> Closes #909 from qingwen220/EAGLE-999. Project: http://git-wip-us.apache.org/repos/asf/eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/7c0d83f3 Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/7c0d83f3 Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/7c0d83f3 Branch: refs/heads/master Commit: 7c0d83f3c4847dac450ba505153a5bf17d30fd73 Parents: 20b4bea Author: Zhao, Qingwen <[email protected]> Authored: Tue Apr 11 17:36:50 2017 +0800 Committer: Zhao, Qingwen <[email protected]> Committed: Tue Apr 11 17:36:50 2017 +0800 ---------------------------------------------------------------------- ...doop.queue.HadoopQueueRunningAppProvider.xml | 2 +- eagle-jpm/eagle-jpm-entity/pom.xml | 3 +- .../jpm/mr/historyentity/JobConfigSerDeser.java | 41 ++++-------- .../eagle/jpm/mr/JobConfigSerDeserTest.java | 65 ++++++++++++++++++++ ....running.MRRunningJobApplicationProvider.xml | 2 +- ...ecurity.auditlog.HdfsAuditLogAppProvider.xml | 4 +- ...ity.oozie.parse.OozieAuditLogAppProvider.xml | 2 +- ....eagle.topology.TopologyCheckAppProvider.xml | 2 +- 8 files changed, 83 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml index 2d68da8..f33797f 100644 --- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml +++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml @@ -57,7 +57,7 @@ <property> <name>dataSinkConfig.HADOOP_QUEUE_STREAM.topic</name> <displayName>Destination(Kafka Topic) Of Queue Stream Data</displayName> - <value>yarn_queue</value> + <value>yarn_queue_${siteId}</value> <description>topic for kafka data sink</description> </property> <property> http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/pom.xml b/eagle-jpm/eagle-jpm-entity/pom.xml index 98982f9..9a38725 100644 --- a/eagle-jpm/eagle-jpm-entity/pom.xml +++ b/eagle-jpm/eagle-jpm-entity/pom.xml @@ -42,10 +42,9 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>3.8.1</version> + <version>${junit.version}</version> <scope>test</scope> </dependency> - </dependencies> </project> http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java index cfa50f9..f4f58d8 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java @@ -19,45 +19,26 @@ package org.apache.eagle.jpm.mr.historyentity; import org.apache.eagle.log.entity.meta.EntitySerDeser; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.eagle.log.entity.meta.MapSerDeser; import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; + public class JobConfigSerDeser implements EntitySerDeser<JobConfig> { + private static final MapSerDeser INSTANCE = new MapSerDeser(); + @Override public JobConfig deserialize(byte[] bytes) { - JobConfig jc = new JobConfig(); - Map<String, String> map = new TreeMap<String, String>(); - jc.setConfig(map); - String sb = Bytes.toString(bytes); - String[] keyValue = sb.split(","); - for (String pair : keyValue) { - String[] str = pair.split(":"); - if (pair.equals("") || str[0].equals("")) { - continue; - } - String key = str[0]; - String value = ""; - if (str.length == 2) { - value = str[1]; - } - map.put(key, value); - } - return jc; + Map map = INSTANCE.deserialize(bytes); + JobConfig config = new JobConfig(); + config.setConfig(map); + return config; } - + @Override - public byte[] serialize(JobConfig conf) { - Map<String, String> map = conf.getConfig(); - StringBuilder sb = new StringBuilder(); - for (Entry<String, String> entry : map.entrySet()) { - sb.append(entry.getKey() + ":" + entry.getValue() + ","); - } - sb.deleteCharAt(sb.length() - 1); - return sb.toString().getBytes(); + public byte[] serialize(JobConfig jobConfig) { + return INSTANCE.serialize(jobConfig.getConfig()); } @Override http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java b/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java new file mode 100644 index 0000000..a4b5307 --- /dev/null +++ b/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.mr; + +import org.apache.eagle.jpm.mr.historyentity.JobConfig; +import org.apache.eagle.jpm.mr.historyentity.JobConfigSerDeser; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class JobConfigSerDeserTest { + + @Test + public void test() { + String value = "INSERT OVERWRITE TABLE kylin_intermediate_KYLIN_HIVE_METRICS_QUERY_CUBE SELECT\n" + + "HIVE_METRICS_QUERY_CUBE.`CUBE_NAME`\n" + + ",HIVE_METRICS_QUERY_CUBE.`SEGMENT_NAME`\n" + + ",HIVE_METRICS_QUERY_CUBE.`CUBOID_SOURCE`\n" + + ",HIVE_METRICS_QUERY_CUBE.`CUBOID_TARGET`\n" + + ",HIVE_METRICS_QUERY_CUBE.`IF_MATCH`\n" + + ",HIVE_METRICS_QUERY_CUBE.`IF_SUCCESS`\n" + + ",HIVE_METRICS_QUERY_CUBE.`KYEAR_BEGIN_DATE`\n" + + ",HIVE_METRICS_QUERY_CUBE.`KMONTH_BEGIN_DATE`\n" + + ",HIVE_METRICS_QUERY_CUBE.`KWEEK_BEGIN_DATE`\n" + + ",HIVE_METRICS_QUERY_CUBE.`KDAY_DATE`\n" + + ",HIVE_METRICS_QUERY_CUBE.`WEIGHT_PER_HIT`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_COUNT`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_TIME_SUM`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_TIME_MAX`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_COUNT_SKIP`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_SCAN`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_RETURN`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_AGGREGATE_FILTER`\n" + + ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_AGGREGATE`\n" + + "FROM KYLIN.HIVE_METRICS_QUERY_CUBE as HIVE_METRICS_QUERY_CUBE\n" + + "WHERE (((HIVE_METRICS_QUERY_CUBE.KDAY_DATE = '2017-04-06' AND HIVE_METRICS_QUERY_CUBE.KDAY_TIME >= '18:00:00') OR (HIVE_METRICS_QUERY_CUBE.KDAY_DATE > '2017-04-06')) AND ((HIVE_METRICS_QUERY_CUBE.KDAY_DATE = '2017-04-06' AND HIVE_METRICS_QUERY_CUBE.KDAY_TIME < '20:00:00') OR (HIVE_METRICS_QUERY_CUBE.KDAY_DATE < '2017-04-06')))"; + + Map<String, String> conf = new HashMap<>(); + conf.put("test1", value); + JobConfig source = new JobConfig(); + source.setConfig(conf); + JobConfigSerDeser serDeser = new JobConfigSerDeser(); + byte[] bytes = serDeser.serialize(source); + JobConfig target = serDeser.deserialize(bytes); + + Assert.assertTrue(target.getConfig().size() == 1); + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml index fe3ab83..907e35a 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml @@ -104,7 +104,7 @@ <property> <name>dataSinkConfig.ACCEPTED_APP_STREAM.topic</name> <displayName>Destination(Kafka Topic) Of App Stream Data</displayName> - <value>yarn_accepted_app_{site}</value> + <value>yarn_accepted_app_{siteId}</value> <description>topic for kafka data sink</description> <required>true</required> </property> http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml index d48340c..d453cd6 100644 --- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml @@ -130,14 +130,14 @@ <property> <name>dataSinkConfig.HDFS_AUDIT_LOG_ENRICHED_STREAM.topic</name> <displayName>Kafka Topic for Auditlog Event Sink</displayName> - <value>hdfs_audit_event_${site}</value> + <value>hdfs_audit_event_${siteId}</value> <description>topic for kafka data sink</description> <required>true</required> </property> <property> <name>dataSinkConfig.HDFS_AUDIT_LOG_TRAFFIC_STREAM.topic</name> <displayName>Kafka Topic for Traffic Data Sink</displayName> - <value>hdfs_traffic_event_${site}</value> + <value>hdfs_traffic_event_${siteId}</value> <description>topic for kafka data sink</description> </property> <property> http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml index ee1715c..5434a9d 100644 --- a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml @@ -69,7 +69,7 @@ <property> <name>dataSourceConfig.topic</name> <displayName>Kafka Topic for Data Consumption</displayName> - <value>oozie_audit_log</value> + <value>oozie_audit_log_${siteId}</value> <description>data source topic</description> </property> <property> http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml index 4bdb1c4..aa36c53 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml +++ b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml @@ -165,7 +165,7 @@ <property> <name>dataSinkConfig.topic</name> <displayName>Topic For Kafka Data Sink</displayName> - <value>topology_check</value> + <value>topology_check_${siteId}</value> <description>topic For kafka data sink</description> </property> <property>
