Repository: incubator-griffin Updated Branches: refs/heads/master 81e9b6aee -> b5fce5247
ut and org controller Author: Chen <[email protected]> Closes #23 from justACT/ut. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/b5fce524 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/b5fce524 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/b5fce524 Branch: refs/heads/master Commit: b5fce52475bf275d6ff3fd97a1e0eee7b4a4d3fc Parents: 81e9b6a Author: Chen <[email protected]> Authored: Thu May 11 12:38:17 2017 +0800 Committer: William Guo <[email protected]> Committed: Thu May 11 12:38:17 2017 +0800 ---------------------------------------------------------------------- service/pom.xml | 28 ++++ .../griffin/core/GriffinWebApplication.java | 40 ++++-- .../griffin/core/measure/repo/MeasureRepo.java | 14 ++ .../griffin/core/metric/MetricController.java | 28 ++++ .../griffin/core/schedule/SparkSubmitJob.java | 18 ++- .../griffin/core/service/GriffinController.java | 20 +++ .../griffin/core/measure/AssertAnnotations.java | 63 ++++++++ .../griffin/core/measure/DataConnectorTest.java | 43 ++++++ .../griffin/core/measure/ReflectTool.java | 33 +++++ .../metastore/HiveMetastoreServiceTest.java | 52 +++++++ .../core/metastore/KafkaSchemaServiceTest.java | 71 +++++++++ .../core/schedule/SparkSubmitJobTest.java | 144 +++++++++++++++++++ .../src/test/resources/application.properties | 22 +++ service/src/test/resources/context.xml | 30 ++++ 14 files changed, 582 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/pom.xml ---------------------------------------------------------------------- diff --git a/service/pom.xml b/service/pom.xml index 88d7a00..d49077f 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -20,6 +20,8 @@ <confluent.version>3.2.0</confluent.version> <quartz.version>2.2.1</quartz.version> <start-class>org.apache.griffin.core.GriffinWebApplication</start-class> + <powermock.version>1.6.6</powermock.version> + <mockito.version>1.10.19</mockito.version> </properties> <repositories> @@ -118,11 +120,37 @@ <version>2.2.2</version> </dependency> + <!-- test --> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>${mockito.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + </dependencies> <profiles> </profiles> http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java b/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java index 7efbcf9..0c2f557 100644 --- a/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java +++ b/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java @@ -40,24 +40,36 @@ public class GriffinWebApplication implements CommandLineRunner{ public void run(String... strings) throws Exception { - HashMap<String,String> configMap1=new HashMap<>(); - configMap1.put("database","default"); - configMap1.put("table.name","test_data_src"); - HashMap<String,String> configMap2=new HashMap<>(); - configMap2.put("database","default"); - configMap2.put("table.name","test_data_tgt"); - String configJson1 = new ObjectMapper().writeValueAsString(configMap1); - String configJson2 = new ObjectMapper().writeValueAsString(configMap2); + HashMap<String,String> configMap1=new HashMap<>(); + configMap1.put("database","default"); + configMap1.put("table.name","test_data_src"); + HashMap<String,String> configMap2=new HashMap<>(); + configMap2.put("database","default"); + configMap2.put("table.name","test_data_tgt"); + String configJson1 = new ObjectMapper().writeValueAsString(configMap1); + String configJson2 = new ObjectMapper().writeValueAsString(configMap2); - DataConnector source = new DataConnector(ConnectorType.HIVE, "1.2", configJson1); - DataConnector target = new DataConnector(ConnectorType.HIVE, "1.2", configJson2); + DataConnector source = new DataConnector(ConnectorType.HIVE, "1.2", configJson1); + DataConnector target = new DataConnector(ConnectorType.HIVE, "1.2", configJson2); - String rules = "$source.uage > 100 AND $source.uid = $target.uid AND $source.uage + 12 = $target.uage + 10 + 2 AND $source.udes + 11 = $target.udes + 1 + 1"; + String rules = "$source.uage > 100 AND $source.uid = $target.uid AND $source.uage + 12 = $target.uage + 10 + 2 AND $source.udes + 11 = $target.udes + 1 + 1"; - EvaluateRule eRule = new EvaluateRule(1,rules); + EvaluateRule eRule = new EvaluateRule(1,rules); - Measure measure = new Measure("accu1","accu1 description", Measure.MearuseType.accuracy, "bullyeye", source, target, eRule,"test1"); - measureRepo.save(measure); + Measure measure = new Measure("bevssoj","bevssoj description", Measure.MearuseType.accuracy, "bullyeye", source, target, eRule,"test1"); + measureRepo.save(measure); + + DataConnector source2 = new DataConnector(ConnectorType.HIVE, "1.2", configJson1); + DataConnector target2 = new DataConnector(ConnectorType.HIVE, "1.2", configJson2); + EvaluateRule eRule2 = new EvaluateRule(1,rules); + Measure measure2 = new Measure("test","test description", Measure.MearuseType.accuracy, "bullyeye", source2, target2, eRule2,"test1"); + measureRepo.save(measure2); + + DataConnector source3 = new DataConnector(ConnectorType.HIVE, "1.2", configJson1); + DataConnector target3 = new DataConnector(ConnectorType.HIVE, "1.2", configJson2); + EvaluateRule eRule3 = new EvaluateRule(1,rules); + Measure measure3 = new Measure("just_inthere","test_just_inthere description", Measure.MearuseType.accuracy, "hadoop", source3, target3, eRule3,"test1"); + measureRepo.save(measure3); } @Bean http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java index 0eec264..b0f8d3c 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java +++ b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java @@ -2,8 +2,22 @@ package org.apache.griffin.core.measure.repo; import org.apache.griffin.core.measure.Measure; +import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; +import java.util.List; + public interface MeasureRepo extends CrudRepository<Measure, Long> { Measure findByName(String name); + + @Query("select DISTINCT m.organization from Measure m") + List<String> findOrganizations(); + + @Query("select m.name from Measure m " + + "where m.organization= ?1") + List<String> findNameByOrganization(String organization); + + @Query("select m.organization from Measure m "+ + "where m.name= ?1") + String findOrgByName(String measureName); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/metric/MetricController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metric/MetricController.java b/service/src/main/java/org/apache/griffin/core/metric/MetricController.java new file mode 100644 index 0000000..5b9f610 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/metric/MetricController.java @@ -0,0 +1,28 @@ +package org.apache.griffin.core.metric; + +import org.apache.griffin.core.measure.repo.MeasureRepo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Created by xiangrchen on 5/8/17. + */ +@RestController +@RequestMapping("/metrics") +public class MetricController { + + +// @RequestMapping("/organization/{org}/{measureName}") +private static final Logger log = LoggerFactory.getLogger(MetricController.class); + @Autowired + MeasureRepo measureRepo; + @RequestMapping("/org/{measureName}") + public String getOrgByMeasureName(@PathVariable("measureName") String measureName){ + return measureRepo.findOrgByName(measureName); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java b/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java index 9f34df5..a8ba755 100644 --- a/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java +++ b/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java @@ -85,16 +85,14 @@ public class SparkSubmitJob implements Job { if (sourcePattern != null && !sourcePattern.isEmpty()) { sourcePatternItemSet = sourcePattern.split("-"); - sourcePatternPartitionSizeMin = Math.min(partitionItemSet.length, sourcePatternItemSet.length); long currentTimstamp = setCurrentTimestamp(currentSystemTimestamp); - setDataConnectorPartitions(measure.getSource(), sourcePatternItemSet, partitionItemSet, currentTimstamp, sourcePatternPartitionSizeMin); + setDataConnectorPartitions(measure.getSource(), sourcePatternItemSet, partitionItemSet, currentTimstamp); jd.getJobDataMap().put("lastTime", currentTimstamp + ""); } if (targetPattern != null && !targetPattern.equals("")) { targetPatternItemSet = targetPattern.split("-"); - targetPatternPartitionSizeMin = Math.min(partitionItemSet.length, targetPatternItemSet.length); long currentTimstamp = setCurrentTimestamp(currentSystemTimestamp); - setDataConnectorPartitions(measure.getTarget(), targetPatternItemSet, partitionItemSet, currentTimstamp, targetPatternPartitionSizeMin); + setDataConnectorPartitions(measure.getTarget(), targetPatternItemSet, partitionItemSet, currentTimstamp); jd.getJobDataMap().put("lastTime", currentTimstamp + ""); } //final String uri = "http://10.9.246.187:8998/batches";; @@ -106,13 +104,14 @@ public class SparkSubmitJob implements Job { // {"file": "/exe/griffin-measure-batch-0.0.1-SNAPSHOT.jar", "className": "org.apache.griffin.measure.batch.Application", "args": ["/benchmark/test/env.json", "/benchmark/test/config-rdm.json"], "name": "griffin-livy", "queue": "default", "numExecutors": 2, "executorCores": 4, "driverMemory": "2g", "executorMemory": "2g", "conf": {"spark.jars.packages": "com.databricks:spark-avro_2.10:2.0.1"}, "jars": ["/livy/datanucleus-api-jdo-3.2.6.jar", "/livy/datanucleus-core-3.2.10.jar", "/livy/datanucleus-rdbms-3.2.9.jar"], "files": ["/livy/hive-site.xml"]}' -H "Content-Type: application/json" } - public Map<String, String> genPartitions(String[] PatternItemSet, String[] partitionItemSet, long timestamp, int comparableSizeMin) { + public Map<String, String> genPartitions(String[] patternItemSet, String[] partitionItemSet, long timestamp) { + int comparableSizeMin=Math.min(patternItemSet.length,partitionItemSet.length); Map<String, String> res = new HashMap<>(); for (int i = 0; i < comparableSizeMin; i++) { /** * in order to get a standard date like 20170427 01 */ - String pattrn = PatternItemSet[i].replace("mm", "MM"); + String pattrn = patternItemSet[i].replace("mm", "MM"); pattrn = pattrn.replace("DD", "dd"); pattrn = pattrn.replace("hh", "HH"); SimpleDateFormat sdf = new SimpleDateFormat(pattrn); @@ -128,11 +127,10 @@ public class SparkSubmitJob implements Job { * @param patternItemSet * @param partitionItemSet * @param timestamp - * @param patternPartitionSizeMin * @return */ - private void setDataConnectorPartitions(DataConnector dc, String[] patternItemSet, String[] partitionItemSet, long timestamp, int patternPartitionSizeMin) { - Map<String, String> partitionItemMap = genPartitions(patternItemSet, partitionItemSet, timestamp, patternPartitionSizeMin); + public void setDataConnectorPartitions(DataConnector dc, String[] patternItemSet, String[] partitionItemSet, long timestamp) { + Map<String, String> partitionItemMap = genPartitions(patternItemSet, partitionItemSet, timestamp); String partitions = partitionItemMap.toString().substring(1, partitionItemMap.toString().length() - 1); Map<String, String> configMap = dc.getConfig(); @@ -193,7 +191,7 @@ public class SparkSubmitJob implements Job { sparkJobDO.setJars(jars); List<String> files = new ArrayList<>(); -// files.add(props.getProperty("sparkJob.files_1")); + files.add(props.getProperty("sparkJob.files_1")); sparkJobDO.setFiles(files); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/service/GriffinController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/service/GriffinController.java b/service/src/main/java/org/apache/griffin/core/service/GriffinController.java index 7a5b5ad..889b441 100644 --- a/service/src/main/java/org/apache/griffin/core/service/GriffinController.java +++ b/service/src/main/java/org/apache/griffin/core/service/GriffinController.java @@ -1,9 +1,16 @@ package org.apache.griffin.core.service; +import org.apache.griffin.core.measure.repo.MeasureRepo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + @RestController public class GriffinController { @@ -13,5 +20,18 @@ public class GriffinController { return "0.1.0"; } + private static final Logger log = LoggerFactory.getLogger(GriffinController.class); + @Autowired + MeasureRepo measureRepo; + + @RequestMapping("/org") + public List<String> getOrg(){ + return measureRepo.findOrganizations(); + } + + @RequestMapping("/org/{org}") + public List<String> getMetricNameListByOrg(@PathVariable("org") String org){ + return measureRepo.findNameByOrganization(org); + } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java b/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java new file mode 100644 index 0000000..0aeb783 --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java @@ -0,0 +1,63 @@ +package org.apache.griffin.core.measure; + +import java.lang.annotation.Annotation; +import java.util.Arrays; +import java.util.List; + +/** + * Created by xiangrchen on 5/10/17. + */ +public class AssertAnnotations { + private static void assertAnnotations( + List<Class> annotationClasses, List<Annotation> annotations) { + // length + if (annotationClasses.size() != annotations.size()) { + throw new AssertionError( + String.format("Expected %d annotations, but found %d" + , annotationClasses.size(), annotations.size() + )); + } + // exists + annotationClasses.forEach( + ac -> { + long cnt + = annotations.stream() + .filter(a -> a.annotationType().isAssignableFrom(ac)) + .count(); + if (cnt == 0) { + throw new AssertionError( + String.format("No annotation of type %s found", ac.getName()) + ); + } + } + ); + } + public static void assertType(Class c, Class... annotationClasses) { + assertAnnotations( + Arrays.asList(annotationClasses) + , Arrays.asList(c.getAnnotations()) + ); + } + public static void assertField( + Class c, String fieldName, Class... annotationClasses) { + try { + assertAnnotations( + Arrays.asList(annotationClasses) + , Arrays.asList(c.getDeclaredField(fieldName).getAnnotations()) + ); + } catch (NoSuchFieldException nsfe) { + throw new AssertionError(nsfe); + } + } + public static void assertMethod( + Class c, String getterName, Class...annotationClasses) { + try { + assertAnnotations( + Arrays.asList(annotationClasses) + , Arrays.asList(c.getDeclaredMethod(getterName).getAnnotations()) + ); + } catch (NoSuchMethodException nsfe) { + throw new AssertionError(nsfe); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java b/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java new file mode 100644 index 0000000..c5da3c8 --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java @@ -0,0 +1,43 @@ +package org.apache.griffin.core.measure; + +import org.junit.Assert; +import org.junit.Test; + +import javax.persistence.Entity; + + +/** + * Created by xiangrchen on 5/10/17. + */ +public class DataConnectorTest { + @Test + public void typeAnnotations() { + // assert + AssertAnnotations.assertType( + DataConnector.class, Entity.class); + } + + @Test + public void fieldAnnotations() { + // assert + AssertAnnotations.assertField(DataConnector.class, "version"); + AssertAnnotations.assertField(DataConnector.class, "config"); + } + + @Test + public void entity() { + // setup + Entity a = ReflectTool.getClassAnnotation(DataConnector.class, Entity.class); + // assert + Assert.assertEquals("", a.name()); + } + +// @Test +// public void config() { +// // setup +// Column c = ReflectTool.getMethodAnnotation( +// DataConnector.class, "getConfig", Column.class); +// // assert +// Assert.assertEquals("config", c.name()); +// } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java b/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java new file mode 100644 index 0000000..123b550 --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java @@ -0,0 +1,33 @@ +package org.apache.griffin.core.measure; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +/** + * Created by xiangrchen on 5/10/17. + */ +public class ReflectTool { + public static <T extends Annotation> T getMethodAnnotation( + Class<?> c, String methodName, Class<T> annotation) { + try { + Method m = c.getDeclaredMethod(methodName); + return (T)m.getAnnotation(annotation); + } catch (NoSuchMethodException nsme) { + throw new RuntimeException(nsme); + } + } + public static <T extends Annotation> T getFieldAnnotation( + Class<?> c, String fieldName, Class<T> annotation) { + try { + Field f = c.getDeclaredField(fieldName); + return (T)f.getAnnotation(annotation); + } catch (NoSuchFieldException nsme) { + throw new RuntimeException(nsme); + } + } + public static <T extends Annotation> T getClassAnnotation( + Class<?> c, Class<T> annotation) { + return (T) c.getAnnotation(annotation); + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java b/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java new file mode 100644 index 0000000..e5223bc --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java @@ -0,0 +1,52 @@ +package org.apache.griffin.core.metastore; + +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.junit.Before; +import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.Mockito.*; +import static org.springframework.test.util.AssertionErrors.assertEquals; + +/** + * Created by xiangrchen on 5/10/17. + */ +public class HiveMetastoreServiceTest { + + HiveMetastoreService hiveMetastoreService; + @Before + public void setup() throws NoSuchFieldException, IllegalAccessException { + Field defaultDbName = HiveMetastoreService.class.getDeclaredField("defaultDbName"); + defaultDbName.setAccessible(true); + hiveMetastoreService=new HiveMetastoreService(); + defaultDbName.set(hiveMetastoreService,"default"); + hiveMetastoreService.client=mock(HiveMetaStoreClient.class); + } + + @Test + public void test_getUseDbName() throws Exception { + String dbName="someDbName"; + String result = Whitebox.invokeMethod(hiveMetastoreService, "getUseDbName", dbName); + assertEquals("success",result,dbName); + + dbName=""; + result = Whitebox.invokeMethod(hiveMetastoreService, "getUseDbName", dbName); + assertEquals("success",result,"default"); + } + + @Test + public void test_getAllDatabases() throws MetaException { + List<String> res= new ArrayList<>(); + when(hiveMetastoreService.client.getAllDatabases()).thenReturn(res); + hiveMetastoreService.getAllDatabases(); + verify(hiveMetastoreService.client).getAllDatabases(); + + when(hiveMetastoreService.client.getAllDatabases()).thenThrow(new MetaException()); + hiveMetastoreService.getAllDatabases(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java b/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java new file mode 100644 index 0000000..103de3c --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java @@ -0,0 +1,71 @@ +package org.apache.griffin.core.metastore; + +import org.junit.Before; +import org.junit.Test; +import org.powermock.reflect.Whitebox; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; + +import static org.springframework.test.util.AssertionErrors.assertEquals; + +/** + * Created by xiangrchen on 5/9/17. + */ +//@RunWith(SpringJUnit4ClassRunner.class) +//@ContextConfiguration +public class KafkaSchemaServiceTest { + + private static final Logger log = LoggerFactory.getLogger(KafkaSchemaServiceTest.class); +// @Value("${kafka.schema.registry.url}") +// private String url; + + private KafkaSchemaService kafkaSchemaService; + + @Before + public void setup() throws NoSuchFieldException, IllegalAccessException { + Field url = KafkaSchemaService.class.getDeclaredField("url"); + url.setAccessible(true); + kafkaSchemaService=new KafkaSchemaService(); + url.set(kafkaSchemaService, "http://localhost:8080";); + } + + @Test + public void test_registryUrl() throws Exception { + String path="/user/id"; + String result = Whitebox.invokeMethod(kafkaSchemaService, "registryUrl", path); + assertEquals("success",result,"http://localhost:8080"+path); + + path="user/id"; + result = Whitebox.invokeMethod(kafkaSchemaService, "registryUrl", path); + assertEquals("success",result,"http://localhost:8080"+"/"+path); + + path=""; + result = Whitebox.invokeMethod(kafkaSchemaService, "registryUrl", path); + assertEquals("success",result,path); + } + +// @Test +// public void test_getSchemaString(){ +// int id=1; +// RestTemplate restTemplate =mock(RestTemplate.class); +// ResponseEntity<SchemaString> mockRes =(ResponseEntity<SchemaString>)mock(ResponseEntity.class); +// String regUrl="http://10.65.159.119:8081"+"/schemas/ids/"+id; +// when(restTemplate.getForEntity(regUrl, SchemaString.class)).thenReturn(mockRes); +// SchemaString result=new SchemaString(); +// when(mockRes.getBody()).thenReturn(result); +// kafkaSchemaService.getSchemaString(id); +// } + +// @Configuration +// @ComponentScan("org.apache.griffin.core") +// static class MyServiceConfiguration { +// @Bean +// PropertyPlaceholderConfigurer propConfig() { +// PropertyPlaceholderConfigurer ppc = new PropertyPlaceholderConfigurer(); +// ppc.setLocation(new ClassPathResource("application.properties")); +// return ppc; +// } +// } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java b/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java new file mode 100644 index 0000000..ea9c50d --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java @@ -0,0 +1,144 @@ +package org.apache.griffin.core.schedule; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.griffin.core.measure.DataConnector; +import org.apache.griffin.core.measure.EvaluateRule; +import org.apache.griffin.core.measure.Measure; +import org.apache.griffin.core.measure.repo.MeasureRepo; +import org.junit.Before; +import org.junit.Test; +import org.quartz.JobDataMap; +import org.quartz.JobDetail; +import org.quartz.JobExecutionContext; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.*; + +/** + * Created by xiangrchen on 5/8/17. + */ +//@RunWith(SpringJUnit4ClassRunner.class) +//@ContextConfiguration(locations = {"classpath:context.xml"}) +public class SparkSubmitJobTest { + + private SparkSubmitJob ssj; + + @Autowired + MeasureRepo measureRepo; + + @Before + public void setUp() throws IOException { + ssj=new SparkSubmitJob(); + ssj.measureRepo=mock(MeasureRepo.class); + } + + @Test + public void test_execute() throws JsonProcessingException { + JobExecutionContext context=mock(JobExecutionContext.class); + JobDetail jd = mock(JobDetail.class); + when(context.getJobDetail()).thenReturn(jd); + + JobDataMap jdmap = mock(JobDataMap.class); + when(jd.getJobDataMap()).thenReturn(jdmap); + + when(jdmap.getString("measure")).thenReturn("bevssoj"); + when(jdmap.getString("sourcePat")).thenReturn("YYYYMMDD-HH"); + when(jdmap.getString("targetPat")).thenReturn("YYYYMMDD-HH"); + when(jdmap.getString("dataStartTimestamp")).thenReturn("1460174400000"); + when(jdmap.getString("lastTime")).thenReturn(""); + when(jdmap.getString("periodTime")).thenReturn("10"); + + HashMap<String,String> configMap1=new HashMap<>(); + configMap1.put("database","default"); + configMap1.put("table.name","test_data_src"); + HashMap<String,String> configMap2=new HashMap<>(); + configMap2.put("database","default"); + configMap2.put("table.name","test_data_tgt"); + String configJson1 = new ObjectMapper().writeValueAsString(configMap1); + String configJson2 = new ObjectMapper().writeValueAsString(configMap2); + DataConnector source = new DataConnector(DataConnector.ConnectorType.HIVE, "1.2", configJson1); + DataConnector target = new DataConnector(DataConnector.ConnectorType.HIVE, "1.2", configJson2); + String rules = "$source.uage > 100 AND $source.uid = $target.uid AND $source.uage + 12 = $target.uage + 10 + 2 AND $source.udes + 11 = $target.udes + 1 + 1"; + EvaluateRule eRule = new EvaluateRule(1,rules); + Measure measure = new Measure("bevssoj","bevssoj description", Measure.MearuseType.accuracy, "bullyeye", source, target, eRule,"test1"); + + when(ssj.measureRepo.findByName("bevssoj")).thenReturn(measure); +// ssj.execute(context); + + RestTemplate restTemplate =mock(RestTemplate.class); +// String uri="http://10.9.246.187:8998/batches";; + String uri=""; + SparkJobDO sparkJobDO=mock(SparkJobDO.class); + when(restTemplate.postForObject(uri, sparkJobDO, String.class)).thenReturn(null); + + + long currentSystemTimestamp=System.currentTimeMillis(); + long currentTimstamp = ssj.setCurrentTimestamp(currentSystemTimestamp); + +// verify(ssj.measureRepo).findByName("bevssoj"); +// verify(jdmap,atLeast(2)).put("lastTime",currentTimstamp+""); + } + + @Test + public void test_genPartitions(){ + String[] patternItemSet={"YYYYMMDD","HH"}; + String[] partitionItemSet={"date","hour"}; + long timestamp=1460174400000l; + Map<String,String> par=ssj.genPartitions(patternItemSet,partitionItemSet,timestamp); + Map<String,String> verifyMap=new HashMap<>(); + verifyMap.put("date","20160409"); + verifyMap.put("hour","12"); + assertEquals(verifyMap,par); + } + + @Test + public void test_setDataConnectorPartitions(){ + DataConnector dc=mock(DataConnector.class); + String[] patternItemSet={"YYYYMMDD","HH"}; + String[] partitionItemSet={"date","hour"}; + long timestamp=1460174400000l; + ssj.setDataConnectorPartitions(dc,patternItemSet,partitionItemSet,timestamp); +// doNothing().when(ssj).setDataConnectorPartitions(dataConnector,patternItemSet,partitionItemSet,timestamp); + Map<String,String> map=new HashMap<>(); + map.put("partitions","date=20160409, hour=12"); + try { + verify(dc).setConfig(map); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + } + + @Test + public void test_setCurrentTimestamp(){ + long timestamp=System.currentTimeMillis(); + ssj.eachJoblastTimestamp=""; + System.out.println(ssj.setCurrentTimestamp(timestamp)); + ssj.eachJoblastTimestamp="1494297256667"; + ssj.periodTime="10"; + System.out.println(ssj.setCurrentTimestamp(timestamp)); + } + + @Test + public void test_setSparkJobDO(){ + ssj=mock(SparkSubmitJob.class); + doNothing().when(ssj).setSparkJobDO(); + } + + @Test + public void test_getsparkJobProperties(){ + ssj=mock(SparkSubmitJob.class); + try { + when(ssj.getsparkJobProperties()).thenReturn(null); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/service/src/test/resources/application.properties b/service/src/test/resources/application.properties new file mode 100644 index 0000000..a344a05 --- /dev/null +++ b/service/src/test/resources/application.properties @@ -0,0 +1,22 @@ +spring.datasource.url= jdbc:mysql://localhost:3306/metastore?autoReconnect=true&useSSL=false +spring.datasource.username =griffin +spring.datasource.password =123456 + +spring.datasource.driver-class-name=com.mysql.jdbc.Driver + +## Hibernate ddl auto (validate,create, create-drop, update) + +spring.jpa.hibernate.ddl-auto = update +spring.jpa.show-sql=true +spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQL5Dialect +# +# +## Naming strategy +spring.jpa.hibernate.naming-strategy = org.hibernate.cfg.ImprovedNamingStrategy + +# hive metastore +hive.metastore.uris = thrift://10.9.246.187:9083 +hive.metastore.dbname = default + +# kafka schema registry +kafka.schema.registry.url = http://10.65.159.119:8081 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/resources/context.xml ---------------------------------------------------------------------- diff --git a/service/src/test/resources/context.xml b/service/src/test/resources/context.xml new file mode 100644 index 0000000..8d81ef8 --- /dev/null +++ b/service/src/test/resources/context.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache + License, Version 2.0 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. --> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:mvc="http://www.springframework.org/schema/mvc"; + xmlns:context="http://www.springframework.org/schema/context"; + xmlns:task="http://www.springframework.org/schema/task"; + xsi:schemaLocation=" + http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd + http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd + http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd + http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd + http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-3.1.xsd";> + + + <!-- DispatcherServlet Context: defines this servlet's request-processing + infrastructure --> + + <context:annotation-config /> + + + <context:component-scan base-package="org.apache.griffin.core" /> + + +</beans>
