http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 65ed683..6d37e96 100644 --- a/pom.xml +++ b/pom.xml @@ -1,879 +1,62 @@ -<?xml version="1.0" encoding="UTF-8"?> - <!-- - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - http://www.apache.org/licenses/LICENSE-2.0 + <modelVersion>4.0.0</modelVersion> - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> + <parent> + <groupId>org.apache</groupId> + <artifactId>apache</artifactId> + <version>17</version> + </parent> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <groupId>com.ebay.oss</groupId> - <artifactId>griffin-parent</artifactId> + <groupId>com.apache.griffin</groupId> + <artifactId>griffin</artifactId> <version>0.1.0-SNAPSHOT</version> <packaging>pom</packaging> - <name>eBay Griffin Parent</name> - <modules> - <module>griffin-ui</module> - <module>griffin-core</module> - <module>griffin-models</module> - </modules> - <inceptionYear>2016</inceptionYear> - <description> - Griffin - Data Quality Service - </description> - <organization> - <name>Apache Software Foundation</name> - <url>http://apache.org</url> - </organization> - <licenses> - <license> - <name>Apache 2.0 License</name> - <url>http://www.apache.org/licenses/LICENSE-2.0.html</url> - <distribution>repo</distribution> - </license> - </licenses> - - <distributionManagement> - <snapshotRepository> - <id>ossrh</id> - <url>https://oss.sonatype.org/content/repositories/snapshots</url> - </snapshotRepository> - <repository> - <id>ossrh</id> - <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url> - </repository> - </distributionManagement> - - <repositories> - <repository> - <id>oss-sonatype</id> - <name>oss-sonatype</name> - <url>https://oss.sonatype.org/content/repositories/snapshots/</url> - <snapshots> - <enabled>true</enabled> - </snapshots> - </repository> - </repositories> - - <prerequisites> - <maven>${maven.version}</maven> - </prerequisites> - <mailingLists> - </mailingLists> - <developers> - - </developers> <properties> - <griffin.version>0.1.0-SNAPSHOT</griffin.version> - <!-- General Properties --> - <java.version>1.7</java.version> - <scala.version>2.10</scala.version> - <scala.compat.version>2.10</scala.compat.version> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - - <!-- Maven Plugins Versions --> - <maven.version>3.3.3</maven.version> - <maven-model.version>4.0.0</maven-model.version> - <maven-release.version>2.5.1</maven-release.version> - <maven-assembly.version>2.5.5</maven-assembly.version> - <maven-source.version>2.2</maven-source.version> - <maven-compiler.version>2.0.2</maven-compiler.version> - <maven-dependency.version>2.1</maven-dependency.version> - <maven-license.version>2.6</maven-license.version> - <maven-surefire.version>2.6</maven-surefire.version> - <maven-jar.version>2.6</maven-jar.version> - <maven-scala.version>2.15.0</maven-scala.version> - <maven-scalatest.version>1.0</maven-scalatest.version> - <maven-tomcat7.version>2.2</maven-tomcat7.version> - <maven-apache-rat.version>0.11</maven-apache-rat.version> - <maven-failsafe.version>2.6</maven-failsafe.version> - - <!-- Environment Versions --> - <hadoop.version>2.2.0</hadoop.version> - <!-- - <hbase.version>0.98.4.2.2.5.1-3-hadoop2</hbase.version> - <hive.version>1.2.1</hive.version> - <spark.core.version>1.4.0</spark.core.version> - --> - <!-- Common Versions - <commons-cli.version>1.2</commons-cli.version> - <commons-lang.version>2.6</commons-lang.version> - <commons-lang3.version>3.3.2</commons-lang3.version> - <commons-math3.version>3.5</commons-math3.version> - <commons-configuration.version>1.6</commons-configuration.version> - <commons-logging.version>1.1.1</commons-logging.version> - <commons-httpclient.version>3.1</commons-httpclient.version> - <commons-beanutils.version>1.8.0</commons-beanutils.version> - <commons-collections4.version>4.0</commons-collections4.version> - --> - <!-- Configuration - <archaius.version>0.6.1</archaius.version> - <typesafe.config.version>1.2.1</typesafe.config.version> - --> - <!-- Serialization --> - <gson.version>2.2.2</gson.version> - <!-- - <guava.version>14.0.1</guava.version> - <fasterxml-jackson.version>2.2.3</fasterxml-jackson.version> - <codehaus-jackson.version>1.9.13</codehaus-jackson.version> - <jsoup.version>1.7.3</jsoup.version> - <io.netty.version>3.6.7.Final</io.netty.version> - <org.json.version>20131018</org.json.version> - <protobuf-java.version>2.5.0</protobuf-java.version> - --> - - <!-- Utility --> - <slf4j.version>1.7.5</slf4j.version> -<!-- <joda-time.version>2.7</joda-time.version> - <joda-convert.version>1.7</joda-convert.version> - <log4j.version>1.2.17</log4j.version> - <log4j-over-slf4j.version>1.7.2</log4j-over-slf4j.version> - <quartz.version>2.2.1</quartz.version> - <scopt.version>3.3.0</scopt.version> --> - - <!-- Streaming - <kafka.version>0.8.1.2.2.0.0-2041</kafka.version> - <kafka-clients.version>0.8.2.0</kafka-clients.version> - <jgrapht.version>0.9.0</jgrapht.version> - <storm-kafka.version>0.9.3.2.2.0.0-2041</storm-kafka.version> - <storm.version>0.9.3</storm.version> - <curator.version>2.8.0</curator.version> - --> - <!-- Query - <tomcat.version>7.0.42</tomcat.version> - <protobuf-java.version>2.6.1</protobuf-java.version> - <antlr4.version>4.5</antlr4.version> - <goovy.version>2.3.4</goovy.version> - <parsii.version>1.3</parsii.version> ---> - <!-- JDBC - <commons-dbcp.version>1.4</commons-dbcp.version> - <javax.persistence.version>2.0.0</javax.persistence.version> - <mysql-connector-java.version>5.1.34</mysql-connector-java.version> - <ddlutils.version>1.0</ddlutils.version> - <torque.version>4.0</torque.version> - --> - <mongo.version>2.11.3</mongo.version> - - <!-- REST Service--> - <jersey.version>1.18</jersey.version> - <jaxb-api.version>2.1</jaxb-api.version> - <jaxb-impl.version>2.2.6</jaxb-impl.version> - <stax-api.version>1.0.1</stax-api.version> - <org.mortbay.jetty.version>6.1.26</org.mortbay.jetty.version> - - <!-- Alert Framework - <velocity.version>1.7</velocity.version> - <javax.mail.version>1.4</javax.mail.version> - <extcos4.version>0.4b</extcos4.version> - <extcos3.version>0.3b</extcos3.version> - <siddhi.version>3.0.2</siddhi.version> ---> - <!-- Testing --> - <unit.test.version>4.10</unit.test.version> - <guice.version>3.0</guice.version> -<!-- <junit-benchmarks.version>0.7.0</junit-benchmarks.version> - <h2.version>1.3.172</h2.version> - <scalatest.version>2.2.4</scalatest.version> - <scala-reflect.version>2.10.0</scala-reflect.version> - <mockito.version>1.8.1-rc1</mockito.version> - <tomcat.embed.version>7.0.55</tomcat.embed.version> --> - - <!-- Machine Learning - <ejml.version>0.25</ejml.version> - --> - <!-- spring security --> - <spring.framework.version>3.2.8.RELEASE</spring.framework.version> - - <mockito.version>1.10.19</mockito.version> - + <java.version>1.8</java.version> </properties> - <dependencyManagement> - <dependencies> - <!-- Commons Dependencies - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - <version>${commons-cli.version}</version> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>${commons-lang.version}</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <version>${commons-lang3.version}</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-math3</artifactId> - <version>${commons-math3.version}</version> - </dependency> - <dependency> - <groupId>commons-configuration</groupId> - <artifactId>commons-configuration</artifactId> - <version>${commons-configuration.version}</version> - </dependency> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - <version>${commons-logging.version}</version> - </dependency> - <dependency> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils</artifactId> - <version>${commons-beanutils.version}</version> - </dependency> - <dependency> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - <version>${commons-httpclient.version}</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-collections4</artifactId> - <version>${commons-collections4.version}</version> - </dependency> - <dependency> - <groupId>commons-dbcp</groupId> - <artifactId>commons-dbcp</artifactId> - <version>${commons-dbcp.version}</version> - </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-reflect</artifactId> - <version>${scala-reflect.version}</version> - </dependency> - --> - <!-- Configuration - <dependency> - <groupId>com.typesafe</groupId> - <artifactId>config</artifactId> - <version>${typesafe.config.version}</version> - </dependency> - <dependency> - <groupId>com.netflix.archaius</groupId> - <artifactId>archaius-core</artifactId> - <version>${archaius.version}</version> - </dependency> ---> - <!-- Utility --> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <version>${slf4j.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>${slf4j.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>log4j-over-slf4j</artifactId> - <version>${slf4j.version}</version> - <scope>compile</scope> - </dependency> - <!-- <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - <version>${log4j.version}</version> - </dependency> - <dependency> - <groupId>joda-time</groupId> - <artifactId>joda-time</artifactId> - <version>${joda-time.version}</version> - </dependency> - <dependency> - <groupId>org.joda</groupId> - <artifactId>joda-convert</artifactId> - <version>${joda-convert.version}</version> - </dependency> - <dependency> - <groupId>org.quartz-scheduler</groupId> - <artifactId>quartz</artifactId> - <version>${quartz.version}</version> - </dependency> - <dependency> - <groupId>com.github.scopt</groupId> - <artifactId>scopt_${scala.version}</artifactId> - <version>${scopt.version}</version> - </dependency> --> - - <!-- Serialization--> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <version>${gson.version}</version> - <scope>compile</scope> - </dependency> - <!-- - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <version>${fasterxml-jackson.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>${fasterxml-jackson.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>${fasterxml-jackson.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-scala_${scala.version}</artifactId> - <version>${fasterxml-jackson.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - <version>${codehaus-jackson.version}</version> - </dependency> - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-core-asl</artifactId> - <version>${codehaus-jackson.version}</version> - </dependency> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <version>${protobuf-java.version}</version> - </dependency> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <version>${org.json.version}</version> - </dependency> - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-jaxrs</artifactId> - <version>${codehaus-jackson.version}</version> - </dependency> - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-xc</artifactId> - <version>${codehaus-jackson.version}</version> - </dependency> - - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${guava.version}</version> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - <scope>compile</scope> - <version>${io.netty.version}</version> - </dependency> - --> - <!-- Streaming Dependencies - <dependency> - <groupId>org.jgrapht</groupId> - <artifactId>jgrapht-core</artifactId> - <version>${jgrapht.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka_${scala.version}</artifactId> - <version>${kafka.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> - <version>${kafka-clients.version}</version> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-core</artifactId> - <version>${storm.version}</version> - <exclusions> - <exclusion> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>log4j-over-slf4j</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-kafka</artifactId> - <version>${storm-kafka.version}</version> - <exclusions> - <exclusion> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>log4j-over-slf4j</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_${scala.version}</artifactId> - <version>${spark.core.version}</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-framework</artifactId> - <version>${curator.version}</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> - <version>${curator.version}</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-client</artifactId> - <version>${curator.version}</version> - </dependency> - --> - - - <!--Griffin REST Service Dependencies --> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - <version>${jersey.version}</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-json</artifactId> - <version>${jersey.version}</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-server</artifactId> - <version>${jersey.version}</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-bundle</artifactId> - <version>${jersey.version}</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - <version>${jersey.version}</version> - </dependency> - <dependency> - <groupId>com.sun.xml.bind</groupId> - <artifactId>jaxb-impl</artifactId> - <version>${jaxb-impl.version}</version> - </dependency> - <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty</artifactId> - <version>${org.mortbay.jetty.version}</version> - </dependency> - <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty-util</artifactId> - <version>${org.mortbay.jetty.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tomcat</groupId> - <artifactId>tomcat-catalina</artifactId> - <version>${tomcat.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>javax.xml.bind</groupId> - <artifactId>jaxb-api</artifactId> - <version>${jaxb-api.version}</version> - <exclusions> - <exclusion> - <groupId>javax.xml.stream</groupId> - <artifactId>stax-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>stax</groupId> - <artifactId>stax-api</artifactId> - <version>${stax-api.version}</version> - </dependency> - <dependency> - <groupId>org.jsoup</groupId> - <artifactId>jsoup</artifactId> - <version>${jsoup.version}</version> - </dependency> - - <!-- Alerting Framework - <dependency> - <groupId>javax.mail</groupId> - <artifactId>mail</artifactId> - <version>${javax.mail.version}</version> - </dependency> - <dependency> - <groupId>org.apache.velocity</groupId> - <artifactId>velocity</artifactId> - <version>${velocity.version}</version> - </dependency> - <dependency> - <groupId>org.wso2.siddhi</groupId> - <artifactId>siddhi-core</artifactId> - <version>${siddhi.version}</version> - </dependency> - <dependency> - <groupId>org.wso2.siddhi</groupId> - <artifactId>siddhi-extension-string</artifactId> - <version>${siddhi.version}</version> - </dependency> - --> - <!-- Testing Dependencies --> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>${unit.test.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <version>${mockito.version}</version> - </dependency> - - <!-- <dependency> - <groupId>com.carrotsearch</groupId> - <artifactId>junit-benchmarks</artifactId> - <version>${junit-benchmarks.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> - <version>${h2.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.version}</artifactId> - <version>${scalatest.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <version>${mockito.version}</version> - </dependency> - <dependency> - <groupId>mysql</groupId> - <artifactId>mysql-connector-java</artifactId> - <version>${mysql-connector-java.version}</version> - </dependency> - <dependency> - <groupId>org.apache.ddlutils</groupId> - <artifactId>ddlutils</artifactId> - <version>${ddlutils.version}</version> - </dependency> - <dependency> - <groupId>org.apache.torque</groupId> - <artifactId>torque-runtime</artifactId> - <version>${torque.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-core</artifactId> - <version>${tomcat.embed.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-el</artifactId> - <version>${tomcat.embed.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-jasper</artifactId> - <version>${tomcat.embed.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-logging-juli</artifactId> - <version>${tomcat.embed.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-logging-log4j</artifactId> - <version>${tomcat.embed.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-websocket</artifactId> - <version>${tomcat.embed.version}</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-test</artifactId> - <version>${curator.version}</version> - </dependency> - --> - <!-- Machine Learning - <dependency> - <groupId>com.googlecode.efficient-java-matrix-library</groupId> - <artifactId>ejml</artifactId> - <version>${ejml.version}</version> - </dependency> - --> - <!-- hadoop-2 --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - </dependency> + <name>Apache Griffin</name> + <modules> + <module>service</module> + <module>measure</module> + <module>ui</module> + </modules> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - <version>${hadoop.version}</version> - </dependency> - <!-- - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-client</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-testing-util</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - <version>${hive.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-jdbc</artifactId> - <version>${hive.version}</version> - </dependency> - --> - </dependencies> - </dependencyManagement> <profiles> <profile> - <id>hadoop-2</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-client</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-testing-util</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-exec</artifactId> - <version>${hive.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-jdbc</artifactId> - <version>${hive.version}</version> - </dependency> - </dependencies> - </dependencyManagement> + <id>prod</id> + <modules> + <module>service</module> + <module>measure</module> + <module>ui</module> + </modules> + </profile> + <profile> + <id>dev</id> + <modules> + <module>service</module> + <module>measure</module> + </modules> </profile> </profiles> - <build> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <version>${maven-apache-rat.version}</version> - <configuration> - <useDefaultExcludes>true</useDefaultExcludes> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - <version>${maven-failsafe.version}</version> - <executions> - <execution> - <goals> - <goal>integration-test</goal> - <goal>verify</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.tomcat.maven</groupId> - <artifactId>tomcat7-maven-plugin</artifactId> - <version>${maven-tomcat7.version}</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <version>${maven-source.version}</version> - </plugin> - <plugin> - <artifactId>maven-compiler-plugin</artifactId> - <version>${maven-compiler.version}</version> - <configuration> - <source>${java.version}</source> - <target>${java.version}</target> - <encoding>UTF-8</encoding> - <maxmem>2048m</maxmem> - <fork>true</fork> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${maven-surefire.version}</version> - <configuration> - <argLine>-XX:-UseGCOverheadLimit</argLine> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>${maven-jar.version}</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <version>${maven-assembly.version}</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <version>${maven-dependency.version}</version> - </plugin> - <plugin> - <groupId>org.scala-tools</groupId> - <artifactId>maven-scala-plugin</artifactId> - <version>${maven-scala.version}</version> - </plugin> - <plugin> - <groupId>org.scalatest</groupId> - <artifactId>scalatest-maven-plugin</artifactId> - <version>${maven-scalatest.version}</version> - </plugin> - <!-- deploy to sonatype.org - http://central.sonatype.org/pages/apache-maven.html - --> - <plugin> - <groupId>org.sonatype.plugins</groupId> - <artifactId>nexus-staging-maven-plugin</artifactId> - <version>1.6.7</version> - <extensions>true</extensions> - <configuration> - <serverId>ossrh</serverId> - <nexusUrl>https://oss.sonatype.org/</nexusUrl> - <autoReleaseAfterClose>true</autoReleaseAfterClose> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-gpg-plugin</artifactId> - <version>1.5</version> - <executions> - <execution> - <id>sign-artifacts</id> - <phase>verify</phase> - <goals> - <goal>sign</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </pluginManagement> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.6.1</version> + <configuration> + <source>1.8</source> + <target>1.8</target> + </configuration> + </plugin> + </plugins> </build> -</project> + +</project> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/pom.xml ---------------------------------------------------------------------- diff --git a/service/pom.xml b/service/pom.xml new file mode 100644 index 0000000..88d7a00 --- /dev/null +++ b/service/pom.xml @@ -0,0 +1,155 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.griffin</groupId> + <artifactId>service</artifactId> + <packaging>jar</packaging> + <version>0.1.0-SNAPSHOT</version> + + <name>Apache Griffin :: Web Service</name> + + <properties> + <java.version>1.8</java.version> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <hadoop.version>2.6.0</hadoop.version> + <hive.version>1.2.1</hive.version> + <scala.version>2.10</scala.version> + <spring.boot.version>1.5.1.RELEASE</spring.boot.version> + <spring.version>4.3.2.RELEASE</spring.version> + <confluent.version>3.2.0</confluent.version> + <quartz.version>2.2.1</quartz.version> + <start-class>org.apache.griffin.core.GriffinWebApplication</start-class> + </properties> + + <repositories> + <repository> + <id>confluent</id> + <url>http://packages.confluent.io/maven/</url> + </repository> + </repositories> + + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>${spring.boot.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-jpa</artifactId> + </dependency> + + <!--dev db--> + + <!--<dependency>--> + <!--<groupId>com.h2database</groupId>--> + <!--<artifactId>h2</artifactId>--> + <!--</dependency>--> + + + <!--prod db--> + <dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.6.3</version> + </dependency> + + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.8.0</version> + </dependency> + <!-- to access metastore from hive--> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <artifactId>servlet-api</artifactId> + <groupId>javax.servlet</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${hive.version}</version> + </dependency> + + <!-- to access confluent schema registry --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>${confluent.version}</version> + </dependency> + + <!--schedule--> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context-support</artifactId> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz</artifactId> + <version>2.2.2</version> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz-jobs</artifactId> + <version>2.2.2</version> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <profiles> + </profiles> + <build> + <finalName>service</finalName> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>repackage</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.6.1</version> + <configuration> + <source>1.8</source> + <target>1.8</target> + </configuration> + + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java b/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java new file mode 100644 index 0000000..7efbcf9 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java @@ -0,0 +1,68 @@ +package org.apache.griffin.core; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.griffin.core.common.SimpleCORSFilter; +import org.apache.griffin.core.measure.DataConnector; +import org.apache.griffin.core.measure.DataConnector.ConnectorType; +import org.apache.griffin.core.measure.EvaluateRule; +import org.apache.griffin.core.measure.Measure; +import org.apache.griffin.core.measure.repo.DataConnectorRepo; +import org.apache.griffin.core.measure.repo.EvaluateRuleRepo; +import org.apache.griffin.core.measure.repo.MeasureRepo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + +import java.util.HashMap; + +//import org.apache.griffin.core.measure.repo.ConnectorConfigRepo; + +@SpringBootApplication +public class GriffinWebApplication implements CommandLineRunner{ + private static final Logger log = LoggerFactory.getLogger(GriffinWebApplication.class); + public static void main(String[] args) { + log.info("application start"); + SpringApplication.run(GriffinWebApplication.class, args); + } + + @Autowired + MeasureRepo measureRepo; + @Autowired + EvaluateRuleRepo evaluateRuleRepo; + @Autowired + DataConnectorRepo connectorRepo; + + + + public void run(String... strings) throws Exception { + HashMap<String,String> configMap1=new HashMap<>(); + configMap1.put("database","default"); + configMap1.put("table.name","test_data_src"); + HashMap<String,String> configMap2=new HashMap<>(); + configMap2.put("database","default"); + configMap2.put("table.name","test_data_tgt"); + String configJson1 = new ObjectMapper().writeValueAsString(configMap1); + String configJson2 = new ObjectMapper().writeValueAsString(configMap2); + + DataConnector source = new DataConnector(ConnectorType.HIVE, "1.2", configJson1); + DataConnector target = new DataConnector(ConnectorType.HIVE, "1.2", configJson2); + + String rules = "$source.uage > 100 AND $source.uid = $target.uid AND $source.uage + 12 = $target.uage + 10 + 2 AND $source.udes + 11 = $target.udes + 1 + 1"; + + EvaluateRule eRule = new EvaluateRule(1,rules); + + Measure measure = new Measure("accu1","accu1 description", Measure.MearuseType.accuracy, "bullyeye", source, target, eRule,"test1"); + measureRepo.save(measure); + } + + @Bean + public SimpleCORSFilter simpleFilter() { + return new SimpleCORSFilter(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/common/SimpleCORSFilter.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/common/SimpleCORSFilter.java b/service/src/main/java/org/apache/griffin/core/common/SimpleCORSFilter.java new file mode 100644 index 0000000..64da4fb --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/common/SimpleCORSFilter.java @@ -0,0 +1,46 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package org.apache.griffin.core.common; +import javax.servlet.*; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +//@Component +public class SimpleCORSFilter implements Filter { + +// private final Logger log = LoggerFactory.getLogger(SimpleCORSFilter.class); +// +// public SimpleCORSFilter() { +// log.info("SimpleCORSFilter init"); +// } + + @Override + public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) throws IOException, ServletException { + HttpServletResponse response = (HttpServletResponse) res; + response.setHeader("Access-Control-Allow-Origin", "*"); + response.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, DELETE,PUT"); + response.setHeader("Access-Control-Max-Age", "3600"); + response.setHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept"); + chain.doFilter(req, res); + } + + @Override + public void init(FilterConfig filterConfig) {} + + @Override + public void destroy() {} + +} + http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/AuditableEntity.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/AuditableEntity.java b/service/src/main/java/org/apache/griffin/core/measure/AuditableEntity.java new file mode 100644 index 0000000..47283c1 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/AuditableEntity.java @@ -0,0 +1,67 @@ +package org.apache.griffin.core.measure; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.MappedSuperclass; +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Date; + +@MappedSuperclass +public abstract class AuditableEntity implements Serializable { + +// private static final long serialVersionUID = 1L; + + @Id + @GeneratedValue(strategy=GenerationType.AUTO) + private Long id; + + @JsonIgnore + Timestamp createdDate = new Timestamp(new Date().getTime()); + + @JsonIgnore + Timestamp modifiedDate; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Timestamp getCreatedDate() { return createdDate; } + public void setCreatedDate(Timestamp createdDate) { this.createdDate = createdDate; } + + public Timestamp getModifiedDate() { return modifiedDate; } + public void setModifiedDate(Timestamp modifiedDate) { this.modifiedDate = modifiedDate; } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + AuditableEntity other = (AuditableEntity) obj; + if (id == null) { + if (other.id != null) + return false; + } else if (!id.equals(other.id)) + return false; + return true; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/DataConnector.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/DataConnector.java b/service/src/main/java/org/apache/griffin/core/measure/DataConnector.java new file mode 100644 index 0000000..7eb5b44 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/DataConnector.java @@ -0,0 +1,66 @@ +package org.apache.griffin.core.measure; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import java.util.HashMap; +import java.util.Map; + +@Entity +public class DataConnector extends AuditableEntity { + + private static final long serialVersionUID = -4748881017029815794L; + + public enum ConnectorType { + HIVE + } + + @Enumerated(EnumType.STRING) + public ConnectorType type; + + public String version; + + private String config; + + public Map<String,String> getConfig() { + Map<String, String> map = new HashMap<String, String>(); + ObjectMapper mapper = new ObjectMapper(); + + try { + //convert JSON string to Map + map = mapper.readValue(config, new TypeReference<HashMap<String, String>>() {}); + } catch (Exception e) { + e.printStackTrace(); + } + return map; + } + + public void setConfig(Map<String,String> config) throws JsonProcessingException { + String configJson = new ObjectMapper().writeValueAsString(config); + this.config = configJson; + } + + public DataConnector() { + } + + public DataConnector(ConnectorType type, String version, String config) { + super(); + this.type = type; + this.version = version; + this.config = config; + } + + @Override + public String toString() { + return "DataConnector{" + + "type=" + type + + ", version='" + version + '\'' + + ", config=" + config + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/EvaluateRule.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/EvaluateRule.java b/service/src/main/java/org/apache/griffin/core/measure/EvaluateRule.java new file mode 100644 index 0000000..773e0b8 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/EvaluateRule.java @@ -0,0 +1,20 @@ + +package org.apache.griffin.core.measure; + +import javax.persistence.Entity; + +@Entity +public class EvaluateRule extends AuditableEntity { + + private static final long serialVersionUID = -3589222812544556642L; + public int sampleRatio; + public String rules; + + public EvaluateRule() { + } + + public EvaluateRule(int sampleRatio, String rules) { + this.sampleRatio = sampleRatio; + this.rules = rules; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/Measure.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/Measure.java b/service/src/main/java/org/apache/griffin/core/measure/Measure.java new file mode 100644 index 0000000..db50c6c --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/Measure.java @@ -0,0 +1,117 @@ +package org.apache.griffin.core.measure; + +import javax.persistence.*; + + +@Entity +public class Measure extends AuditableEntity { + + private static final long serialVersionUID = -4748881017029815794L; + + @Column(unique=true) + public String name; + + public String description; + + public static enum MearuseType { + accuracy, + } + + public String organization; + @Enumerated(EnumType.STRING) + public MearuseType type; + + @ManyToOne(fetch = FetchType.EAGER,cascade = {CascadeType.PERSIST, CascadeType.REMOVE}) + @JoinColumn(name = "source_id") + public DataConnector source; + + @ManyToOne(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE}) + @JoinColumn(name = "target_id") + public DataConnector target; + + @OneToOne(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE}) + @JoinColumn(name = "evaluateRule_id") + public EvaluateRule evaluateRule; + + public String owner; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getOrganization() { + return organization; + } + + public void setOrganization(String organization) { + this.organization = organization; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public MearuseType getType() { + return type; + } + + public void setType(MearuseType type) { + this.type = type; + } + + public DataConnector getSource() { + return source; + } + + public void setSource(DataConnector source) { + this.source = source; + } + + public DataConnector getTarget() { + return target; + } + + public void setTarget(DataConnector target) { + this.target = target; + } + + public EvaluateRule getEvaluateRule() { + return evaluateRule; + } + + public void setEvaluateRule(EvaluateRule evaluateRule) { + this.evaluateRule = evaluateRule; + } + + public Measure() { + } + + public Measure(String name, String description, MearuseType type, String organization, DataConnector source, DataConnector target, EvaluateRule evaluateRule, String owner) { + this.name = name; + this.description=description; + this.organization = organization; + this.type = type; + this.source = source; + this.target = target; + this.evaluateRule = evaluateRule; + this.owner = owner; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java new file mode 100644 index 0000000..21c0b93 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java @@ -0,0 +1,89 @@ +package org.apache.griffin.core.measure; + +import org.apache.griffin.core.measure.repo.MeasureRepo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +@RestController +public class MeasureController { + private static final Logger log = LoggerFactory.getLogger(MeasureController.class); + public enum Message { + success, + fail, + notFound + } + + @Autowired + MeasureRepo measureRepo; + @RequestMapping("/measures") + public Iterable<Measure> getMeasures() { + return measureRepo.findAll(); + } + + @RequestMapping("/measures/{id}") + public Measure getMeasuresById(@PathVariable("id") long id) { + return measureRepo.findOne(id); + } + + @RequestMapping("/measures/findByName/{measureName}") + public Measure getMeasuresByName(@PathVariable("measureName") String measureName) { + return measureRepo.findByName(measureName); + } + + @RequestMapping(value = "/measures/deleteById/{MeasureId}",method = RequestMethod.DELETE) + public void deleteMeasuresById(@PathVariable("MeasureId") Long MeasureId) { measureRepo.delete(MeasureId);} + + @RequestMapping(value = "/measures/deleteByName/{measureName}",method = RequestMethod.DELETE) + public Message deleteMeasuresByName(@PathVariable("measureName") String measureName) { + Message msg; + Measure temp_mesaure=measureRepo.findByName(measureName); + if(temp_mesaure==null){ + msg=Message.notFound; + } + else{ + measureRepo.delete(temp_mesaure.getId()); + msg=Message.success; + } + return msg; + } + + @RequestMapping(value = "/measures/update",method = RequestMethod.POST) + @ResponseBody + public MeasureController.Message updateMeasure(@RequestBody Measure measure) { + MeasureController.Message msg; + String name=measure.getName(); + Measure temp_mesaure=measureRepo.findByName(name); + if (temp_mesaure==null){ + msg=Message.notFound; + }else{ + //in this way, id will changed + measureRepo.delete(temp_mesaure.getId()); + measureRepo.save(measure); + msg=Message.success; + } + return msg; + } + + + @RequestMapping(value = "/measures/add", method = RequestMethod.POST) + @ResponseBody + public Message createNewMeasure(@RequestBody Measure measure) { + Message msg; + System.out.println(measure); + String name=measure.getName(); + Measure temp_mesaure=measureRepo.findByName(name); + if (temp_mesaure==null){ + if (measureRepo.save(measure)!=null) + msg=Message.success; + else{ + msg = Message.fail; + } + } else{ + log.info("Failed to create new measure "+name+", it already exists"); + return Message.fail; + } + return msg; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/repo/ConnectorConfigRepo.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/repo/ConnectorConfigRepo.java b/service/src/main/java/org/apache/griffin/core/measure/repo/ConnectorConfigRepo.java new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/repo/DataConnectorRepo.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/repo/DataConnectorRepo.java b/service/src/main/java/org/apache/griffin/core/measure/repo/DataConnectorRepo.java new file mode 100644 index 0000000..3631a86 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/repo/DataConnectorRepo.java @@ -0,0 +1,10 @@ +package org.apache.griffin.core.measure.repo; + + +import org.apache.griffin.core.measure.DataConnector; +import org.springframework.data.repository.CrudRepository; + +public interface DataConnectorRepo extends CrudRepository<DataConnector, Long> { + + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/repo/EvaluateRuleRepo.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/repo/EvaluateRuleRepo.java b/service/src/main/java/org/apache/griffin/core/measure/repo/EvaluateRuleRepo.java new file mode 100644 index 0000000..fd659ce --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/repo/EvaluateRuleRepo.java @@ -0,0 +1,10 @@ +package org.apache.griffin.core.measure.repo; + + +import org.apache.griffin.core.measure.EvaluateRule; +import org.springframework.data.repository.CrudRepository; + +public interface EvaluateRuleRepo extends CrudRepository<EvaluateRule, Long> { + + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java new file mode 100644 index 0000000..0eec264 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java @@ -0,0 +1,9 @@ +package org.apache.griffin.core.measure.repo; + + +import org.apache.griffin.core.measure.Measure; +import org.springframework.data.repository.CrudRepository; + +public interface MeasureRepo extends CrudRepository<Measure, Long> { + Measure findByName(String name); +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreController.java b/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreController.java new file mode 100644 index 0000000..3b035e9 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreController.java @@ -0,0 +1,56 @@ +package org.apache.griffin.core.metastore; + + +import org.apache.hadoop.hive.metastore.api.Table; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/metadata/hive") +public class HiveMetastoreController { + + @Autowired + HiveMetastoreService hiveMetastoreService; + + @RequestMapping("/db") + public Iterable<String> getAllDatabases() { + return hiveMetastoreService.getAllDatabases(); + } + + @RequestMapping("/table") + public Iterable<String> getDefAllTables() { + return hiveMetastoreService.getAllTableNames(""); + } + + @RequestMapping("/{db}/table") + public Iterable<String> getAllTableNamess(@PathVariable("db") String dbName) { + return hiveMetastoreService.getAllTableNames(dbName); + } + + @RequestMapping("/{db}/alltables") + public List<Table> getAllTables(@PathVariable("db") String dbName) { + return hiveMetastoreService.getAllTable(dbName); + } + + @RequestMapping("/alltables") + public Map<String,List<Table>> getAllTables() { + return hiveMetastoreService.getAllTable(); + } + + @RequestMapping("/table/{table}") + public Table getDefTable(@PathVariable("table") String tableName) { + return hiveMetastoreService.getTable("", tableName); + } + + @RequestMapping("/{db}/table/{table}") + public Table getTable(@PathVariable("db") String dbName, @PathVariable("table") String tableName) { + return hiveMetastoreService.getTable(dbName, tableName); + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreProxy.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreProxy.java b/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreProxy.java new file mode 100644 index 0000000..5939d0e --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreProxy.java @@ -0,0 +1,42 @@ +package org.apache.griffin.core.metastore; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +@Component +public class HiveMetastoreProxy +{ + private static final Logger log = LoggerFactory.getLogger(HiveMetastoreProxy.class); + + @Value("${hive.metastore.uris}") + private String uris; + + + private HiveMetaStoreClient client = null; + + @Bean + public HiveMetaStoreClient initHiveMetastoreClient(){ + HiveConf hiveConf = new HiveConf(); + hiveConf.set("hive.metastore.local", "false"); + hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); + try { + client= new HiveMetaStoreClient(hiveConf); + } catch (MetaException e) { + log.error("Failed to connect hive metastore",e.getMessage()); + client = null; + } + + return client; + } + + public void destroy() throws Exception { + if(null!=client) client.close(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreService.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreService.java b/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreService.java new file mode 100644 index 0000000..ba7ade4 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/metastore/HiveMetastoreService.java @@ -0,0 +1,105 @@ +package org.apache.griffin.core.metastore; + + +import org.apache.avro.generic.GenericData; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@Service +public class HiveMetastoreService { + + private static final Logger log = LoggerFactory.getLogger(HiveMetastoreService.class); + + @Autowired + HiveMetaStoreClient client; + + @Value("${hive.metastore.dbname}") + private String defaultDbName; + + private String getUseDbName(String dbName) { + if (!StringUtils.hasText(dbName)) return defaultDbName; + else return dbName; + } + + public Iterable<String> getAllDatabases() { + Iterable<String> results = null; + try { + results = client.getAllDatabases(); + } catch (MetaException e) { + log.error("Can not get databases : ",e.getMessage()); + } + return results; + } + + public Iterable<String> getAllTableNames(String dbName) { + Iterable<String> results = null; + String useDbName = getUseDbName(dbName); + try { + results = client.getAllTables(useDbName); + } catch (Exception e) { + log.warn("Exception fetching tables info" + e.getMessage()); + } + return results; + } + + public List<Table> getAllTable(String db){ + List<Table> results = new ArrayList<Table>(); + String useDbName = getUseDbName(db); + try { + Iterable<String> tables = client.getAllTables(useDbName); + for (String table: tables) { + Table tmp = client.getTable(db,table); + results.add(tmp); + } + } catch (Exception e) { + log.warn("Exception fetching tables info" + e.getMessage()); + } + return results; + } + + public Map<String,List<Table>> getAllTable(){ + Map<String,List<Table>> results = new HashMap<String, List<Table>>(); + Iterable<String> dbs = getAllDatabases(); + for(String db: dbs){ + List<Table> alltables = new ArrayList<Table>(); + String useDbName = getUseDbName(db); + try { + Iterable<String> tables = client.getAllTables(useDbName); + for (String table: tables) { + Table tmp = client.getTable(db,table); + alltables.add(tmp); + } + } catch (Exception e) { + log.warn("Exception fetching tables info" + e.getMessage()); + } + results.put(db,alltables); + } + return results; + } + + public Table getTable(String dbName, String tableName) { + Table result = null; + String useDbName = getUseDbName(dbName); + try { + result = client.getTable(useDbName, tableName); + } catch (Exception e) { + log.warn("Exception fetching table info : " +tableName + " : " + e.getMessage()); + } + return result; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaController.java b/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaController.java new file mode 100644 index 0000000..1491518 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaController.java @@ -0,0 +1,48 @@ +package org.apache.griffin.core.metastore; + +import io.confluent.kafka.schemaregistry.client.rest.entities.Config; +import io.confluent.kafka.schemaregistry.client.rest.entities.Schema; +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaString; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/metadata/kafka") +public class KafkaSchemaController { + + @Autowired + KafkaSchemaService kafkaSchemaService; + + @RequestMapping("/schema/{id}") + public SchemaString getSchemaString(@PathVariable("id") Integer id) { + return kafkaSchemaService.getSchemaString(id); + } + + @RequestMapping("/subject") + public Iterable<String> getSubjects() { + return kafkaSchemaService.getSubjects(); + } + + @RequestMapping("/subject/{subject}/version") + public Iterable<Integer> getSubjectVersions(@PathVariable("subject") String subject) { + return kafkaSchemaService.getSubjectVersions(subject); + } + + @RequestMapping("/subject/{subject}/version/{version}") + public Schema getSubjectSchema(@PathVariable("subject") String subject, @PathVariable("version") String version) { + return kafkaSchemaService.getSubjectSchema(subject, version); + } + + @RequestMapping("/config") + public Config getTopLevelConfig() { + return kafkaSchemaService.getTopLevelConfig(); + } + + @RequestMapping("/config/{subject}") + public Config getSubjectLevelConfig(@PathVariable("subject") String subject) { + return kafkaSchemaService.getSubjectLevelConfig(subject); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaService.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaService.java b/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaService.java new file mode 100644 index 0000000..2113312 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/metastore/KafkaSchemaService.java @@ -0,0 +1,118 @@ +package org.apache.griffin.core.metastore; + +import io.confluent.kafka.schemaregistry.client.rest.entities.Config; +import io.confluent.kafka.schemaregistry.client.rest.entities.Schema; +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaString; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; + +import java.util.Arrays; + +@Service +public class KafkaSchemaService { + + private static final Logger log = LoggerFactory.getLogger(KafkaSchemaService.class); + + @Value("${kafka.schema.registry.url}") + private String url; + + + private String registryUrl(final String path) { + if (StringUtils.hasText(path)) { + String usePath = path; + if (!path.startsWith("/")) usePath = "/" + path; + return this.url + usePath; + } + return ""; + } + + public SchemaString getSchemaString(Integer id) { + String path = "/schemas/ids/" + id; + String regUrl = registryUrl(path); + SchemaString result = null; + try { + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity<SchemaString> res = restTemplate.getForEntity(regUrl, SchemaString.class); + result = res.getBody(); + } catch (Exception e) { + log.error("Exception getting schema of id " + id + " : ", e.getMessage()); + } + return result; + } + + public Iterable<String> getSubjects() { + String path = "/subjects"; + String regUrl = registryUrl(path); + Iterable<String> result = null; + try { + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity<String[]> res = restTemplate.getForEntity(regUrl, String[].class); + result = Arrays.asList(res.getBody()); + } catch (Exception e) { + log.error("Exception getting subjects : ", e.getMessage()); + } + return result; + } + + public Iterable<Integer> getSubjectVersions(String subject) { + String path = "/subjects/" + subject + "/versions"; + String regUrl = registryUrl(path); + Iterable<Integer> result = null; + try { + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity<Integer[]> res = restTemplate.getForEntity(regUrl, Integer[].class); + result = Arrays.asList(res.getBody()); + } catch (Exception e) { + log.error("Exception getting subject " + subject + " versions : ", e.getMessage()); + } + return result; + } + + public Schema getSubjectSchema(String subject, String version) { + String path = "/subjects/" + subject + "/versions/" + version; + String regUrl = registryUrl(path); + Schema result = null; + try { + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity<Schema> res = restTemplate.getForEntity(regUrl, Schema.class); + result = res.getBody(); + } catch (Exception e) { + log.error("Exception getting subject " + subject + " with version " + version + " : ", e.getMessage()); + } + return result; + } + + public Config getTopLevelConfig() { + String path = "/config"; + String regUrl = registryUrl(path); + Config result = null; + try { + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity<Config> res = restTemplate.getForEntity(regUrl, Config.class); + result = res.getBody(); + } catch (Exception e) { + log.error("Exception getting top level config : ", e.getMessage()); + } + return result; + } + + public Config getSubjectLevelConfig(String subject) { + String path = "/config/" + subject; + String regUrl = registryUrl(path); + Config result = null; + try { + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity<Config> res = restTemplate.getForEntity(regUrl, Config.class); + result = res.getBody(); + } catch (Exception e) { + log.error("Exception getting subject " + subject + " level config : ", e.getMessage()); + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/schedule/AutowiringSpringBeanJobFactory.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/schedule/AutowiringSpringBeanJobFactory.java b/service/src/main/java/org/apache/griffin/core/schedule/AutowiringSpringBeanJobFactory.java new file mode 100644 index 0000000..8e10384 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/schedule/AutowiringSpringBeanJobFactory.java @@ -0,0 +1,26 @@ +package org.apache.griffin.core.schedule; + +import org.quartz.spi.TriggerFiredBundle; +import org.springframework.beans.factory.config.AutowireCapableBeanFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.scheduling.quartz.SpringBeanJobFactory; + +public final class AutowiringSpringBeanJobFactory extends SpringBeanJobFactory + implements ApplicationContextAware { + + private transient AutowireCapableBeanFactory beanFactory; + + @Override + public void setApplicationContext(final ApplicationContext context) { + beanFactory = context.getAutowireCapableBeanFactory(); + } + + @Override + protected Object createJobInstance(final TriggerFiredBundle bundle) + throws Exception { + final Object job = super.createJobInstance(bundle); + beanFactory.autowireBean(job); + return job; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/schedule/Conf.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/schedule/Conf.java b/service/src/main/java/org/apache/griffin/core/schedule/Conf.java new file mode 100644 index 0000000..4ccd6fd --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/schedule/Conf.java @@ -0,0 +1,16 @@ +package org.apache.griffin.core.schedule; + +/** + * Created by xiangrchen on 4/26/17. + */ +public class Conf { + String spark_jars_packages; + + public String getSpark_jars_packages() { + return spark_jars_packages; + } + + public void setSpark_jars_packages(String spark_jars_packages) { + this.spark_jars_packages = spark_jars_packages; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/service/src/main/java/org/apache/griffin/core/schedule/SchedulerConfig.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/schedule/SchedulerConfig.java b/service/src/main/java/org/apache/griffin/core/schedule/SchedulerConfig.java new file mode 100644 index 0000000..916b104 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/schedule/SchedulerConfig.java @@ -0,0 +1,44 @@ +package org.apache.griffin.core.schedule; + +import org.quartz.spi.JobFactory; +import org.springframework.beans.factory.config.PropertiesFactoryBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ClassPathResource; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; + +import javax.sql.DataSource; +import java.io.IOException; +import java.util.Properties; + +@Configuration +public class SchedulerConfig { + + @Bean + public JobFactory jobFactory(ApplicationContext applicationContext) { + AutowiringSpringBeanJobFactory jobFactory = new AutowiringSpringBeanJobFactory(); + jobFactory.setApplicationContext(applicationContext); + return jobFactory; + } + + @Bean + public SchedulerFactoryBean schedulerFactoryBean(DataSource dataSource, JobFactory jobFactory) throws IOException { + SchedulerFactoryBean factory = new SchedulerFactoryBean(); + factory.setOverwriteExistingJobs(true); + factory.setDataSource(dataSource); + factory.setJobFactory(jobFactory); + + factory.setQuartzProperties(quartzProperties()); + + return factory; + } + + @Bean + public Properties quartzProperties() throws IOException { + PropertiesFactoryBean propertiesFactoryBean = new PropertiesFactoryBean(); + propertiesFactoryBean.setLocation(new ClassPathResource("/quartz.properties")); + propertiesFactoryBean.afterPropertiesSet(); + return propertiesFactoryBean.getObject(); + } +}
