This is an automated email from the ASF dual-hosted git repository. guoyp pushed a commit to branch griffin-1.0.0-dev in repository https://gitbox.apache.org/repos/asf/griffin.git
commit 5978a103297fc180c859f6c21d5b8a62947a2213 Author: William <[email protected]> AuthorDate: Fri Sep 30 16:07:31 2022 +0800 init griffin new data quality workflow --- anomalydetection/pom.xml | 409 +++++++++++++++++++++ .../src/main/resources/assembly/assembly.xml | 84 +++++ .../src/test/resources/application.properties | 77 ++++ core/pom.xml | 409 +++++++++++++++++++++ .../apache/griffin/core/GriffinWebApplication.java | 40 ++ core/src/main/resources/assembly/assembly.xml | 84 +++++ core/src/test/resources/application.properties | 77 ++++ dispatcher/pom.xml | 409 +++++++++++++++++++++ .../src/main/resources/assembly/assembly.xml | 84 +++++ .../src/test/resources/application.properties | 77 ++++ pom.xml | 9 +- scheduler/pom.xml | 409 +++++++++++++++++++++ scheduler/src/main/resources/assembly/assembly.xml | 84 +++++ .../src/test/resources/application.properties | 77 ++++ 14 files changed, 2325 insertions(+), 4 deletions(-) diff --git a/anomalydetection/pom.xml b/anomalydetection/pom.xml new file mode 100644 index 00000000..2a501a48 --- /dev/null +++ b/anomalydetection/pom.xml @@ -0,0 +1,409 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.griffin</groupId> + <artifactId>griffin</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + + <artifactId>anomalydetection</artifactId> + <packaging>jar</packaging> + + <name>Apache Griffin :: Anomaly Detection</name> + + <properties> + <hadoop.version>2.7.1</hadoop.version> + <hive.version>2.2.0</hive.version> + <spring.boot.version>2.1.7.RELEASE</spring.boot.version> + <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version> + <confluent.version>3.2.0</confluent.version> + <quartz.version>2.2.2</quartz.version> + <start-class>org.apache.griffin.core.GriffinWebApplication</start-class> + <powermock.version>2.0.2</powermock.version> + <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version> + <derby.version>10.14.1.0</derby.version> + <eclipselink.version>2.6.0</eclipselink.version> + <mysql.java.version>5.1.47</mysql.java.version> + <postgresql.version>42.4.1</postgresql.version> + <livy.core.version>0.3.0</livy.core.version> + <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version> + <jackson-databind.version>2.9.9.3</jackson-databind.version> + </properties> + + <repositories> + <repository> + <id>confluent</id> + <url>http://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>${spring.boot.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-log4j2</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-properties-migrator</artifactId> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-json</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-jpa</artifactId> + <exclusions> + <exclusion> + <groupId>org.hibernate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.aspectj</groupId> + <artifactId>aspectjrt</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-aspects</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.security.kerberos</groupId> + <artifactId>spring-security-kerberos-client</artifactId> + <version>${spring.security.kerberos.version}</version> + </dependency> + <!--eclipse link--> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>${postgresql.version}</version> + </dependency> + <!--<dependency>--> + <!--<groupId>mysql</groupId>--> + <!--<artifactId>mysql-connector-java</artifactId>--> + <!--<version>${mysql.java.version}</version>--> + <!--</dependency>--> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.retry</groupId> + <artifactId>spring-retry</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson-databind.version}</version> + </dependency> + + <!-- to access metastore from hive--> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <!--<scope>provided</scope>--> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>jetty-all</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>de.ruedigermoeller</groupId> + <artifactId>fst</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access Hive using JDBC --> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>servlet-api-2.5</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access confluent schema registry --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>${confluent.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!--schedule--> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context-support</artifactId> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz</artifactId> + <version>${quartz.version}</version> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz-jobs</artifactId> + <version>${quartz.version}</version> + </dependency> + + <!--livy-core--> + <dependency> + <groupId>com.cloudera.livy</groupId> + <artifactId>livy-core_2.10</artifactId> + <version>${livy.core.version}</version> + </dependency> + + <!-- test --> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-client</artifactId> + <version>${elasticsearch-rest-client.version}</version> + </dependency> + </dependencies> + <profiles> + <!--if you need mysql, please uncomment mysql-connector-java --> + <!--<profile>--> + <!--<id>mysql</id>--> + <!--<activation>--> + <!--<property>--> + <!--<name>mysql</name>--> + <!--</property>--> + <!--</activation>--> + <!--</profile>--> + <profile> + <id>dev</id> + <activation> + <property> + <name>dev</name> + </property> + </activation> + </profile> + <profile> + <id>postgresql</id> + <activation> + <activeByDefault>true</activeByDefault> + <property> + <name>prod</name> + </property> + </activation> + </profile> + </profiles> + <build> + <plugins> + <plugin> + <groupId>com.ethlo.persistence.tools</groupId> + <artifactId>eclipselink-maven-plugin</artifactId> + <version>2.7.0</version> + <executions> + <execution> + <phase>process-classes</phase> + <goals> + <goal>weave</goal> + </goals> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + </dependencies> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classifier>lib</classifier> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <version>${spring-boot-maven-plugin.version}</version> + <executions> + <execution> + <goals> + <goal>build-info</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>false</executable> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <skipAssembly>false</skipAssembly> + <outputDirectory>../target</outputDirectory> + <descriptors> + <descriptor>src/main/resources/assembly/assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/anomalydetection/src/main/resources/assembly/assembly.xml b/anomalydetection/src/main/resources/assembly/assembly.xml new file mode 100644 index 00000000..70af6651 --- /dev/null +++ b/anomalydetection/src/main/resources/assembly/assembly.xml @@ -0,0 +1,84 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd +http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 "> + <id>${project.version}</id> + <formats> + <format>tar.gz</format> + </formats> + + <includeBaseDirectory>true</includeBaseDirectory> + + <!-- Adds dependencies to zip package under lib directory --> + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>./lib</outputDirectory> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>README*</include> + <include>LICENSE*</include> + <include>NOTICE*</include> + </includes> + </fileSet> + + <fileSet> + <directory>src/main/resources</directory> + <outputDirectory>config/</outputDirectory> + <includes> + <include>env/*</include> + <include>*.xml</include> + <include>*.sql</include> + <include>*.json</include> + <include>*.properties</include> + </includes> + <excludes> + <exclude>*.txt</exclude> + <exclude>logback-test.xml</exclude> + </excludes> + </fileSet> + + <fileSet> + <directory>src/main/resources/shell</directory> + <outputDirectory>bin</outputDirectory> + <includes> + <include>*.*</include> + </includes> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </fileSet> + + <fileSet> + <directory>${project.build.directory}</directory> + <outputDirectory>./lib</outputDirectory> + <includes> + <include>*${version}.jar</include> + </includes> + </fileSet> + </fileSets> +</assembly> \ No newline at end of file diff --git a/anomalydetection/src/test/resources/application.properties b/anomalydetection/src/test/resources/application.properties new file mode 100644 index 00000000..81da26e3 --- /dev/null +++ b/anomalydetection/src/test/resources/application.properties @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +spring.jpa.generate-ddl=true +spring.datasource.driver-class-name=org.h2.Driver +spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1 +spring.datasource.username=test +spring.datasource.password=test +spring.datasource.schema=classpath:Init_quartz_h2_test.sql +spring.jpa.show-sql=true +# Hive metastore +hive.metastore.uris=thrift://localhost:39083 +hive.metastore.dbname=default +hive.hmshandler.retry.attempts=15 +hive.hmshandler.retry.interval=2000ms +# Hive cache time +cache.evict.hive.fixedRate.in.milliseconds=900000 +# Kafka schema registry +kafka.schema.registry.url=http://localhost:8081 +# Update job instance state at regular intervals +jobInstance.fixedDelay.in.milliseconds=60000 +# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds +jobInstance.expired.milliseconds=604800000 +# schedule predicate job every 5 minutes and repeat 12 times at most +#interval time unit m:minute h:hour d:day,only support these three units +predicate.job.interval=5m +predicate.job.repeat.count=12 +# external properties directory location +external.config.location= +# external BATCH or STREAMING env +external.env.location= +# login strategy ("default" or "ldap") +login.strategy=default +# ldap +ldap.url=ldap://hostname:port [email protected] +ldap.searchBase=DC=org,DC=example +ldap.searchPattern=(sAMAccountName={0}) +# elasticsearch +elasticsearch.host=localhost +elasticsearch.port=9200 +elasticsearch.scheme=http +# elasticsearch.user = user +# elasticsearch.password = password +# livy +livy.uri=http://localhost:38998/batches +livy.need.queue=false +livy.task.max.concurrent.count=20 +livy.task.submit.interval.second=3 +livy.task.appId.retry.count=3 +livy.need.kerberos=false +livy.server.auth.kerberos.principal=livy/kerberos.principal +livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file +# yarn url +yarn.uri=http://localhost:38088 + +# griffin event listener +internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook + +logging.file=logs/griffin-service.log + +spring.main.allow-bean-definition-overriding=true \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml new file mode 100644 index 00000000..57aadedf --- /dev/null +++ b/core/pom.xml @@ -0,0 +1,409 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.griffin</groupId> + <artifactId>griffin</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + + <artifactId>core</artifactId> + <packaging>jar</packaging> + + <name>Apache Griffin :: Core</name> + + <properties> + <hadoop.version>2.7.1</hadoop.version> + <hive.version>2.2.0</hive.version> + <spring.boot.version>2.1.7.RELEASE</spring.boot.version> + <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version> + <confluent.version>3.2.0</confluent.version> + <quartz.version>2.2.2</quartz.version> + <start-class>org.apache.griffin.core.GriffinWebApplication</start-class> + <powermock.version>2.0.2</powermock.version> + <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version> + <derby.version>10.14.1.0</derby.version> + <eclipselink.version>2.6.0</eclipselink.version> + <mysql.java.version>5.1.47</mysql.java.version> + <postgresql.version>42.4.1</postgresql.version> + <livy.core.version>0.3.0</livy.core.version> + <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version> + <jackson-databind.version>2.9.9.3</jackson-databind.version> + </properties> + + <repositories> + <repository> + <id>confluent</id> + <url>http://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>${spring.boot.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-log4j2</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-properties-migrator</artifactId> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-json</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-jpa</artifactId> + <exclusions> + <exclusion> + <groupId>org.hibernate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.aspectj</groupId> + <artifactId>aspectjrt</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-aspects</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.security.kerberos</groupId> + <artifactId>spring-security-kerberos-client</artifactId> + <version>${spring.security.kerberos.version}</version> + </dependency> + <!--eclipse link--> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>${postgresql.version}</version> + </dependency> + <!--<dependency>--> + <!--<groupId>mysql</groupId>--> + <!--<artifactId>mysql-connector-java</artifactId>--> + <!--<version>${mysql.java.version}</version>--> + <!--</dependency>--> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.retry</groupId> + <artifactId>spring-retry</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson-databind.version}</version> + </dependency> + + <!-- to access metastore from hive--> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <!--<scope>provided</scope>--> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>jetty-all</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>de.ruedigermoeller</groupId> + <artifactId>fst</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access Hive using JDBC --> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>servlet-api-2.5</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access confluent schema registry --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>${confluent.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!--schedule--> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context-support</artifactId> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz</artifactId> + <version>${quartz.version}</version> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz-jobs</artifactId> + <version>${quartz.version}</version> + </dependency> + + <!--livy-core--> + <dependency> + <groupId>com.cloudera.livy</groupId> + <artifactId>livy-core_2.10</artifactId> + <version>${livy.core.version}</version> + </dependency> + + <!-- test --> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-client</artifactId> + <version>${elasticsearch-rest-client.version}</version> + </dependency> + </dependencies> + <profiles> + <!--if you need mysql, please uncomment mysql-connector-java --> + <!--<profile>--> + <!--<id>mysql</id>--> + <!--<activation>--> + <!--<property>--> + <!--<name>mysql</name>--> + <!--</property>--> + <!--</activation>--> + <!--</profile>--> + <profile> + <id>dev</id> + <activation> + <property> + <name>dev</name> + </property> + </activation> + </profile> + <profile> + <id>postgresql</id> + <activation> + <activeByDefault>true</activeByDefault> + <property> + <name>prod</name> + </property> + </activation> + </profile> + </profiles> + <build> + <plugins> + <plugin> + <groupId>com.ethlo.persistence.tools</groupId> + <artifactId>eclipselink-maven-plugin</artifactId> + <version>2.7.0</version> + <executions> + <execution> + <phase>process-classes</phase> + <goals> + <goal>weave</goal> + </goals> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + </dependencies> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classifier>lib</classifier> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <version>${spring-boot-maven-plugin.version}</version> + <executions> + <execution> + <goals> + <goal>build-info</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>false</executable> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <skipAssembly>false</skipAssembly> + <outputDirectory>../target</outputDirectory> + <descriptors> + <descriptor>src/main/resources/assembly/assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java b/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java new file mode 100644 index 00000000..7c7ddb73 --- /dev/null +++ b/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ +package org.apache.griffin.core; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; + +@SpringBootApplication +@EnableScheduling +public class GriffinWebApplication { + private static final Logger LOGGER = LoggerFactory + .getLogger(GriffinWebApplication.class); + + public static void main(String[] args) { + SpringApplication.run(GriffinWebApplication.class, args); + LOGGER.info("application started"); + } + + +} diff --git a/core/src/main/resources/assembly/assembly.xml b/core/src/main/resources/assembly/assembly.xml new file mode 100644 index 00000000..70af6651 --- /dev/null +++ b/core/src/main/resources/assembly/assembly.xml @@ -0,0 +1,84 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd +http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 "> + <id>${project.version}</id> + <formats> + <format>tar.gz</format> + </formats> + + <includeBaseDirectory>true</includeBaseDirectory> + + <!-- Adds dependencies to zip package under lib directory --> + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>./lib</outputDirectory> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>README*</include> + <include>LICENSE*</include> + <include>NOTICE*</include> + </includes> + </fileSet> + + <fileSet> + <directory>src/main/resources</directory> + <outputDirectory>config/</outputDirectory> + <includes> + <include>env/*</include> + <include>*.xml</include> + <include>*.sql</include> + <include>*.json</include> + <include>*.properties</include> + </includes> + <excludes> + <exclude>*.txt</exclude> + <exclude>logback-test.xml</exclude> + </excludes> + </fileSet> + + <fileSet> + <directory>src/main/resources/shell</directory> + <outputDirectory>bin</outputDirectory> + <includes> + <include>*.*</include> + </includes> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </fileSet> + + <fileSet> + <directory>${project.build.directory}</directory> + <outputDirectory>./lib</outputDirectory> + <includes> + <include>*${version}.jar</include> + </includes> + </fileSet> + </fileSets> +</assembly> \ No newline at end of file diff --git a/core/src/test/resources/application.properties b/core/src/test/resources/application.properties new file mode 100644 index 00000000..81da26e3 --- /dev/null +++ b/core/src/test/resources/application.properties @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +spring.jpa.generate-ddl=true +spring.datasource.driver-class-name=org.h2.Driver +spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1 +spring.datasource.username=test +spring.datasource.password=test +spring.datasource.schema=classpath:Init_quartz_h2_test.sql +spring.jpa.show-sql=true +# Hive metastore +hive.metastore.uris=thrift://localhost:39083 +hive.metastore.dbname=default +hive.hmshandler.retry.attempts=15 +hive.hmshandler.retry.interval=2000ms +# Hive cache time +cache.evict.hive.fixedRate.in.milliseconds=900000 +# Kafka schema registry +kafka.schema.registry.url=http://localhost:8081 +# Update job instance state at regular intervals +jobInstance.fixedDelay.in.milliseconds=60000 +# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds +jobInstance.expired.milliseconds=604800000 +# schedule predicate job every 5 minutes and repeat 12 times at most +#interval time unit m:minute h:hour d:day,only support these three units +predicate.job.interval=5m +predicate.job.repeat.count=12 +# external properties directory location +external.config.location= +# external BATCH or STREAMING env +external.env.location= +# login strategy ("default" or "ldap") +login.strategy=default +# ldap +ldap.url=ldap://hostname:port [email protected] +ldap.searchBase=DC=org,DC=example +ldap.searchPattern=(sAMAccountName={0}) +# elasticsearch +elasticsearch.host=localhost +elasticsearch.port=9200 +elasticsearch.scheme=http +# elasticsearch.user = user +# elasticsearch.password = password +# livy +livy.uri=http://localhost:38998/batches +livy.need.queue=false +livy.task.max.concurrent.count=20 +livy.task.submit.interval.second=3 +livy.task.appId.retry.count=3 +livy.need.kerberos=false +livy.server.auth.kerberos.principal=livy/kerberos.principal +livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file +# yarn url +yarn.uri=http://localhost:38088 + +# griffin event listener +internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook + +logging.file=logs/griffin-service.log + +spring.main.allow-bean-definition-overriding=true \ No newline at end of file diff --git a/dispatcher/pom.xml b/dispatcher/pom.xml new file mode 100644 index 00000000..fce4ed45 --- /dev/null +++ b/dispatcher/pom.xml @@ -0,0 +1,409 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.griffin</groupId> + <artifactId>griffin</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + + <artifactId>dispatcher</artifactId> + <packaging>jar</packaging> + + <name>Apache Griffin :: Dispatcher</name> + + <properties> + <hadoop.version>2.7.1</hadoop.version> + <hive.version>2.2.0</hive.version> + <spring.boot.version>2.1.7.RELEASE</spring.boot.version> + <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version> + <confluent.version>3.2.0</confluent.version> + <quartz.version>2.2.2</quartz.version> + <start-class>org.apache.griffin.core.GriffinWebApplication</start-class> + <powermock.version>2.0.2</powermock.version> + <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version> + <derby.version>10.14.1.0</derby.version> + <eclipselink.version>2.6.0</eclipselink.version> + <mysql.java.version>5.1.47</mysql.java.version> + <postgresql.version>42.4.1</postgresql.version> + <livy.core.version>0.3.0</livy.core.version> + <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version> + <jackson-databind.version>2.9.9.3</jackson-databind.version> + </properties> + + <repositories> + <repository> + <id>confluent</id> + <url>http://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>${spring.boot.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-log4j2</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-properties-migrator</artifactId> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-json</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-jpa</artifactId> + <exclusions> + <exclusion> + <groupId>org.hibernate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.aspectj</groupId> + <artifactId>aspectjrt</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-aspects</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.security.kerberos</groupId> + <artifactId>spring-security-kerberos-client</artifactId> + <version>${spring.security.kerberos.version}</version> + </dependency> + <!--eclipse link--> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>${postgresql.version}</version> + </dependency> + <!--<dependency>--> + <!--<groupId>mysql</groupId>--> + <!--<artifactId>mysql-connector-java</artifactId>--> + <!--<version>${mysql.java.version}</version>--> + <!--</dependency>--> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.retry</groupId> + <artifactId>spring-retry</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson-databind.version}</version> + </dependency> + + <!-- to access metastore from hive--> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <!--<scope>provided</scope>--> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>jetty-all</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>de.ruedigermoeller</groupId> + <artifactId>fst</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access Hive using JDBC --> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>servlet-api-2.5</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access confluent schema registry --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>${confluent.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!--schedule--> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context-support</artifactId> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz</artifactId> + <version>${quartz.version}</version> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz-jobs</artifactId> + <version>${quartz.version}</version> + </dependency> + + <!--livy-core--> + <dependency> + <groupId>com.cloudera.livy</groupId> + <artifactId>livy-core_2.10</artifactId> + <version>${livy.core.version}</version> + </dependency> + + <!-- test --> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-client</artifactId> + <version>${elasticsearch-rest-client.version}</version> + </dependency> + </dependencies> + <profiles> + <!--if you need mysql, please uncomment mysql-connector-java --> + <!--<profile>--> + <!--<id>mysql</id>--> + <!--<activation>--> + <!--<property>--> + <!--<name>mysql</name>--> + <!--</property>--> + <!--</activation>--> + <!--</profile>--> + <profile> + <id>dev</id> + <activation> + <property> + <name>dev</name> + </property> + </activation> + </profile> + <profile> + <id>postgresql</id> + <activation> + <activeByDefault>true</activeByDefault> + <property> + <name>prod</name> + </property> + </activation> + </profile> + </profiles> + <build> + <plugins> + <plugin> + <groupId>com.ethlo.persistence.tools</groupId> + <artifactId>eclipselink-maven-plugin</artifactId> + <version>2.7.0</version> + <executions> + <execution> + <phase>process-classes</phase> + <goals> + <goal>weave</goal> + </goals> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + </dependencies> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classifier>lib</classifier> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <version>${spring-boot-maven-plugin.version}</version> + <executions> + <execution> + <goals> + <goal>build-info</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>false</executable> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <skipAssembly>false</skipAssembly> + <outputDirectory>../target</outputDirectory> + <descriptors> + <descriptor>src/main/resources/assembly/assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/dispatcher/src/main/resources/assembly/assembly.xml b/dispatcher/src/main/resources/assembly/assembly.xml new file mode 100644 index 00000000..70af6651 --- /dev/null +++ b/dispatcher/src/main/resources/assembly/assembly.xml @@ -0,0 +1,84 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd +http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 "> + <id>${project.version}</id> + <formats> + <format>tar.gz</format> + </formats> + + <includeBaseDirectory>true</includeBaseDirectory> + + <!-- Adds dependencies to zip package under lib directory --> + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>./lib</outputDirectory> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>README*</include> + <include>LICENSE*</include> + <include>NOTICE*</include> + </includes> + </fileSet> + + <fileSet> + <directory>src/main/resources</directory> + <outputDirectory>config/</outputDirectory> + <includes> + <include>env/*</include> + <include>*.xml</include> + <include>*.sql</include> + <include>*.json</include> + <include>*.properties</include> + </includes> + <excludes> + <exclude>*.txt</exclude> + <exclude>logback-test.xml</exclude> + </excludes> + </fileSet> + + <fileSet> + <directory>src/main/resources/shell</directory> + <outputDirectory>bin</outputDirectory> + <includes> + <include>*.*</include> + </includes> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </fileSet> + + <fileSet> + <directory>${project.build.directory}</directory> + <outputDirectory>./lib</outputDirectory> + <includes> + <include>*${version}.jar</include> + </includes> + </fileSet> + </fileSets> +</assembly> \ No newline at end of file diff --git a/dispatcher/src/test/resources/application.properties b/dispatcher/src/test/resources/application.properties new file mode 100644 index 00000000..81da26e3 --- /dev/null +++ b/dispatcher/src/test/resources/application.properties @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +spring.jpa.generate-ddl=true +spring.datasource.driver-class-name=org.h2.Driver +spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1 +spring.datasource.username=test +spring.datasource.password=test +spring.datasource.schema=classpath:Init_quartz_h2_test.sql +spring.jpa.show-sql=true +# Hive metastore +hive.metastore.uris=thrift://localhost:39083 +hive.metastore.dbname=default +hive.hmshandler.retry.attempts=15 +hive.hmshandler.retry.interval=2000ms +# Hive cache time +cache.evict.hive.fixedRate.in.milliseconds=900000 +# Kafka schema registry +kafka.schema.registry.url=http://localhost:8081 +# Update job instance state at regular intervals +jobInstance.fixedDelay.in.milliseconds=60000 +# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds +jobInstance.expired.milliseconds=604800000 +# schedule predicate job every 5 minutes and repeat 12 times at most +#interval time unit m:minute h:hour d:day,only support these three units +predicate.job.interval=5m +predicate.job.repeat.count=12 +# external properties directory location +external.config.location= +# external BATCH or STREAMING env +external.env.location= +# login strategy ("default" or "ldap") +login.strategy=default +# ldap +ldap.url=ldap://hostname:port [email protected] +ldap.searchBase=DC=org,DC=example +ldap.searchPattern=(sAMAccountName={0}) +# elasticsearch +elasticsearch.host=localhost +elasticsearch.port=9200 +elasticsearch.scheme=http +# elasticsearch.user = user +# elasticsearch.password = password +# livy +livy.uri=http://localhost:38998/batches +livy.need.queue=false +livy.task.max.concurrent.count=20 +livy.task.submit.interval.second=3 +livy.task.appId.retry.count=3 +livy.need.kerberos=false +livy.server.auth.kerberos.principal=livy/kerberos.principal +livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file +# yarn url +yarn.uri=http://localhost:38088 + +# griffin event listener +internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook + +logging.file=logs/griffin-service.log + +spring.main.allow-bean-definition-overriding=true \ No newline at end of file diff --git a/pom.xml b/pom.xml index d2d4ffee..119d0af4 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ under the License. <groupId>org.apache.griffin</groupId> <artifactId>griffin</artifactId> - <version>0.7.0-SNAPSHOT</version> + <version>1.0.0-SNAPSHOT</version> <packaging>pom</packaging> <name>Apache Griffin ${project.version}</name> <url>http://griffin.apache.org</url> @@ -97,9 +97,10 @@ under the License. <modules> - <module>ui</module> - <module>service</module> - <module>measure</module> + <module>core</module> + <module>dispatcher</module> + <module>scheduler</module> + <module>anomalydetection</module> </modules> <dependencyManagement> diff --git a/scheduler/pom.xml b/scheduler/pom.xml new file mode 100644 index 00000000..03a88ff6 --- /dev/null +++ b/scheduler/pom.xml @@ -0,0 +1,409 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.griffin</groupId> + <artifactId>griffin</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + + <artifactId>scheduler</artifactId> + <packaging>jar</packaging> + + <name>Apache Griffin :: Scheduler</name> + + <properties> + <hadoop.version>2.7.1</hadoop.version> + <hive.version>2.2.0</hive.version> + <spring.boot.version>2.1.7.RELEASE</spring.boot.version> + <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version> + <confluent.version>3.2.0</confluent.version> + <quartz.version>2.2.2</quartz.version> + <start-class>org.apache.griffin.core.GriffinWebApplication</start-class> + <powermock.version>2.0.2</powermock.version> + <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version> + <derby.version>10.14.1.0</derby.version> + <eclipselink.version>2.6.0</eclipselink.version> + <mysql.java.version>5.1.47</mysql.java.version> + <postgresql.version>42.4.1</postgresql.version> + <livy.core.version>0.3.0</livy.core.version> + <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version> + <jackson-databind.version>2.9.9.3</jackson-databind.version> + </properties> + + <repositories> + <repository> + <id>confluent</id> + <url>http://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>${spring.boot.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-log4j2</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-properties-migrator</artifactId> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-json</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-jpa</artifactId> + <exclusions> + <exclusion> + <groupId>org.hibernate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.aspectj</groupId> + <artifactId>aspectjrt</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-aspects</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.security.kerberos</groupId> + <artifactId>spring-security-kerberos-client</artifactId> + <version>${spring.security.kerberos.version}</version> + </dependency> + <!--eclipse link--> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>${postgresql.version}</version> + </dependency> + <!--<dependency>--> + <!--<groupId>mysql</groupId>--> + <!--<artifactId>mysql-connector-java</artifactId>--> + <!--<version>${mysql.java.version}</version>--> + <!--</dependency>--> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.retry</groupId> + <artifactId>spring-retry</artifactId> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson-databind.version}</version> + </dependency> + + <!-- to access metastore from hive--> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <!--<scope>provided</scope>--> + <exclusions> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>jetty-all</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>de.ruedigermoeller</groupId> + <artifactId>fst</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access Hive using JDBC --> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>*</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.orbit</groupId> + <artifactId>javax.servlet</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>servlet-api-2.5</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!-- to access confluent schema registry --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>${confluent.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <!--schedule--> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context-support</artifactId> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz</artifactId> + <version>${quartz.version}</version> + </dependency> + <dependency> + <groupId>org.quartz-scheduler</groupId> + <artifactId>quartz-jobs</artifactId> + <version>${quartz.version}</version> + </dependency> + + <!--livy-core--> + <dependency> + <groupId>com.cloudera.livy</groupId> + <artifactId>livy-core_2.10</artifactId> + <version>${livy.core.version}</version> + </dependency> + + <!-- test --> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.vaadin.external.google</groupId> + <artifactId>android-json</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito2</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>${powermock.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.elasticsearch.client</groupId> + <artifactId>elasticsearch-rest-client</artifactId> + <version>${elasticsearch-rest-client.version}</version> + </dependency> + </dependencies> + <profiles> + <!--if you need mysql, please uncomment mysql-connector-java --> + <!--<profile>--> + <!--<id>mysql</id>--> + <!--<activation>--> + <!--<property>--> + <!--<name>mysql</name>--> + <!--</property>--> + <!--</activation>--> + <!--</profile>--> + <profile> + <id>dev</id> + <activation> + <property> + <name>dev</name> + </property> + </activation> + </profile> + <profile> + <id>postgresql</id> + <activation> + <activeByDefault>true</activeByDefault> + <property> + <name>prod</name> + </property> + </activation> + </profile> + </profiles> + <build> + <plugins> + <plugin> + <groupId>com.ethlo.persistence.tools</groupId> + <artifactId>eclipselink-maven-plugin</artifactId> + <version>2.7.0</version> + <executions> + <execution> + <phase>process-classes</phase> + <goals> + <goal>weave</goal> + </goals> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <version>${eclipselink.version}</version> + </dependency> + </dependencies> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classifier>lib</classifier> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <version>${spring-boot-maven-plugin.version}</version> + <executions> + <execution> + <goals> + <goal>build-info</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>false</executable> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <skipAssembly>false</skipAssembly> + <outputDirectory>../target</outputDirectory> + <descriptors> + <descriptor>src/main/resources/assembly/assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/scheduler/src/main/resources/assembly/assembly.xml b/scheduler/src/main/resources/assembly/assembly.xml new file mode 100644 index 00000000..70af6651 --- /dev/null +++ b/scheduler/src/main/resources/assembly/assembly.xml @@ -0,0 +1,84 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd +http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 "> + <id>${project.version}</id> + <formats> + <format>tar.gz</format> + </formats> + + <includeBaseDirectory>true</includeBaseDirectory> + + <!-- Adds dependencies to zip package under lib directory --> + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>./lib</outputDirectory> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>README*</include> + <include>LICENSE*</include> + <include>NOTICE*</include> + </includes> + </fileSet> + + <fileSet> + <directory>src/main/resources</directory> + <outputDirectory>config/</outputDirectory> + <includes> + <include>env/*</include> + <include>*.xml</include> + <include>*.sql</include> + <include>*.json</include> + <include>*.properties</include> + </includes> + <excludes> + <exclude>*.txt</exclude> + <exclude>logback-test.xml</exclude> + </excludes> + </fileSet> + + <fileSet> + <directory>src/main/resources/shell</directory> + <outputDirectory>bin</outputDirectory> + <includes> + <include>*.*</include> + </includes> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </fileSet> + + <fileSet> + <directory>${project.build.directory}</directory> + <outputDirectory>./lib</outputDirectory> + <includes> + <include>*${version}.jar</include> + </includes> + </fileSet> + </fileSets> +</assembly> \ No newline at end of file diff --git a/scheduler/src/test/resources/application.properties b/scheduler/src/test/resources/application.properties new file mode 100644 index 00000000..81da26e3 --- /dev/null +++ b/scheduler/src/test/resources/application.properties @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +spring.jpa.generate-ddl=true +spring.datasource.driver-class-name=org.h2.Driver +spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1 +spring.datasource.username=test +spring.datasource.password=test +spring.datasource.schema=classpath:Init_quartz_h2_test.sql +spring.jpa.show-sql=true +# Hive metastore +hive.metastore.uris=thrift://localhost:39083 +hive.metastore.dbname=default +hive.hmshandler.retry.attempts=15 +hive.hmshandler.retry.interval=2000ms +# Hive cache time +cache.evict.hive.fixedRate.in.milliseconds=900000 +# Kafka schema registry +kafka.schema.registry.url=http://localhost:8081 +# Update job instance state at regular intervals +jobInstance.fixedDelay.in.milliseconds=60000 +# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds +jobInstance.expired.milliseconds=604800000 +# schedule predicate job every 5 minutes and repeat 12 times at most +#interval time unit m:minute h:hour d:day,only support these three units +predicate.job.interval=5m +predicate.job.repeat.count=12 +# external properties directory location +external.config.location= +# external BATCH or STREAMING env +external.env.location= +# login strategy ("default" or "ldap") +login.strategy=default +# ldap +ldap.url=ldap://hostname:port [email protected] +ldap.searchBase=DC=org,DC=example +ldap.searchPattern=(sAMAccountName={0}) +# elasticsearch +elasticsearch.host=localhost +elasticsearch.port=9200 +elasticsearch.scheme=http +# elasticsearch.user = user +# elasticsearch.password = password +# livy +livy.uri=http://localhost:38998/batches +livy.need.queue=false +livy.task.max.concurrent.count=20 +livy.task.submit.interval.second=3 +livy.task.appId.retry.count=3 +livy.need.kerberos=false +livy.server.auth.kerberos.principal=livy/kerberos.principal +livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file +# yarn url +yarn.uri=http://localhost:38088 + +# griffin event listener +internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook + +logging.file=logs/griffin-service.log + +spring.main.allow-bean-definition-overriding=true \ No newline at end of file
