service update to hive 2
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/64ce1fe8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/64ce1fe8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/64ce1fe8 Branch: refs/heads/griffin-0.2.0-incubating-rc4 Commit: 64ce1fe8aab5754ac8b36aaf8ab6edf0f80444ef Parents: b99c5da Author: Lionel Liu <[email protected]> Authored: Wed Apr 18 21:48:08 2018 +0800 Committer: Lionel Liu <[email protected]> Committed: Wed Apr 18 21:48:08 2018 +0800 ---------------------------------------------------------------------- service/pom.xml | 8 +++++++- .../java/org/apache/griffin/core/job/SparkSubmitJob.java | 7 +++++-- service/src/main/resources/application.properties | 8 ++++---- service/src/main/resources/sparkJob.properties | 11 +++++------ ui/angular/src/app/service/service.service.ts | 6 +++--- 5 files changed, 24 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/64ce1fe8/service/pom.xml ---------------------------------------------------------------------- diff --git a/service/pom.xml b/service/pom.xml index e2d96ae..6a64ba1 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -36,7 +36,7 @@ under the License. <java.version>1.8</java.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <hadoop.version>2.7.1</hadoop.version> - <hive.version>1.2.1</hive.version> + <hive.version>2.2.0</hive.version> <scala.version>2.10</scala.version> <spring.boot.version>1.5.1.RELEASE</spring.boot.version> <confluent.version>3.2.0</confluent.version> @@ -141,6 +141,12 @@ under the License. <groupId>org.apache.hive</groupId> <artifactId>hive-metastore</artifactId> <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.eclipse.jetty.aggregate</groupId> + <artifactId>jetty-all</artifactId> + </exclusion> + </exclusions> </dependency> <!-- to access confluent schema registry --> http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/64ce1fe8/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java index 8a4bfe2..e66c7f3 100644 --- a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java +++ b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java @@ -193,8 +193,11 @@ public class SparkSubmitJob implements Job { private void setPropConf() { Map<String, String> conf = new HashMap<>(); - conf.put("spark.yarn.dist.files", livyConfProps.getProperty("spark.yarn.dist.files")); - livyConf.setConf(conf); + String v = livyConfProps.getProperty("spark.yarn.dist.files"); + if (!StringUtils.isEmpty(v)) { + conf.put("spark.yarn.dist.files", v); + livyConf.setConf(conf); + } } private void saveJobInstance(JobDetail jd) throws SchedulerException, IOException { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/64ce1fe8/service/src/main/resources/application.properties ---------------------------------------------------------------------- diff --git a/service/src/main/resources/application.properties b/service/src/main/resources/application.properties index c8f1992..b3e7cfa 100644 --- a/service/src/main/resources/application.properties +++ b/service/src/main/resources/application.properties @@ -17,7 +17,7 @@ # under the License. # -spring.datasource.url = jdbc:postgresql://localhost:5432/quartz?autoReconnect=true&useSSL=false +spring.datasource.url = jdbc:postgresql://10.148.181.248:35432/quartz?autoReconnect=true&useSSL=false spring.datasource.username = griffin spring.datasource.password = 123456 spring.jpa.generate-ddl=true @@ -31,7 +31,7 @@ spring.jpa.show-sql = true #spring.jpa.properties.persistence-xml-location=persistence.xml # Hive metastore -hive.metastore.uris = thrift://localhost:9083 +hive.metastore.uris = thrift://10.148.181.248:39083 hive.metastore.dbname = default hive.hmshandler.retry.attempts = 15 hive.hmshandler.retry.interval = 2000ms @@ -67,8 +67,8 @@ ldap.searchPattern = (sAMAccountName={0}) fs.defaultFS = hdfs://hdfs-default-name # elasticsearch -elasticsearch.host = localhost -elasticsearch.port = 9200 +elasticsearch.host = 10.148.181.248 +elasticsearch.port = 39200 elasticsearch.scheme = http # elasticsearch.user = user # elasticsearch.password = password \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/64ce1fe8/service/src/main/resources/sparkJob.properties ---------------------------------------------------------------------- diff --git a/service/src/main/resources/sparkJob.properties b/service/src/main/resources/sparkJob.properties index 6323914..370b5ac 100644 --- a/service/src/main/resources/sparkJob.properties +++ b/service/src/main/resources/sparkJob.properties @@ -33,17 +33,16 @@ sparkJob.driverMemory=1g sparkJob.executorMemory=1g # shouldn't config in server, but in -sparkJob.jars = hdfs:///livy/datanucleus-api-jdo-3.2.6.jar;\ - hdfs:///livy/datanucleus-core-3.2.10.jar;\ - hdfs:///livy/datanucleus-rdbms-3.2.9.jar +sparkJob.jars = -spark.yarn.dist.files = hdfs:///home/spark_conf/hive-site.xml +# +spark.yarn.dist.files = # livy # livy.uri=http://10.9.246.187:8998/batches -livy.uri=http://localhost:8998/batches +livy.uri=http://10.148.181.248:38998/batches # spark-admin # spark.uri=http://10.149.247.156:28088 # spark.uri=http://10.9.246.187:8088 -spark.uri=http://localhost:8088 \ No newline at end of file +spark.uri=http://10.148.181.248:38088 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/64ce1fe8/ui/angular/src/app/service/service.service.ts ---------------------------------------------------------------------- diff --git a/ui/angular/src/app/service/service.service.ts b/ui/angular/src/app/service/service.service.ts index ff15e1e..dcbcfaf 100644 --- a/ui/angular/src/app/service/service.service.ts +++ b/ui/angular/src/app/service/service.service.ts @@ -21,9 +21,9 @@ import { Injectable } from "@angular/core"; @Injectable() export class ServiceService { constructor() {} - public BACKEND_SERVER = 'http://10.149.247.90:38080'; + // public BACKEND_SERVER = 'http://10.149.247.90:38080'; // public BACKEND_SERVER = 'http://localhost:8080'; - //public BACKEND_SERVER = ""; + public BACKEND_SERVER = ""; public API_ROOT_PATH = "/api/v1"; public config = { @@ -109,4 +109,4 @@ export class ServiceService { this.BACKEND_SERVER + this.API_ROOT_PATH + "/notifications" } }; -} \ No newline at end of file +}
