[3/4] hive git commit: HIVE-19210: Create separate module for streaming ingest (Prasanth Jayachandran reviewed by Eugene Koifman)

2018-04-16 Thread prasanthj
http://git-wip-us.apache.org/repos/asf/hive/blob/67a8442b/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java
--
diff --git a/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java 
b/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java
new file mode 100644
index 000..b04e137
--- /dev/null
+++ b/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java
@@ -0,0 +1,1117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.streaming;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.LockComponentBuilder;
+import org.apache.hadoop.hive.metastore.LockRequestBuilder;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
+import org.apache.hadoop.hive.ql.DriverFactory;
+import org.apache.hadoop.hive.ql.IDriver;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hive.hcatalog.common.HCatUtil;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Information about the hive end point (i.e. table or partition) to write to.
+ * A light weight object that does NOT internally hold on to resources such as
+ * network connections. It can be stored in Hashed containers such as sets and 
hash tables.
+ */
+public class HiveEndPoint {
+  public final String metaStoreUri;
+  public final String database;
+  public final String table;
+  public final ArrayList partitionVals;
+
+
+  static final private Logger LOG = 
LoggerFactory.getLogger(HiveEndPoint.class.getName());
+
+  /**
+   *
+   * @param metaStoreUri   URI of the metastore to connect to eg: 
thrift://localhost:9083
+   * @param database   Name of the Hive database
+   * @param table  Name of table to stream to
+   * @param partitionVals  Indicates the specific partition to stream to. Can 
be null or empty List
+   *   if streaming to a table without partitions. The 
order of values in this
+   *   list must correspond exactly to the order of 
partition columns specified
+   *   during the table creation. E.g. For a table 
partitioned by
+   *   (continent string, country string), partitionVals 
could be the list
+   *   ("Asia", "India").
+   */
+  public HiveEndPoint(String metaStoreUri
+  , String database, String table, List partitionVals) {
+this.metaStoreUri = metaStoreUri;
+if (database==null) {
+  throw new IllegalArgumentException("Database cannot be null for 
HiveEndPoint");
+}
+this.database = database;
+this.table = table;
+if (table==null) {
+  throw new IllegalArgumentException("Table cannot be null for 
HiveEndPoint");
+}
+this.partitionVals = partitionVals==null ? new ArrayList()
+ : new ArrayList( 
partitionVals );
+  }
+
+
+  /**
+   * 

[3/4] hive git commit: HIVE-19210: Create separate module for streaming ingest (Prasanth Jayachandran reviewed by Eugene Koifman)

2018-04-16 Thread prasanthj
http://git-wip-us.apache.org/repos/asf/hive/blob/6bd32a0d/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java
--
diff --git a/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java 
b/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java
new file mode 100644
index 000..b04e137
--- /dev/null
+++ b/streaming/src/java/org/apache/hive/streaming/HiveEndPoint.java
@@ -0,0 +1,1117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.streaming;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.LockComponentBuilder;
+import org.apache.hadoop.hive.metastore.LockRequestBuilder;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
+import org.apache.hadoop.hive.ql.DriverFactory;
+import org.apache.hadoop.hive.ql.IDriver;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hive.hcatalog.common.HCatUtil;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Information about the hive end point (i.e. table or partition) to write to.
+ * A light weight object that does NOT internally hold on to resources such as
+ * network connections. It can be stored in Hashed containers such as sets and 
hash tables.
+ */
+public class HiveEndPoint {
+  public final String metaStoreUri;
+  public final String database;
+  public final String table;
+  public final ArrayList partitionVals;
+
+
+  static final private Logger LOG = 
LoggerFactory.getLogger(HiveEndPoint.class.getName());
+
+  /**
+   *
+   * @param metaStoreUri   URI of the metastore to connect to eg: 
thrift://localhost:9083
+   * @param database   Name of the Hive database
+   * @param table  Name of table to stream to
+   * @param partitionVals  Indicates the specific partition to stream to. Can 
be null or empty List
+   *   if streaming to a table without partitions. The 
order of values in this
+   *   list must correspond exactly to the order of 
partition columns specified
+   *   during the table creation. E.g. For a table 
partitioned by
+   *   (continent string, country string), partitionVals 
could be the list
+   *   ("Asia", "India").
+   */
+  public HiveEndPoint(String metaStoreUri
+  , String database, String table, List partitionVals) {
+this.metaStoreUri = metaStoreUri;
+if (database==null) {
+  throw new IllegalArgumentException("Database cannot be null for 
HiveEndPoint");
+}
+this.database = database;
+this.table = table;
+if (table==null) {
+  throw new IllegalArgumentException("Table cannot be null for 
HiveEndPoint");
+}
+this.partitionVals = partitionVals==null ? new ArrayList()
+ : new ArrayList( 
partitionVals );
+  }
+
+
+  /**
+   *