[hive] branch master updated: HIVE-21009: Adding ability for user to set bind user (David McGinnis reviewed by Prasanth Jayachandran) (addendum)
This is an automated email from the ASF dual-hosted git repository. prasanthj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 6508716 HIVE-21009: Adding ability for user to set bind user (David McGinnis reviewed by Prasanth Jayachandran) (addendum) 6508716 is described below commit 650871623ef435d359c07d4c90d76c439c45e976 Author: David McGinnis <656337+davidov...@users.noreply.github.com> AuthorDate: Wed Feb 6 22:22:20 2019 -0800 HIVE-21009: Adding ability for user to set bind user (David McGinnis reviewed by Prasanth Jayachandran) (addendum) --- service/src/test/resources/creds/test.jceks | Bin 0 -> 534 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/service/src/test/resources/creds/test.jceks b/service/src/test/resources/creds/test.jceks new file mode 100755 index 000..8d58c41 Binary files /dev/null and b/service/src/test/resources/creds/test.jceks differ
[hive] branch master updated: HIVE-21009: Adding ability for user to set bind user (David McGinnis reviewed by Prasanth Jayachandran)
This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0e4d16b HIVE-21009: Adding ability for user to set bind user (David
McGinnis reviewed by Prasanth Jayachandran)
0e4d16b is described below
commit 0e4d16b462bf9abd7ec58e60936e24ee4302736c
Author: David McGinnis <656337+davidov...@users.noreply.github.com>
AuthorDate: Wed Feb 6 14:52:16 2019 -0800
HIVE-21009: Adding ability for user to set bind user (David McGinnis
reviewed by Prasanth Jayachandran)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 10 ++
service/pom.xml| 11 ++
.../auth/LdapAuthenticationProviderImpl.java | 32 +-
.../auth/TestLdapAuthenticationProviderImpl.java | 113 +
4 files changed, 164 insertions(+), 2 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index a3b03ca..2156ff1 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3499,6 +3499,16 @@ public class HiveConf extends Configuration {
"For example:
(&(objectClass=group)(objectClass=top)(instanceType=4)(cn=Domain*)) \n" +
"(&(objectClass=person)(|(sAMAccountName=admin)(|(memberOf=CN=Domain
Admins,CN=Users,DC=domain,DC=com)" +
"(memberOf=CN=Administrators,CN=Builtin,DC=domain,DC=com"),
+
HIVE_SERVER2_PLAIN_LDAP_BIND_USER("hive.server2.authentication.ldap.binddn",
null,
+"The user with which to bind to the LDAP server, and search for the
full domain name " +
+"of the user being authenticated.\n" +
+"This should be the full domain name of the user, and should have
search access across all " +
+"users in the LDAP tree.\n" +
+"If not specified, then the user being authenticated will be used as
the bind user.\n" +
+"For example: CN=bindUser,CN=Users,DC=subdomain,DC=domain,DC=com"),
+
HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD("hive.server2.authentication.ldap.bindpw",
null,
+"The password for the bind user, to be used to search for the full
name of the user being authenticated.\n" +
+"If the username is specified, this parameter must also be
specified."),
HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class",
null,
"Custom authentication class. Used when property\n" +
"'hive.server2.authentication' is set to 'CUSTOM'. Provided class\n" +
diff --git a/service/pom.xml b/service/pom.xml
index eca6f3b..30b7398 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -36,6 +36,17 @@
org.apache.hive
+ hive-common
+ ${project.version}
+
+
+ org.eclipse.jetty.aggregate
+ jetty-all
+
+
+
+
+ org.apache.hive
hive-exec
${project.version}
diff --git
a/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
b/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
index 73bbb6b..0120513 100644
---
a/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
+++
b/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
@@ -18,9 +18,10 @@
package org.apache.hive.service.auth;
import javax.security.sasl.AuthenticationException;
-
+import javax.naming.NamingException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
+import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.lang.StringUtils;
@@ -68,9 +69,36 @@ public class LdapAuthenticationProviderImpl implements
PasswdAuthenticationProvi
@Override
public void Authenticate(String user, String password) throws
AuthenticationException {
DirSearch search = null;
+String bindUser =
this.conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER);
+String bindPassword = null;
+try {
+ char[] rawPassword =
this.conf.getPassword(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD.toString());
+ if (rawPassword != null) {
+bindPassword = new String(rawPassword);
+ }
+} catch (IOException e) {
+ bindPassword = null;
+}
+boolean usedBind = bindUser != null && bindPassword != null;
+if (!usedBind) {
+ // If no bind user or bind password was specified,
+ // we assume the user we are authenticating has the ability to search
+ // the LDAP tree, so we use it as the "binding" account.
+ // This is the way it worked before bind users were allowed in the LDAP
authenticator,
+ // so we keep existing systems working.
+
[hive] branch master updated: HIVE-21214 : MoveTask : Use attemptId instead of file size for deduplication of files compareTempOrDuplicateFiles() (Deepak Jaiswal, reviewed by Jason Dere)
This is an automated email from the ASF dual-hosted git repository.
djaiswal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new fae6256 HIVE-21214 : MoveTask : Use attemptId instead of file size
for deduplication of files compareTempOrDuplicateFiles() (Deepak Jaiswal,
reviewed by Jason Dere)
fae6256 is described below
commit fae6256ace38d106b62d3bcade7b84b51bf4e1ec
Author: Deepak Jaiswal
AuthorDate: Tue Feb 5 14:06:19 2019 -0800
HIVE-21214 : MoveTask : Use attemptId instead of file size for
deduplication of files compareTempOrDuplicateFiles() (Deepak Jaiswal, reviewed
by Jason Dere)
---
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 71 ++
1 file changed, 59 insertions(+), 12 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 8937b43..b84b052 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1308,7 +1308,7 @@ public final class Utilities {
* filename to extract taskid from
*/
public static String getTaskIdFromFilename(String filename) {
-return getIdFromFilename(filename, FILE_NAME_TO_TASK_ID_REGEX);
+return getTaskIdFromFilename(filename, FILE_NAME_TO_TASK_ID_REGEX);
}
/**
@@ -1319,10 +1319,19 @@ public final class Utilities {
* filename to extract taskid from
*/
public static String getPrefixedTaskIdFromFilename(String filename) {
-return getIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX);
+return getTaskIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX);
}
- private static String getIdFromFilename(String filename, Pattern pattern) {
+ private static String getTaskIdFromFilename(String filename, Pattern
pattern) {
+return getIdFromFilename(filename, pattern, 1);
+ }
+
+ public static int getAttemptIdFromFilename(String filename) {
+String attemptStr = getIdFromFilename(filename,
FILE_NAME_PREFIXED_TASK_ID_REGEX, 3);
+return Integer.parseInt(attemptStr.substring(1));
+ }
+
+ private static String getIdFromFilename(String filename, Pattern pattern,
int group) {
String taskId = filename;
int dirEnd = filename.lastIndexOf(Path.SEPARATOR);
if (dirEnd != -1) {
@@ -1334,7 +1343,7 @@ public final class Utilities {
LOG.warn("Unable to get task id from file name: {}. Using last component
{}"
+ " as task id.", filename, taskId);
} else {
- taskId = m.group(1);
+ taskId = m.group(group);
}
LOG.debug("TaskId for {} = {}", filename, taskId);
return taskId;
@@ -1823,10 +1832,10 @@ public final class Utilities {
private static FileStatus compareTempOrDuplicateFiles(FileSystem fs,
FileStatus file, FileStatus existingFile) throws IOException {
-// Compare the file sizes of all the attempt files for the same task, the
largest win
-// any attempt files could contain partial results (due to task failures or
-// speculative runs), but the largest should be the correct one since the
result
-// of a successful run should never be smaller than a failed/speculative
run.
+// Pick the one with mewest attempt ID. For sanity, check the file sizes
too.
+// If the file size of newest attempt is less than that for older one,
+// Throw an exception as it maybe a correctness issue causing it.
+// This breaks speculative execution if it ends prematurely.
FileStatus toDelete = null, toRetain = null;
// "LOAD .. INTO" and "INSERT INTO" commands will generate files with
@@ -1847,12 +1856,26 @@ public final class Utilities {
return existingFile;
}
-if (existingFile.getLen() >= file.getLen()) {
- toDelete = file;
+int existingFileAttemptId =
getAttemptIdFromFilename(existingFile.getPath().getName());
+int fileAttemptId = getAttemptIdFromFilename(file.getPath().getName());
+
+long existingFileSz = getFileSizeRecursively(fs, existingFile);
+long fileSz = getFileSizeRecursively(fs, file);
+// Files may come in any order irrespective of their attempt IDs
+if (existingFileAttemptId > fileAttemptId &&
+existingFileSz >= fileSz) {
+ // keep existing
toRetain = existingFile;
-} else {
- toDelete = existingFile;
+ toDelete = file;
+} else if (existingFileAttemptId < fileAttemptId &&
+existingFileSz <= fileSz) {
+ // keep file
toRetain = file;
+ toDelete = existingFile;
+} else {
+ throw new IOException(" File " + filePath +
+" with newer attempt ID " + fileAttemptId + " is smaller than the file
"
++ existingFile.getPath() + " with older attempt ID " +
existingFileAttemptId);
}
if (!fs.delete(toDelete.getPath(), true)) {
throw
[hive] branch master updated: HIVE-21199: Replace all occurences of new Byte with Byte.valueOf (Ivan Suller via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8a50f16 HIVE-21199: Replace all occurences of new Byte with
Byte.valueOf (Ivan Suller via Zoltan Haindrich)
8a50f16 is described below
commit 8a50f1699ab36d5357edc7372539d3ed5bba1dea
Author: Ivan Suller
AuthorDate: Wed Feb 6 15:31:46 2019 +0100
HIVE-21199: Replace all occurences of new Byte with Byte.valueOf (Ivan
Suller via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
---
.../java/org/apache/hive/beeline/Reflector.java| 2 +-
.../hive/hcatalog/data/TestDefaultHCatRecord.java | 78 +++---
.../hive/hcatalog/data/TestHCatRecordSerDe.java| 34 +-
.../apache/hive/hcatalog/data/TestJsonSerDe.java | 34 +-
.../ql/optimizer/ConstantPropagateProcFactory.java | 18 ++---
.../stats/annotation/StatsRulesProcFactory.java| 4 +-
.../hive/ql/udf/generic/TestGenericUDFDateAdd.java | 2 +-
.../hive/ql/udf/generic/TestGenericUDFDateSub.java | 2 +-
.../hive/ql/udf/generic/TestGenericUDFRound.java | 9 ++-
.../hadoop/hive/serde2/lazy/LazySimpleSerDe.java | 2 +-
.../serde2/avro/TestAvroLazyObjectInspector.java | 4 +-
11 files changed, 96 insertions(+), 93 deletions(-)
diff --git a/beeline/src/java/org/apache/hive/beeline/Reflector.java
b/beeline/src/java/org/apache/hive/beeline/Reflector.java
index 60bfe40..455c11c 100644
--- a/beeline/src/java/org/apache/hive/beeline/Reflector.java
+++ b/beeline/src/java/org/apache/hive/beeline/Reflector.java
@@ -108,7 +108,7 @@ class Reflector {
if (toType == String.class) {
return new String(ob.toString());
} else if (toType == Byte.class || toType == byte.class) {
- return new Byte(ob.toString());
+ return Byte.valueOf(ob.toString());
} else if (toType == Character.class || toType == char.class) {
return new Character(ob.toString().charAt(0));
} else if (toType == Short.class || toType == short.class) {
diff --git
a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java
b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java
index d57d171..b9d3f64 100644
---
a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java
+++
b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestDefaultHCatRecord.java
@@ -170,9 +170,9 @@ public class TestDefaultHCatRecord extends TestCase {
rlist.add(new byte[]{1, 2, 3});
Map mapcol = new HashMap(3);
-mapcol.put(new Short("2"), "hcat is cool");
-mapcol.put(new Short("3"), "is it?");
-mapcol.put(new Short("4"), "or is it not?");
+mapcol.put(Short.valueOf("2"), "hcat is cool");
+mapcol.put(Short.valueOf("3"), "is it?");
+mapcol.put(Short.valueOf("4"), "or is it not?");
rlist.add(mapcol);
List listcol = new ArrayList();
@@ -186,66 +186,66 @@ public class TestDefaultHCatRecord extends TestCase {
private HCatRecord[] getHCatRecords() {
List rec_1 = new ArrayList(8);
-rec_1.add(new Byte("123"));
-rec_1.add(new Short("456"));
-rec_1.add(new Integer(789));
-rec_1.add(new Long(1000L));
-rec_1.add(new Float(5.3F));
-rec_1.add(new Double(5.3D));
-rec_1.add(new Boolean(true));
-rec_1.add(new String("hcat and hadoop"));
+rec_1.add(Byte.valueOf("123"));
+rec_1.add(Short.valueOf("456"));
+rec_1.add(Integer.valueOf(789));
+rec_1.add(Long.valueOf(1000L));
+rec_1.add(Float.valueOf(5.3F));
+rec_1.add(Double.valueOf(5.3D));
+rec_1.add(Boolean.TRUE);
+rec_1.add("hcat and hadoop");
rec_1.add(null);
rec_1.add("null");
HCatRecord tup_1 = new DefaultHCatRecord(rec_1);
List rec_2 = new ArrayList(8);
-rec_2.add(new Byte("123"));
-rec_2.add(new Short("456"));
-rec_2.add(new Integer(789));
-rec_2.add(new Long(1000L));
-rec_2.add(new Float(5.3F));
-rec_2.add(new Double(5.3D));
-rec_2.add(new Boolean(true));
-rec_2.add(new String("hcat and hadoop"));
+rec_2.add(Byte.valueOf("123"));
+rec_2.add(Short.valueOf("456"));
+rec_2.add(Integer.valueOf(789));
+rec_2.add(Long.valueOf(1000L));
+rec_2.add(Float.valueOf(5.3F));
+rec_2.add(Double.valueOf(5.3D));
+rec_2.add(Boolean.TRUE);
+rec_2.add("hcat and hadoop");
rec_2.add(null);
rec_2.add("null");
HCatRecord tup_2 = new DefaultHCatRecord(rec_2);
List rec_3 = new ArrayList(10);
-rec_3.add(new Byte("123"));
-rec_3.add(new Short("456"));
-rec_3.add(new Integer(789));
-rec_3.add(new Long(1000L));
-rec_3.add(new Double(5.3D));
-rec_3.add(new String("hcat and hadoop"));
+rec_3.add(Byte.valueOf("123"));
+rec_3.add(Short.valueOf("456"));
+rec_3.add(Integer.valueOf(789));
+rec_3.add(Long.valueOf(1000L));
+
[hive] branch master updated: HIVE-20295: Remove !isNumber check after failed constant interpretation (Ivan Suller via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 269dc5d HIVE-20295: Remove !isNumber check after failed constant
interpretation (Ivan Suller via Zoltan Haindrich)
269dc5d is described below
commit 269dc5dde1e8290da93f204f2bb951bd4af40098
Author: Ivan Suller
AuthorDate: Wed Feb 6 10:48:21 2019 +0100
HIVE-20295: Remove !isNumber check after failed constant interpretation
(Ivan Suller via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
---
.../hadoop/hive/ql/parse/TypeCheckProcFactory.java | 31 +++--
.../hive/ql/parse/TestTypeCheckProcFactory.java| 146 +
.../results/clientpositive/infer_const_type.q.out | 13 +-
.../clientpositive/llap/orc_llap_counters.q.out| 11 +-
.../clientpositive/llap/orc_ppd_basic.q.out| 11 +-
.../clientpositive/llap/vectorization_0.q.out | 16 +--
.../clientpositive/parquet_vectorization_0.q.out | 14 +-
.../spark/parquet_vectorization_0.q.out| 14 +-
.../clientpositive/spark/vectorization_0.q.out | 16 +--
9 files changed, 214 insertions(+), 58 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index b49bb36..a2dd554 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -106,6 +106,7 @@ import org.apache.hive.common.util.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@@ -1345,7 +1346,8 @@ public class TypeCheckProcFactory {
return valueDesc;
}
-private static ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo,
ExprNodeDesc constChild) {
+@VisibleForTesting
+protected static ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo
colTypeInfo, ExprNodeDesc constChild) {
if (constChild instanceof ExprNodeConstantDesc) {
// Try to narrow type of constant
Object constVal = ((ExprNodeConstantDesc) constChild).getValue();
@@ -1373,32 +1375,36 @@ public class TypeCheckProcFactory {
return colTypeInfo;
}
+private static BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+throw new NumberFormatException("The given string is not a valid
number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+}
+
private static Object interpretConstantAsPrimitive(PrimitiveTypeInfo
colTypeInfo, Object constVal,
TypeInfo constTypeInfo) {
- String constTypeInfoName = constTypeInfo.getTypeName();
if (constVal instanceof Number || constVal instanceof String) {
try {
PrimitiveTypeEntry primitiveTypeEntry =
colTypeInfo.getPrimitiveTypeEntry();
if
(PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
-return (new Integer(constVal.toString()));
+return toBigDecimal(constVal.toString()).intValueExact();
} else if
(PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
-return (new Long(constVal.toString()));
+return toBigDecimal(constVal.toString()).longValueExact();
} else if
(PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
-return (new Double(constVal.toString()));
+return Double.valueOf(constVal.toString());
} else if
(PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
-return (new Float(constVal.toString()));
+return Float.valueOf(constVal.toString());
} else if
(PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
-return (new Byte(constVal.toString()));
+return toBigDecimal(constVal.toString()).byteValueExact();
} else if
(PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
-return (new Short(constVal.toString()));
+return toBigDecimal(constVal.toString()).shortValueExact();
} else if
(PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
return HiveDecimal.create(constVal.toString());
}
-} catch (NumberFormatException nfe) {
+} catch (NumberFormatException | ArithmeticException nfe) {
LOG.trace("Failed to narrow type of constant", nfe);
- if (!NumberUtils.isNumber(constVal.toString())) {
-return null;
- }
+ return null;
}
}
@@ -1419,6 +1425,7 @@ public class TypeCheckProcFactory {
// if
