This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/2.0 by this push: new 41ba8e8 Allow kerberized run (#142) 41ba8e8 is described below commit 41ba8e83114a70fb6aa8340fc332f314d073668e Author: BukrosSzabolcs <szabo...@cloudera.com> AuthorDate: Sat Jul 10 01:52:56 2021 +0200 Allow kerberized run (#142) Add login functionality to client creation and pass the delegation token when yarn is used to allow running tests on a kerberized cluster. (security tests excluded) --- .../apache/accumulo/testing/KerberosHelper.java | 83 ++++++++++++++++++++++ .../java/org/apache/accumulo/testing/TestEnv.java | 1 + .../testing/continuous/ContinuousMoru.java | 8 ++- .../testing/continuous/ContinuousVerify.java | 8 ++- .../apache/accumulo/testing/mapreduce/RowHash.java | 9 ++- .../accumulo/testing/mapreduce/TeraSortIngest.java | 5 +- .../testing/randomwalk/multitable/CopyTool.java | 11 ++- .../randomwalk/sequential/MapRedVerifyTool.java | 10 ++- 8 files changed, 121 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/apache/accumulo/testing/KerberosHelper.java b/src/main/java/org/apache/accumulo/testing/KerberosHelper.java new file mode 100644 index 0000000..84bc325 --- /dev/null +++ b/src/main/java/org/apache/accumulo/testing/KerberosHelper.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.testing; + +import java.io.IOException; +import java.util.Properties; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.security.tokens.DelegationToken; +import org.apache.accumulo.core.conf.ClientProperty; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KerberosHelper { + private static final Logger log = LoggerFactory.getLogger(KerberosHelper.class); + + public static void saslLogin(Properties clientProps, Configuration conf) { + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); + try { + UserGroupInformation.loginUserFromKeytab( + ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), + ClientProperty.AUTH_TOKEN.getValue(clientProps)); + } catch (IOException e) { + log.warn("Sasl login failed", e); + } + } + } + + public static Properties configDelegationToken(Properties clientProps) + throws AccumuloException, AccumuloSecurityException { + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + AccumuloClient client = Accumulo.newClient().from(clientProps).build(); + DelegationToken dt = client.securityOperations().getDelegationToken(null); + clientProps = Accumulo.newClientProperties().from(clientProps) + .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build(); + client.close(); + } + return clientProps; + } + + public static Properties configDelegationToken(TestEnv env) + throws AccumuloException, AccumuloSecurityException { + Properties clientProps = env.getClientProps(); + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + AccumuloClient client = env.getAccumuloClient(); + clientProps = configDelegationToken(clientProps, client); + client.close(); + } + return clientProps; + } + + public static Properties configDelegationToken(Properties clientProps, AccumuloClient client) + throws AccumuloException, AccumuloSecurityException { + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + DelegationToken dt = client.securityOperations().getDelegationToken(null); + clientProps = Accumulo.newClientProperties().from(clientProps) + .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build(); + } + return clientProps; + } +} diff --git a/src/main/java/org/apache/accumulo/testing/TestEnv.java b/src/main/java/org/apache/accumulo/testing/TestEnv.java index d395ad4..73ee307 100644 --- a/src/main/java/org/apache/accumulo/testing/TestEnv.java +++ b/src/main/java/org/apache/accumulo/testing/TestEnv.java @@ -171,6 +171,7 @@ public class TestEnv implements AutoCloseable { */ public synchronized AccumuloClient getAccumuloClient() { if (client == null) { + KerberosHelper.saslLogin(clientProps, getHadoopConfiguration()); client = Accumulo.newClient().from(clientProps).build(); } return client; diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java index 99e2fdf..a9fb439 100644 --- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java +++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java @@ -19,6 +19,7 @@ package org.apache.accumulo.testing.continuous; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; +import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -30,6 +31,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; import org.apache.accumulo.testing.TestProps; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -123,14 +125,16 @@ public class ContinuousMoru extends Configured implements Tool { Set<Range> ranges = env.getAccumuloClient().tableOperations() .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps); - AccumuloInputFormat.configure().clientProperties(env.getClientProps()) + Properties clientProps = KerberosHelper.configDelegationToken(env); + + AccumuloInputFormat.configure().clientProperties(clientProps) .table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job); job.setMapperClass(CMapper.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); - AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true) + AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true) .defaultTable(env.getAccumuloTableName()).store(job); Configuration conf = job.getConfiguration(); diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java index 5db4350..fd05c7e 100644 --- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java +++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousVerify.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.Properties; import java.util.Random; import java.util.Set; @@ -30,6 +31,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; +import org.apache.accumulo.testing.KerberosHelper; import org.apache.accumulo.testing.TestProps; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; @@ -173,8 +175,10 @@ public class ContinuousVerify extends Configured implements Tool { table = tableName; } - AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(table) - .ranges(ranges).autoAdjustRanges(false).offlineScan(scanOffline).store(job); + Properties clientProps = KerberosHelper.configDelegationToken(env.getClientProps(), client); + + AccumuloInputFormat.configure().clientProperties(clientProps).table(table).ranges(ranges) + .autoAdjustRanges(false).offlineScan(scanOffline).store(job); job.setMapperClass(CMapper.class); job.setMapOutputKeyClass(LongWritable.class); diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java index 45e24f0..e7363ae 100644 --- a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java +++ b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java @@ -28,6 +28,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; import org.apache.accumulo.testing.TestEnv; import org.apache.accumulo.testing.TestProps; import org.apache.hadoop.conf.Configured; @@ -76,10 +77,12 @@ public class RowHash extends Configured implements Tool { String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE); String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE); - AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable) + Properties clientProps = KerberosHelper.configDelegationToken(env); + + AccumuloInputFormat.configure().clientProperties(clientProps).table(inputTable) .fetchColumns(cols).store(job); - AccumuloOutputFormat.configure().clientProperties(env.getClientProps()) - .defaultTable(outputTable).createTables(true).store(job); + AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(outputTable) + .createTables(true).store(job); job.getConfiguration().set("mapreduce.job.classloader", "true"); job.setMapperClass(HashDataMapper.class); diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java index 1961b93..8dc915b 100644 --- a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java +++ b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java @@ -30,6 +30,7 @@ import java.util.Random; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; import org.apache.accumulo.testing.TestEnv; import org.apache.accumulo.testing.TestProps; import org.apache.hadoop.conf.Configuration; @@ -376,7 +377,9 @@ public class TeraSortIngest extends Configured implements Tool { String tableName = testProps.getProperty(TestProps.TERASORT_TABLE); Objects.requireNonNull(tableName); - AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true) + Properties clientProps = KerberosHelper.configDelegationToken(env); + + AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true) .defaultTable(tableName).store(job); Configuration conf = job.getConfiguration(); diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java index 54866f2..44167a0 100644 --- a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java +++ b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java @@ -26,6 +26,8 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; @@ -56,10 +58,13 @@ public class CopyTool extends Configured implements Tool { return 1; } - Properties props = Accumulo.newClientProperties().from(args[0]).build(); job.setInputFormatClass(AccumuloInputFormat.class); - AccumuloInputFormat.configure().clientProperties(props).table(args[1]) + Properties clientProps = Accumulo.newClientProperties().from(args[0]).build(); + KerberosHelper.saslLogin(clientProps, new Configuration(false)); + clientProps = KerberosHelper.configDelegationToken(clientProps); + + AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1]) .auths(Authorizations.EMPTY).store(job); job.setMapperClass(SeqMapClass.class); @@ -69,7 +74,7 @@ public class CopyTool extends Configured implements Tool { job.getConfiguration().set("mapreduce.job.classloader", "true"); job.setOutputFormatClass(AccumuloOutputFormat.class); - AccumuloOutputFormat.configure().clientProperties(props).createTables(true) + AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true) .defaultTable(args[2]).store(job); job.waitForCompletion(true); diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java index f1e560c..454bd83 100644 --- a/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java +++ b/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java @@ -26,6 +26,8 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; @@ -89,10 +91,12 @@ public class MapRedVerifyTool extends Configured implements Tool { return 1; } - Properties props = Accumulo.newClientProperties().from(args[0]).build(); + Properties clientProps = Accumulo.newClientProperties().from(args[0]).build(); + KerberosHelper.saslLogin(clientProps, new Configuration(false)); + clientProps = KerberosHelper.configDelegationToken(clientProps); - AccumuloInputFormat.configure().clientProperties(props).table(args[1]).store(job); - AccumuloOutputFormat.configure().clientProperties(props).defaultTable(args[2]) + AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1]).store(job); + AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(args[2]) .createTables(true).store(job); job.setInputFormatClass(AccumuloInputFormat.class);