[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r202108057
--- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh ---
@@ -0,0 +1,59 @@
+#/usr/bin/env bash
+
+set -u
+set -x
+set -e
+
+function cleanup {
+set +e
+set +u
+kdestroy
+pushd ${PY_ENV_PATH}/bin
+. deactivate ""
+popd
+rm -rf $PY_ENV_PATH
+}
+
+trap cleanup EXIT
+
+echo "LAUNCHING SCRIPT"
+
+LOCAL_PY=$1
+PRINC=$2
+KEYTAB_LOC=$3
+KRB5_CFG_FILE=$4
+PQS_PORT=$5
+PYTHON_SCRIPT=$6
+
+PY_ENV_PATH=$( mktemp -d )
+
+conda create -y -p $PY_ENV_PATH || virtualenv $PY_ENV_PATH
+
+pushd ${PY_ENV_PATH}/bin
+
+# conda activate does stuff with unbound variables :(
+set +u
+. activate ""
+
+popd
+
+set -u
+echo "INSTALLING COMPONENTS"
+pip install -e file:///${LOCAL_PY}/requests-kerberos
+pip install -e file:///${LOCAL_PY}/phoenixdb-module
+
+export KRB5_CONFIG=$KRB5_CFG_FILE
+cat $KRB5_CONFIG
+export KRB5_TRACE=/dev/stdout
+
+#echo "RUNNING KINIT"
+kinit -kt $KEYTAB_LOC $PRINC
--- End diff --
> I just tried on the command line and MAC OS (Heimdal) kinit does not
require a directory
Yeah, convention is to use `${tmpdir}/krb5cc_$(current-user uid)`.
> pass the when executing python or just continue running in the same
shell, which is why I stopped attempts to make ny further reductions to the
shell script
Oh right, I forgot they would bash the environment. Let's just let this be
for now. Will be easier to come back to it later.
---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r202106445
--- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh ---
@@ -0,0 +1,59 @@
+#/usr/bin/env bash
+
+set -u
+set -x
+set -e
+
+function cleanup {
+set +e
+set +u
+kdestroy
--- End diff --
Ok, cool. I didn't think kdestroy was doing more than just cleaning up
those token :)
---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201717820
--- Diff:
phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
---
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.security.token.TokenProvider;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.phoenix.query.ConfigurationFactory;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.queryserver.client.ThinClientUtil;
+import org.apache.phoenix.queryserver.server.QueryServer;
+import org.apache.phoenix.util.InstanceResolver;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import java.nio.file.Paths;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SecureQueryServerPhoenixDBIT {
+private static final Log LOG =
LogFactory.getLog(SecureQueryServerPhoenixDBIT.class);
+
+private static final File TEMP_DIR = new File(getTempDirForClass());
+private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs");
+private static final List USER_KEYTAB_FILES = new ArrayList<>();
+
+private static final String SPNEGO_PRINCIPAL = "HTTP/localhost";
+private static final String PQS_PRINCIPAL = "phoenixqs/localhost";
+private static final String SERVICE_PRINCIPAL =
"securecluster/localhost";
+private static File KEYTAB;
+
+private static MiniKdc KDC;
+private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+private static LocalHBaseCluster HBASE_CLUSTER;
+private static int NUM_CREATED_USERS;
+
+private static ExecutorService PQS_EXECUTOR;
+private static QueryServer PQS;
+private static int PQS_PORT;
+private static String PQS_URL;
+
+private static String getTempDirForClass() {
+StringBuilder sb = new StringBuilder(32);
+sb.append(System.getProperty("user.dir")).append(File.separator);
+sb.append("target").append(File.separator);
+sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName());
+return sb.toString();
+}
+
+private static void updateDefaultRealm() throws Exception {
+// (at least) one other phoenix test triggers the caching of this
field before the KDC is up
+
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201716375 --- Diff: python/phoenixdb-module/phoenixdb/__init__.py --- @@ -1,11 +1,10 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Copyright 2015 Lukas Lalinsky --- End diff -- I am not sure how that went back in, it is possible that I may have copied __init__.py from the time I was doing this work on my own before I found out that this has been moved to phoenix. I will change the header ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201482973 --- Diff: phoenix-queryserver/pom.xml --- @@ -47,6 +47,11 @@ org.apache.maven.plugins maven-failsafe-plugin + + +**/SecureQueryServerPhoenixDBIT.java --- End diff -- There are a few prerequisites - Either anaconda or virtual env *must* to be installed - System *must* provide either MIT or Heimdal kerberos utilities and libraries ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201482297
--- Diff:
phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
---
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.security.token.TokenProvider;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.phoenix.query.ConfigurationFactory;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.queryserver.client.ThinClientUtil;
+import org.apache.phoenix.queryserver.server.QueryServer;
+import org.apache.phoenix.util.InstanceResolver;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import java.nio.file.Paths;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SecureQueryServerPhoenixDBIT {
+private static final Log LOG =
LogFactory.getLog(SecureQueryServerPhoenixDBIT.class);
+
+private static final File TEMP_DIR = new File(getTempDirForClass());
+private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs");
+private static final List USER_KEYTAB_FILES = new ArrayList<>();
+
+private static final String SPNEGO_PRINCIPAL = "HTTP/localhost";
+private static final String PQS_PRINCIPAL = "phoenixqs/localhost";
+private static final String SERVICE_PRINCIPAL =
"securecluster/localhost";
+private static File KEYTAB;
+
+private static MiniKdc KDC;
+private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+private static LocalHBaseCluster HBASE_CLUSTER;
+private static int NUM_CREATED_USERS;
+
+private static ExecutorService PQS_EXECUTOR;
+private static QueryServer PQS;
+private static int PQS_PORT;
+private static String PQS_URL;
+
+private static String getTempDirForClass() {
+StringBuilder sb = new StringBuilder(32);
+sb.append(System.getProperty("user.dir")).append(File.separator);
+sb.append("target").append(File.separator);
+sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName());
+return sb.toString();
+}
+
+private static void updateDefaultRealm() throws Exception {
+// (at least) one other phoenix test triggers the caching of this
field before the KDC is up
+
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201482215
--- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh ---
@@ -0,0 +1,59 @@
+#/usr/bin/env bash
+
+set -u
+set -x
+set -e
+
+function cleanup {
+set +e
+set +u
+kdestroy
--- End diff --
Not to be overly pedantic, but you would want to still pass krb5ccname and
just call kdestroy to make sure proper cleanup is done
---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201481976
--- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh ---
@@ -0,0 +1,59 @@
+#/usr/bin/env bash
+
+set -u
+set -x
+set -e
+
+function cleanup {
+set +e
+set +u
+kdestroy
+pushd ${PY_ENV_PATH}/bin
+. deactivate ""
+popd
+rm -rf $PY_ENV_PATH
+}
+
+trap cleanup EXIT
+
+echo "LAUNCHING SCRIPT"
+
+LOCAL_PY=$1
+PRINC=$2
+KEYTAB_LOC=$3
+KRB5_CFG_FILE=$4
+PQS_PORT=$5
+PYTHON_SCRIPT=$6
+
+PY_ENV_PATH=$( mktemp -d )
+
+conda create -y -p $PY_ENV_PATH || virtualenv $PY_ENV_PATH
+
+pushd ${PY_ENV_PATH}/bin
+
+# conda activate does stuff with unbound variables :(
+set +u
+. activate ""
+
+popd
+
+set -u
+echo "INSTALLING COMPONENTS"
+pip install -e file:///${LOCAL_PY}/requests-kerberos
+pip install -e file:///${LOCAL_PY}/phoenixdb-module
+
+export KRB5_CONFIG=$KRB5_CFG_FILE
+cat $KRB5_CONFIG
+export KRB5_TRACE=/dev/stdout
+
+#echo "RUNNING KINIT"
+kinit -kt $KEYTAB_LOC $PRINC
--- End diff --
I tried something similar
+File KRB5CCNAME = File.createTempFile("krb5ccname", null);
+kinitEnv.put("KRB5CCNAME", KRB5CCNAME.getAbsolutePath());
This stalled, although looking at the code now it probably should have been
a directory, which is why kinit stalled
I can try this again
---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201418146 --- Diff: phoenix-queryserver/pom.xml --- @@ -47,6 +47,11 @@ org.apache.maven.plugins maven-failsafe-plugin + + +**/SecureQueryServerPhoenixDBIT.java --- End diff -- You not intending for this test to be executed during the normal build process? ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201078376 --- Diff: python/requests-kerberos/LICENSE --- @@ -0,0 +1,15 @@ +ISC License --- End diff -- Just calling out that this is allowed: ISC is a Category-A license per https://www.apache.org/legal/resolved.html ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201077800 --- Diff: python/phoenixdb-module/phoenixdb/__init__.py --- @@ -1,11 +1,10 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Copyright 2015 Lukas Lalinsky --- End diff -- Any reason for the re-add of this? We don't need this after the IP Clearance process, I think. NOTICE file should be sufficient. ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201076404
--- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh ---
@@ -0,0 +1,59 @@
+#/usr/bin/env bash
+
+set -u
+set -x
+set -e
+
+function cleanup {
+set +e
+set +u
+kdestroy
--- End diff --
If we use a custom directory for the `kinit`, then this just becomes
removing that custom directory.
---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201075953
--- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh ---
@@ -0,0 +1,59 @@
+#/usr/bin/env bash
+
+set -u
+set -x
+set -e
+
+function cleanup {
+set +e
+set +u
+kdestroy
+pushd ${PY_ENV_PATH}/bin
+. deactivate ""
+popd
+rm -rf $PY_ENV_PATH
+}
+
+trap cleanup EXIT
+
+echo "LAUNCHING SCRIPT"
+
+LOCAL_PY=$1
+PRINC=$2
+KEYTAB_LOC=$3
+KRB5_CFG_FILE=$4
+PQS_PORT=$5
+PYTHON_SCRIPT=$6
+
+PY_ENV_PATH=$( mktemp -d )
+
+conda create -y -p $PY_ENV_PATH || virtualenv $PY_ENV_PATH
+
+pushd ${PY_ENV_PATH}/bin
+
+# conda activate does stuff with unbound variables :(
+set +u
+. activate ""
+
+popd
+
+set -u
+echo "INSTALLING COMPONENTS"
+pip install -e file:///${LOCAL_PY}/requests-kerberos
+pip install -e file:///${LOCAL_PY}/phoenixdb-module
+
+export KRB5_CONFIG=$KRB5_CFG_FILE
+cat $KRB5_CONFIG
+export KRB5_TRACE=/dev/stdout
+
+#echo "RUNNING KINIT"
+kinit -kt $KEYTAB_LOC $PRINC
--- End diff --
Can we kinit to a custom location? e.g. the `-c` option. Then, later, we
just set the variable `KRB5CCNAME` in the shell ENV.
This would help prevent us from bashing the user's ticket (if they already
have one).
---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201077202
--- Diff:
phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
---
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.security.token.TokenProvider;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.phoenix.query.ConfigurationFactory;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.queryserver.client.ThinClientUtil;
+import org.apache.phoenix.queryserver.server.QueryServer;
+import org.apache.phoenix.util.InstanceResolver;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import java.nio.file.Paths;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SecureQueryServerPhoenixDBIT {
+private static final Log LOG =
LogFactory.getLog(SecureQueryServerPhoenixDBIT.class);
+
+private static final File TEMP_DIR = new File(getTempDirForClass());
+private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs");
+private static final List USER_KEYTAB_FILES = new ArrayList<>();
+
+private static final String SPNEGO_PRINCIPAL = "HTTP/localhost";
+private static final String PQS_PRINCIPAL = "phoenixqs/localhost";
+private static final String SERVICE_PRINCIPAL =
"securecluster/localhost";
+private static File KEYTAB;
+
+private static MiniKdc KDC;
+private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+private static LocalHBaseCluster HBASE_CLUSTER;
+private static int NUM_CREATED_USERS;
+
+private static ExecutorService PQS_EXECUTOR;
+private static QueryServer PQS;
+private static int PQS_PORT;
+private static String PQS_URL;
+
+private static String getTempDirForClass() {
+StringBuilder sb = new StringBuilder(32);
+sb.append(System.getProperty("user.dir")).append(File.separator);
+sb.append("target").append(File.separator);
+sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName());
+return sb.toString();
+}
+
+private static void updateDefaultRealm() throws Exception {
+// (at least) one other phoenix test triggers the caching of this
field before the KDC is up
+
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201077355
--- Diff:
phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
---
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.security.token.TokenProvider;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.phoenix.query.ConfigurationFactory;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.queryserver.client.ThinClientUtil;
+import org.apache.phoenix.queryserver.server.QueryServer;
+import org.apache.phoenix.util.InstanceResolver;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import java.nio.file.Paths;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SecureQueryServerPhoenixDBIT {
+private static final Log LOG =
LogFactory.getLog(SecureQueryServerPhoenixDBIT.class);
+
+private static final File TEMP_DIR = new File(getTempDirForClass());
+private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs");
+private static final List USER_KEYTAB_FILES = new ArrayList<>();
+
+private static final String SPNEGO_PRINCIPAL = "HTTP/localhost";
+private static final String PQS_PRINCIPAL = "phoenixqs/localhost";
+private static final String SERVICE_PRINCIPAL =
"securecluster/localhost";
+private static File KEYTAB;
+
+private static MiniKdc KDC;
+private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+private static LocalHBaseCluster HBASE_CLUSTER;
+private static int NUM_CREATED_USERS;
+
+private static ExecutorService PQS_EXECUTOR;
+private static QueryServer PQS;
+private static int PQS_PORT;
+private static String PQS_URL;
+
+private static String getTempDirForClass() {
+StringBuilder sb = new StringBuilder(32);
+sb.append(System.getProperty("user.dir")).append(File.separator);
+sb.append("target").append(File.separator);
+sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName());
+return sb.toString();
+}
+
+private static void updateDefaultRealm() throws Exception {
+// (at least) one other phoenix test triggers the caching of this
field before the KDC is up
+
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/307#discussion_r201076875
--- Diff:
phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
---
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.security.token.TokenProvider;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.phoenix.query.ConfigurationFactory;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.queryserver.client.ThinClientUtil;
+import org.apache.phoenix.queryserver.server.QueryServer;
+import org.apache.phoenix.util.InstanceResolver;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import java.nio.file.Paths;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SecureQueryServerPhoenixDBIT {
+private static final Log LOG =
LogFactory.getLog(SecureQueryServerPhoenixDBIT.class);
+
+private static final File TEMP_DIR = new File(getTempDirForClass());
+private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs");
+private static final List USER_KEYTAB_FILES = new ArrayList<>();
+
+private static final String SPNEGO_PRINCIPAL = "HTTP/localhost";
+private static final String PQS_PRINCIPAL = "phoenixqs/localhost";
+private static final String SERVICE_PRINCIPAL =
"securecluster/localhost";
+private static File KEYTAB;
+
+private static MiniKdc KDC;
+private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+private static LocalHBaseCluster HBASE_CLUSTER;
+private static int NUM_CREATED_USERS;
+
+private static ExecutorService PQS_EXECUTOR;
+private static QueryServer PQS;
+private static int PQS_PORT;
+private static String PQS_URL;
+
+private static String getTempDirForClass() {
+StringBuilder sb = new StringBuilder(32);
+sb.append(System.getProperty("user.dir")).append(File.separator);
+sb.append("target").append(File.separator);
+sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName());
+return sb.toString();
+}
+
+private static void updateDefaultRealm() throws Exception {
+// (at least) one other phoenix test triggers the caching of this
field before the KDC is up
+
[GitHub] phoenix pull request #307: Phoenix 4688 Kerberize python phoenixdb
GitHub user pu239ppy opened a pull request: https://github.com/apache/phoenix/pull/307 Phoenix 4688 Kerberize python phoenixdb Lets rip out httplib and replace with requests and use requests kerberos Notes - This PR mirrors requests kerberos until such time that the maintainers of reuests-kerberos can merge https://github.com/requests/requests-kerberos/pull/115 - This is trivial comparing to the integration test required You can merge this pull request into a Git repository by running: $ git pull https://github.com/pu239ppy/phoenix PHOENIX-4688 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/phoenix/pull/307.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #307 commit 6a5448237fee59c36b167f445bfdb23ce27a308f Author: Lev Bronshtein Date: 2018-06-06T15:06:42Z moved to a separate subdirectory commit 6373e0332b24010d7b357065fface0bb7a97f0d5 Author: Lev Bronshtein Date: 2018-06-06T15:11:31Z whoops should have been ls -la commit c3c86b912f33b964f8cc607b1c43b3eb3ff56b3d Author: Lev Bronshtein Date: 2018-06-06T15:14:35Z added my fork of requests-kerberos module commit 03fc4c53b9448704d569fd4af574eecd1ea5536d Author: Lev Bronshtein Date: 2018-06-06T16:00:45Z Now with KERBEROS commit 5fb158af09150e5ab35eb9b96fd93550b93629a2 Author: Lev Bronshtein Date: 2018-06-06T17:40:08Z documentation commit e919c76760f5574477a22bee0028b58d6e1460b2 Author: Lev Bronshtein Date: 2018-06-25T22:06:52Z phoenixdb qualifier commit 3f23299553673b35aebe0f258142d999e1fe54f9 Author: Lev Bronshtein Date: 2018-06-26T02:53:59Z no need to maintain a separate directory name for forked project commit 7f2f19c30538d3db54110e7296829fffa87113c4 Author: Lev Bronshtein Date: 2018-06-26T13:34:54Z add test script to run python commit 0207cc5e9c292eda1859a4ce96e802c4bf3044fd Author: Lev Bronshtein Date: 2018-06-26T13:37:32Z make excutable commit 56c7a9a9c07d003d76813cb48243b10598d6cef2 Author: Lev Bronshtein Date: 2018-06-26T14:49:30Z pass command line parameters commit b2c7c206d830baafc39cdf833153505425187967 Author: Lev Bronshtein Date: 2018-06-26T14:54:07Z phoenix URL commit 4b6ebc1153643e9e033e84f1cb636f872468dfdd Author: Lev Bronshtein Date: 2018-06-26T21:51:38Z lets not do heredoc commit 8203449d342fcee41d0d72227ea80a7b73f62879 Author: Lev Bronshtein Date: 2018-06-26T21:52:00Z get STDOUT/ERR commit 81dd5b35d18466e0758d3b56960f33ac1d84a365 Author: Lev Bronshtein Date: 2018-06-27T11:22:33Z typo in realms commit be0f774c10fc790166f6af060d06e7e3b575df07 Author: Lev Bronshtein Date: 2018-06-27T11:23:13Z few safegurds commit ddfd1e324df83aec7c7bb425adfe435c7c36e11d Author: Lev Bronshtein Date: 2018-06-27T11:42:54Z Add KDC port to list of params commit b04a8eed246ea1987a282a34ad832d08d8b390ed Author: Lev Bronshtein Date: 2018-06-27T12:45:12Z use krb5.conf generated by the MINI KDC commit 2a4969b9cad2ad178c85804ed458b63f62e0d8dc Author: Lev Bronshtein Date: 2018-06-27T13:00:01Z use example from README commit 032879b8abeae71db016ca26a6b4f27000fb504a Author: Lev Bronshtein Date: 2018-06-27T13:00:21Z comments commit 6500a024beaa161841b731fdec09523d1c57daf4 Author: Lev Bronshtein Date: 2018-06-27T13:01:49Z lets just hardcode this, what difference doe sit make commit d7830fcfcc43fac8ff4de176554c39983e718383 Author: Lev Bronshtein Date: 2018-06-27T16:54:16Z avoiding unbound variable mech_oid commit 7fa5c5d76c74350f91d2373ff452f59810c77da7 Author: Lev Bronshtein Date: 2018-06-27T16:55:18Z have to pass PQS port as it changes on every run commit e473255835df659386f6b221c31184f6aeabc2c8 Author: Lev Bronshtein Date: 2018-06-27T17:00:38Z pass PQS port to python commit 7b17feb11cf3d73daaa0584e9359fa64998d2738 Author: Lev Bronshtein Date: 2018-06-27T17:53:34Z OS agnostic path commit 312bb27c06006b7ff12cf32ff8e422e3940a08f5 Author: Lev Bronshtein Date: 2018-06-27T19:20:36Z shell script inherits proxy settings form caller no need to set, cook up a custom heimdal krb5.conf if mac commit 10021e341d25b530fda7a69f1cd9ef37917a3c14 Author: Lev Bronshtein Date: 2018-06-27T19:42:50Z tell shell script where to find python script commit 7652edf8a4574f073f2911fe0062ba95ba799167 Author: Lev Bronshtein Date: 2018-06-27T21:38:58Z no longer need to do any cleanup commit 1aa9147566b6e2c9f5a0eb154f6c09113f143956 Author: Lev Bronshtein Date: 2018-06-28T16:55:13Z call kinit and pass along credentials commit 933328e01fd02909cda72436104f7a85e0705cb5 Author: Lev Bronshtein Date: 2018-06-28T18:47:06Z stalls while trying to execute kinit, I will leave this for someone else to figure out commit 690b5e9c116121962b585c5a97ca2ff7fe30f992 Au
