http://git-wip-us.apache.org/repos/asf/knox/blob/2bb8d1e0/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java ---------------------------------------------------------------------- diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java index b4877b3..c56081d 100644 --- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java +++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java @@ -83,3627 +83,3628 @@ import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs; @Category( { FunctionalTests.class, MediumTests.class } ) public class GatewayBasicFuncTest { -// private static final long SHORT_TIMEOUT = 1000L; -// private static final long MEDIUM_TIMEOUT = 10 * SHORT_TIMEOUT; -// private static final long LONG_TIMEOUT = 10 * MEDIUM_TIMEOUT; -// -// private static final Charset UTF8 = Charset.forName("UTF-8"); -// -// // Uncomment to cause the test to hang after the gateway instance is setup. -// // This will allow the gateway instance to be hit directly via some external client. -//// @Test -//// public void hang() throws IOException { -//// System.out.println( "Server on port " + driver.gateway.getAddresses()[0].getPort() ); -//// System.out.println(); -//// System.in.read(); -//// } -// -// private static Logger log = LoggerFactory.getLogger( GatewayBasicFuncTest.class ); -// -// public static GatewayFuncTestDriver driver = new GatewayFuncTestDriver(); -// -// // Controls the host name to which the gateway dispatch requests. This may be the name of a sandbox VM -// // or an EC2 instance. Currently only a single host is supported. -// private static final String TEST_HOST = "vm.local"; -// -// // Specifies if the test requests should go through the gateway or directly to the services. -// // This is frequently used to verify the behavior of the test both with and without the gateway. -// private static final boolean USE_GATEWAY = true; -// -// // Specifies if the test requests should be sent to mock services or the real services. -// // This is frequently used to verify the behavior of the test both with and without mock services. -// private static final boolean USE_MOCK_SERVICES = true; -// -// // Specifies if the GATEWAY_HOME created for the test should be deleted when the test suite is complete. -// // This is frequently used during debugging to keep the GATEWAY_HOME around for inspection. -// private static final boolean CLEANUP_TEST = true; -// -//// private static final boolean USE_GATEWAY = false; -//// private static final boolean USE_MOCK_SERVICES = false; -//// private static final boolean CLEANUP_TEST = false; -// -// private static int findFreePort() throws IOException { -// ServerSocket socket = new ServerSocket(0); -// int port = socket.getLocalPort(); -// socket.close(); -// return port; -// } -// -// /** -// * Creates a deployment of a gateway instance that all test methods will share. This method also creates a -// * registry of sorts for all of the services that will be used by the test methods. -// * The createTopology method is used to create the topology file that would normally be read from disk. -// * The driver.setupGateway invocation is where the creation of GATEWAY_HOME occurs. -// * @throws Exception Thrown if any failure occurs. -// */ -// @BeforeClass -// public static void setupSuite() throws Exception { -// //Log.setLog( new NoOpLogger() ); -// LOG_ENTER(); -// GatewayTestConfig config = new GatewayTestConfig(); -// config.setGatewayPath( "gateway" ); -// driver.setResourceBase(GatewayBasicFuncTest.class); -// driver.setupLdap(findFreePort()); -// driver.setupService("WEBHDFS", "http://"; + TEST_HOST + ":50070/webhdfs", "/cluster/webhdfs", USE_MOCK_SERVICES); -// driver.setupService( "DATANODE", "http://"; + TEST_HOST + ":50075/webhdfs", "/cluster/webhdfs/data", USE_MOCK_SERVICES ); -// driver.setupService( "WEBHCAT", "http://"; + TEST_HOST + ":50111/templeton", "/cluster/templeton", USE_MOCK_SERVICES ); -// driver.setupService( "OOZIE", "http://"; + TEST_HOST + ":11000/oozie", "/cluster/oozie", USE_MOCK_SERVICES ); -// driver.setupService( "HIVE", "http://"; + TEST_HOST + ":10000", "/cluster/hive", USE_MOCK_SERVICES ); -// driver.setupService( "WEBHBASE", "http://"; + TEST_HOST + ":60080", "/cluster/hbase", USE_MOCK_SERVICES ); -// driver.setupService( "NAMENODE", "hdfs://" + TEST_HOST + ":8020", null, USE_MOCK_SERVICES ); -// driver.setupService( "JOBTRACKER", "thrift://" + TEST_HOST + ":8021", null, USE_MOCK_SERVICES ); -// driver.setupService( "RESOURCEMANAGER", "http://"; + TEST_HOST + ":8088/ws", "/cluster/resourcemanager", USE_MOCK_SERVICES ); -// driver.setupService( "FALCON", "http://"; + TEST_HOST + ":15000", "/cluster/falcon", USE_MOCK_SERVICES ); -// driver.setupService( "STORM", "http://"; + TEST_HOST + ":8477", "/cluster/storm", USE_MOCK_SERVICES ); -// driver.setupService( "STORM-LOGVIEWER", "http://"; + TEST_HOST + ":8477", "/cluster/storm", USE_MOCK_SERVICES ); -// driver.setupGateway( config, "cluster", createTopology(), USE_GATEWAY ); -// LOG_EXIT(); -// } -// -// @AfterClass -// public static void cleanupSuite() throws Exception { -// LOG_ENTER(); -// if( CLEANUP_TEST ) { -// driver.cleanup(); -// } -// LOG_EXIT(); -// } -// -// @After -// public void cleanupTest() { -// driver.reset(); -// } -// -// /** -// * Creates a topology that is deployed to the gateway instance for the test suite. -// * Note that this topology is shared by all of the test methods in this suite. -// * @return A populated XML structure for a topology file. -// */ -// private static XMLTag createTopology() { -// XMLTag xml = XMLDoc.newDocument( true ) -// .addRoot( "topology" ) -// .addTag( "gateway" ) -// .addTag( "provider" ) -// .addTag( "role" ).addText( "webappsec" ) -// .addTag("name").addText("WebAppSec") -// .addTag("enabled").addText("true") -// .addTag( "param" ) -// .addTag("name").addText("csrf.enabled") -// .addTag("value").addText("true").gotoParent().gotoParent() -// .addTag("provider") -// .addTag("role").addText("authentication") -// .addTag("name").addText("ShiroProvider") -// .addTag("enabled").addText("true") -// .addTag( "param" ) -// .addTag("name").addText("main.ldapRealm") -// .addTag("value").addText("org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm").gotoParent() -// .addTag( "param" ) -// .addTag( "name" ).addText( "main.ldapRealm.userDnTemplate" ) -// .addTag( "value" ).addText( "uid={0},ou=people,dc=hadoop,dc=apache,dc=org" ).gotoParent() -// .addTag( "param" ) -// .addTag( "name" ).addText( "main.ldapRealm.contextFactory.url" ) -// .addTag( "value" ).addText( driver.getLdapUrl() ).gotoParent() -// .addTag( "param" ) -// .addTag( "name" ).addText( "main.ldapRealm.contextFactory.authenticationMechanism" ) -// .addTag( "value" ).addText( "simple" ).gotoParent() -// .addTag( "param" ) -// .addTag( "name" ).addText( "urls./**" ) -// .addTag( "value" ).addText( "authcBasic" ).gotoParent().gotoParent() -// .addTag("provider") -// .addTag("role").addText("identity-assertion") -// .addTag("enabled").addText("true") -// .addTag("name").addText("Default").gotoParent() -// .addTag("provider") -// .addTag( "role" ).addText( "authorization" ) -// .addTag( "enabled" ).addText( "true" ) -// .addTag("name").addText("AclsAuthz").gotoParent() -// .addTag("param") -// .addTag("name").addText( "webhdfs-acl" ) -// .addTag("value").addText( "hdfs;*;*" ).gotoParent() -// .gotoRoot() -// .addTag("service") -// .addTag("role").addText("WEBHDFS") -// .addTag("url").addText(driver.getRealUrl("WEBHDFS")).gotoParent() -// .addTag( "service" ) -// .addTag( "role" ).addText( "NAMENODE" ) -// .addTag( "url" ).addText( driver.getRealUrl( "NAMENODE" ) ).gotoParent() -// .addTag( "service" ) -// .addTag( "role" ).addText( "DATANODE" ) -// .addTag( "url" ).addText( driver.getRealUrl( "DATANODE" ) ).gotoParent() -// .addTag( "service" ) -// .addTag( "role" ).addText( "JOBTRACKER" ) -// .addTag( "url" ).addText( driver.getRealUrl( "JOBTRACKER" ) ).gotoParent() -// .addTag( "service" ) -// .addTag( "role" ).addText( "WEBHCAT" ) -// .addTag( "url" ).addText( driver.getRealUrl( "WEBHCAT" ) ).gotoParent() -// .addTag( "service" ) -// .addTag( "role" ).addText( "OOZIE" ) -// .addTag( "url" ).addText( driver.getRealUrl( "OOZIE" ) ).gotoParent() -// .addTag( "service" ) -// .addTag( "role" ).addText( "HIVE" ) -// .addTag( "url" ).addText( driver.getRealUrl( "HIVE" ) ).gotoParent() -// .addTag( "service" ) -// .addTag( "role" ).addText( "WEBHBASE" ) -// .addTag( "url" ).addText( driver.getRealUrl( "WEBHBASE" ) ).gotoParent() -// .addTag("service") -// .addTag("role").addText("RESOURCEMANAGER") -// .addTag("url").addText(driver.getRealUrl("RESOURCEMANAGER")).gotoParent() -// .addTag("service") -// .addTag("role").addText("FALCON") -// .addTag("url").addText(driver.getRealUrl("FALCON")).gotoParent() -// .addTag("service") -// .addTag("role").addText("STORM") -// .addTag("url").addText(driver.getRealUrl("STORM")).gotoParent() -// .addTag("service") -// .addTag("role").addText("STORM-LOGVIEWER") -// .addTag("url").addText(driver.getRealUrl("STORM-LOGVIEWER")).gotoParent() -// .addTag("service") -// .addTag("role").addText("SERVICE-TEST") -// .gotoRoot(); -//// System.out.println( "GATEWAY=" + xml.toString() ); -// return xml; -// } -// -// @Test( timeout = MEDIUM_TIMEOUT ) -// public void testBasicJsonUseCase() throws IOException { -// LOG_ENTER(); -// String root = "/tmp/GatewayBasicFuncTest/testBasicJsonUseCase"; -// String username = "hdfs"; -// String password = "hdfs-password"; -// /* Create a directory. -// curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]" -// -// The client receives a respond with a boolean JSON object: -// HTTP/1.1 HttpStatus.SC_OK OK -// Content-Type: application/json -// Transfer-Encoding: chunked -// -// {"boolean": true} -// */ -// driver.getMock( "WEBHDFS" ) -// .expect() -// .method( "PUT" ) -// .pathInfo( "/v1" + root + "/dir" ) -// .queryParam( "op", "MKDIRS" ) -// .queryParam( "user.name", username ) -// .respond() -// .status( HttpStatus.SC_OK ) -// .content( driver.getResourceBytes( "webhdfs-success.json" ) ) -// .contentType( "application/json" ); -// Cookie cookie = given() -// //.log().all() -// .auth().preemptive().basic( username, password ) -// .header("X-XSRF-Header", "jksdhfkhdsf") -// .queryParam( "op", "MKDIRS" ) -// .expect() -// //.log().all() -// .statusCode( HttpStatus.SC_OK ) -// .header( "Set-Cookie", containsString( "JSESSIONID" ) ) -// .header( "Set-Cookie", containsString( "HttpOnly" ) ) -// .contentType( "application/json" ) -// .content( "boolean", is( true ) ) -// .when().put( driver.getUrl( "WEBHDFS" ) + "/v1" + root + "/dir" ).getDetailedCookie( "JSESSIONID" ); -// assertThat( cookie.isSecured(), is( true ) ); -// assertThat( cookie.getPath(), is( "/gateway/cluster" ) ); -// assertThat( cookie.getValue().length(), greaterThan( 16 ) ); -// driver.assertComplete(); -// LOG_EXIT(); -// } -// -// @Test( timeout = MEDIUM_TIMEOUT ) -// public void testBasicOutboundHeaderUseCase() throws IOException { -// LOG_ENTER(); -// String root = "/tmp/GatewayBasicFuncTest/testBasicOutboundHeaderUseCase"; -// String username = "hdfs"; -// String password = "hdfs-password"; -// InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; -// String gatewayHostName = gatewayAddress.getHostName(); -// String gatewayAddrName = InetAddress.getByName(gatewayHostName).getHostAddress(); -// -// driver.getMock( "WEBHDFS" ) -// .expect() -// .method( "PUT" ) -// .pathInfo( "/v1" + root + "/dir/file" ) -// .header( "Host", driver.getRealAddr( "WEBHDFS" ) ) -// .queryParam( "op", "CREATE" ) -// .queryParam( "user.name", username ) -// .respond() -// .status( HttpStatus.SC_TEMPORARY_REDIRECT ) -// .header("Location", driver.getRealUrl("DATANODE") + "/v1" + root + "/dir/file?op=CREATE&user.name=hdfs"); -// Response response = given() -// //.log().all() -// .auth().preemptive().basic( username, password ) -// .header("X-XSRF-Header", "jksdhfkhdsf") -// .queryParam( "op", "CREATE" ) -// .expect() -// //.log().ifError() -// .statusCode( HttpStatus.SC_TEMPORARY_REDIRECT ) -// .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" ); -// String location = response.getHeader( "Location" ); -// //System.out.println( location ); -// log.debug( "Redirect location: " + response.getHeader( "Location" ) ); -// if( driver.isUseGateway() ) { -// MatcherAssert.assertThat( location, anyOf( -// startsWith( "http://"; + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ), -// startsWith( "http://"; + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) ); -// MatcherAssert.assertThat( location, containsString( "?_=" ) ); -// } -// MatcherAssert.assertThat(location, not(containsString("host="))); -// MatcherAssert.assertThat(location, not(containsString("port="))); -// LOG_EXIT(); -// } -// -// @Test( timeout = MEDIUM_TIMEOUT ) -// public void testHdfsTildeUseCase() throws IOException { -// LOG_ENTER(); -// String root = "/tmp/GatewayBasicFuncTest/testHdfsTildeUseCase"; -// String username = "hdfs"; -// String password = "hdfs-password"; -// InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; -// -// // Attempt to delete the test directory in case a previous run failed. -// // Ignore any result. -// // Cleanup anything that might have been leftover because the test failed previously. -// driver.getMock( "WEBHDFS" ) -// .expect() -// .method( "DELETE" ) -// .from( "testHdfsTildeUseCase" ) -// .pathInfo( "/v1/user/hdfs" + root ) -// .queryParam( "op", "DELETE" ) -// .queryParam( "user.name", username ) -// .queryParam( "recursive", "true" ) -// .respond() -// .status( HttpStatus.SC_OK ); -// given() -// .auth().preemptive().basic( username, password ) -// .header("X-XSRF-Header", "jksdhfkhdsf") -// .queryParam( "op", "DELETE" ) -// .queryParam( "recursive", "true" ) -// .expect() -// //.log().all(); -// .statusCode( HttpStatus.SC_OK ) -// .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1/~" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) ); -// driver.assertComplete(); -// -// driver.getMock( "WEBHDFS" ) -// .expect() -// .method( "PUT" ) -// .pathInfo( "/v1/user/hdfs/dir" ) -// .queryParam( "op", "MKDIRS" ) -// .queryParam( "user.name", username ) -// .respond() -// .status( HttpStatus.SC_OK ) -// .content( driver.getResourceBytes( "webhdfs-success.json" ) ) -// .contentType("application/json"); -// given() -// //.log().all() -// .auth().preemptive().basic( username, password ) -// .header("X-XSRF-Header", "jksdhfkhdsf") -// .queryParam( "op", "MKDIRS" ) -// .expect() -// //.log().all(); -// .statusCode( HttpStatus.SC_OK ) -// .contentType( "application/json" ) -// .content( "boolean", is( true ) ) -// .when().put( driver.getUrl( "WEBHDFS" ) + "/v1/~/dir" ); -// driver.assertComplete(); -// LOG_EXIT(); + private static final long SHORT_TIMEOUT = 1000L; + private static final long MEDIUM_TIMEOUT = 10 * SHORT_TIMEOUT; + private static final long LONG_TIMEOUT = 10 * MEDIUM_TIMEOUT; + + private static final Charset UTF8 = Charset.forName("UTF-8"); + + // Uncomment to cause the test to hang after the gateway instance is setup. + // This will allow the gateway instance to be hit directly via some external client. +// @Test +// public void hang() throws IOException { +// System.out.println( "Server on port " + driver.gateway.getAddresses()[0].getPort() ); +// System.out.println(); +// System.in.read(); // } -// -// @Test( timeout = MEDIUM_TIMEOUT ) -// public void testBasicHdfsUseCase() throws IOException { -// LOG_ENTER(); -// String root = "/tmp/GatewayBasicFuncTest/testBasicHdfsUseCase"; -// String username = "hdfs"; -// String password = "hdfs-password"; -// InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; -// String gatewayHostName = gatewayAddress.getHostName(); -// String gatewayAddrName = InetAddress.getByName( gatewayHostName ).getHostAddress(); -// -// // Attempt to delete the test directory in case a previous run failed. -// // Ignore any result. -// // Cleanup anything that might have been leftover because the test failed previously. -// driver.getMock( "WEBHDFS" ) -// .expect() -// .method( "DELETE" ) -// .from( "testBasicHdfsUseCase-1" ) -// .pathInfo( "/v1" + root ) -// .queryParam( "op", "DELETE" ) -// .queryParam( "user.name", username ) -// .queryParam( "recursive", "true" ) -// .respond() -// .status( HttpStatus.SC_OK ); -// given() -// //.log().all() -// .auth().preemptive().basic( username, password ) -// .header("X-XSRF-Header", "jksdhfkhdsf") -// .queryParam( "op", "DELETE" ) -// .queryParam( "recursive", "true" ) -// .expect() + + private static Logger log = LoggerFactory.getLogger( GatewayBasicFuncTest.class ); + + public static GatewayFuncTestDriver driver = new GatewayFuncTestDriver(); + + // Controls the host name to which the gateway dispatch requests. This may be the name of a sandbox VM + // or an EC2 instance. Currently only a single host is supported. + private static final String TEST_HOST = "vm.local"; + + // Specifies if the test requests should go through the gateway or directly to the services. + // This is frequently used to verify the behavior of the test both with and without the gateway. + private static final boolean USE_GATEWAY = true; + + // Specifies if the test requests should be sent to mock services or the real services. + // This is frequently used to verify the behavior of the test both with and without mock services. + private static final boolean USE_MOCK_SERVICES = true; + + // Specifies if the GATEWAY_HOME created for the test should be deleted when the test suite is complete. + // This is frequently used during debugging to keep the GATEWAY_HOME around for inspection. + private static final boolean CLEANUP_TEST = true; + +// private static final boolean USE_GATEWAY = false; +// private static final boolean USE_MOCK_SERVICES = false; +// private static final boolean CLEANUP_TEST = false; + + private static int findFreePort() throws IOException { + ServerSocket socket = new ServerSocket(0); + int port = socket.getLocalPort(); + socket.close(); + return port; + } + + /** + * Creates a deployment of a gateway instance that all test methods will share. This method also creates a + * registry of sorts for all of the services that will be used by the test methods. + * The createTopology method is used to create the topology file that would normally be read from disk. + * The driver.setupGateway invocation is where the creation of GATEWAY_HOME occurs. + * @throws Exception Thrown if any failure occurs. + */ + @BeforeClass + public static void setupSuite() throws Exception { + //Log.setLog( new NoOpLogger() ); + LOG_ENTER(); + GatewayTestConfig config = new GatewayTestConfig(); + config.setGatewayPath( "gateway" ); + driver.setResourceBase(GatewayBasicFuncTest.class); + driver.setupLdap(findFreePort()); + driver.setupService("WEBHDFS", "http://"; + TEST_HOST + ":50070/webhdfs", "/cluster/webhdfs", USE_MOCK_SERVICES); + driver.setupService( "DATANODE", "http://"; + TEST_HOST + ":50075/webhdfs", "/cluster/webhdfs/data", USE_MOCK_SERVICES ); + driver.setupService( "WEBHCAT", "http://"; + TEST_HOST + ":50111/templeton", "/cluster/templeton", USE_MOCK_SERVICES ); + driver.setupService( "OOZIE", "http://"; + TEST_HOST + ":11000/oozie", "/cluster/oozie", USE_MOCK_SERVICES ); + driver.setupService( "HIVE", "http://"; + TEST_HOST + ":10000", "/cluster/hive", USE_MOCK_SERVICES ); + driver.setupService( "WEBHBASE", "http://"; + TEST_HOST + ":60080", "/cluster/hbase", USE_MOCK_SERVICES ); + driver.setupService( "NAMENODE", "hdfs://" + TEST_HOST + ":8020", null, USE_MOCK_SERVICES ); + driver.setupService( "JOBTRACKER", "thrift://" + TEST_HOST + ":8021", null, USE_MOCK_SERVICES ); + driver.setupService( "RESOURCEMANAGER", "http://"; + TEST_HOST + ":8088/ws", "/cluster/resourcemanager", USE_MOCK_SERVICES ); + driver.setupService( "FALCON", "http://"; + TEST_HOST + ":15000", "/cluster/falcon", USE_MOCK_SERVICES ); + driver.setupService( "STORM", "http://"; + TEST_HOST + ":8477", "/cluster/storm", USE_MOCK_SERVICES ); + driver.setupService( "STORM-LOGVIEWER", "http://"; + TEST_HOST + ":8477", "/cluster/storm", USE_MOCK_SERVICES ); + driver.setupGateway( config, "cluster", createTopology(), USE_GATEWAY ); + LOG_EXIT(); + } + + @AfterClass + public static void cleanupSuite() throws Exception { + LOG_ENTER(); + if( CLEANUP_TEST ) { + driver.cleanup(); + } + LOG_EXIT(); + } + + @After + public void cleanupTest() { + driver.reset(); + } + + /** + * Creates a topology that is deployed to the gateway instance for the test suite. + * Note that this topology is shared by all of the test methods in this suite. + * @return A populated XML structure for a topology file. + */ + private static XMLTag createTopology() { + XMLTag xml = XMLDoc.newDocument( true ) + .addRoot( "topology" ) + .addTag( "gateway" ) + .addTag( "provider" ) + .addTag( "role" ).addText( "webappsec" ) + .addTag("name").addText("WebAppSec") + .addTag("enabled").addText("true") + .addTag( "param" ) + .addTag("name").addText("csrf.enabled") + .addTag("value").addText("true").gotoParent().gotoParent() + .addTag("provider") + .addTag("role").addText("authentication") + .addTag("name").addText("ShiroProvider") + .addTag("enabled").addText("true") + .addTag( "param" ) + .addTag("name").addText("main.ldapRealm") + .addTag("value").addText("org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm").gotoParent() + .addTag( "param" ) + .addTag( "name" ).addText( "main.ldapRealm.userDnTemplate" ) + .addTag( "value" ).addText( "uid={0},ou=people,dc=hadoop,dc=apache,dc=org" ).gotoParent() + .addTag( "param" ) + .addTag( "name" ).addText( "main.ldapRealm.contextFactory.url" ) + .addTag( "value" ).addText( driver.getLdapUrl() ).gotoParent() + .addTag( "param" ) + .addTag( "name" ).addText( "main.ldapRealm.contextFactory.authenticationMechanism" ) + .addTag( "value" ).addText( "simple" ).gotoParent() + .addTag( "param" ) + .addTag( "name" ).addText( "urls./**" ) + .addTag( "value" ).addText( "authcBasic" ).gotoParent().gotoParent() + .addTag("provider") + .addTag("role").addText("identity-assertion") + .addTag("enabled").addText("true") + .addTag("name").addText("Default").gotoParent() + .addTag("provider") + .addTag( "role" ).addText( "authorization" ) + .addTag( "enabled" ).addText( "true" ) + .addTag("name").addText("AclsAuthz").gotoParent() + .addTag("param") + .addTag("name").addText( "webhdfs-acl" ) + .addTag("value").addText( "hdfs;*;*" ).gotoParent() + .gotoRoot() + .addTag("service") + .addTag("role").addText("WEBHDFS") + .addTag("url").addText(driver.getRealUrl("WEBHDFS")).gotoParent() + .addTag( "service" ) + .addTag( "role" ).addText( "NAMENODE" ) + .addTag( "url" ).addText( driver.getRealUrl( "NAMENODE" ) ).gotoParent() + .addTag( "service" ) + .addTag( "role" ).addText( "DATANODE" ) + .addTag( "url" ).addText( driver.getRealUrl( "DATANODE" ) ).gotoParent() + .addTag( "service" ) + .addTag( "role" ).addText( "JOBTRACKER" ) + .addTag( "url" ).addText( driver.getRealUrl( "JOBTRACKER" ) ).gotoParent() + .addTag( "service" ) + .addTag( "role" ).addText( "WEBHCAT" ) + .addTag( "url" ).addText( driver.getRealUrl( "WEBHCAT" ) ).gotoParent() + .addTag( "service" ) + .addTag( "role" ).addText( "OOZIE" ) + .addTag( "url" ).addText( driver.getRealUrl( "OOZIE" ) ).gotoParent() + .addTag( "service" ) + .addTag( "role" ).addText( "HIVE" ) + .addTag( "url" ).addText( driver.getRealUrl( "HIVE" ) ).gotoParent() + .addTag( "service" ) + .addTag( "role" ).addText( "WEBHBASE" ) + .addTag( "url" ).addText( driver.getRealUrl( "WEBHBASE" ) ).gotoParent() + .addTag("service") + .addTag("role").addText("RESOURCEMANAGER") + .addTag("url").addText(driver.getRealUrl("RESOURCEMANAGER")).gotoParent() + .addTag("service") + .addTag("role").addText("FALCON") + .addTag("url").addText(driver.getRealUrl("FALCON")).gotoParent() + .addTag("service") + .addTag("role").addText("STORM") + .addTag("url").addText(driver.getRealUrl("STORM")).gotoParent() + .addTag("service") + .addTag("role").addText("STORM-LOGVIEWER") + .addTag("url").addText(driver.getRealUrl("STORM-LOGVIEWER")).gotoParent() + .addTag("service") + .addTag("role").addText("SERVICE-TEST") + .gotoRoot(); +// System.out.println( "GATEWAY=" + xml.toString() ); + return xml; + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testBasicJsonUseCase() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayBasicFuncTest/testBasicJsonUseCase"; + String username = "hdfs"; + String password = "hdfs-password"; + /* Create a directory. + curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]" + + The client receives a respond with a boolean JSON object: + HTTP/1.1 HttpStatus.SC_OK OK + Content-Type: application/json + Transfer-Encoding: chunked + + {"boolean": true} + */ + driver.getMock( "WEBHDFS" ) + .expect() + .method( "PUT" ) + .pathInfo( "/v1" + root + "/dir" ) + .queryParam( "op", "MKDIRS" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "webhdfs-success.json" ) ) + .contentType( "application/json" ); + Cookie cookie = given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "MKDIRS" ) + .expect() + //.log().all() + .statusCode( HttpStatus.SC_OK ) + .header( "Set-Cookie", containsString( "JSESSIONID" ) ) + .header( "Set-Cookie", containsString( "HttpOnly" ) ) + .contentType( "application/json" ) + .content( "boolean", is( true ) ) + .when().put( driver.getUrl( "WEBHDFS" ) + "/v1" + root + "/dir" ).getDetailedCookie( "JSESSIONID" ); + assertThat( cookie.isSecured(), is( true ) ); + assertThat( cookie.getPath(), is( "/gateway/cluster" ) ); + assertThat( cookie.getValue().length(), greaterThan( 16 ) ); + driver.assertComplete(); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testBasicOutboundHeaderUseCase() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayBasicFuncTest/testBasicOutboundHeaderUseCase"; + String username = "hdfs"; + String password = "hdfs-password"; + InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; + String gatewayHostName = gatewayAddress.getHostName(); + String gatewayAddrName = InetAddress.getByName(gatewayHostName).getHostAddress(); + + driver.getMock( "WEBHDFS" ) + .expect() + .method( "PUT" ) + .pathInfo( "/v1" + root + "/dir/file" ) + .header( "Host", driver.getRealAddr( "WEBHDFS" ) ) + .queryParam( "op", "CREATE" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_TEMPORARY_REDIRECT ) + .header("Location", driver.getRealUrl("DATANODE") + "/v1" + root + "/dir/file?op=CREATE&user.name=hdfs"); + Response response = given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "CREATE" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_TEMPORARY_REDIRECT ) + .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" ); + String location = response.getHeader( "Location" ); + //System.out.println( location ); + log.debug( "Redirect location: " + response.getHeader( "Location" ) ); + if( driver.isUseGateway() ) { + MatcherAssert.assertThat( location, anyOf( + startsWith( "http://"; + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ), + startsWith( "http://"; + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) ); + MatcherAssert.assertThat( location, containsString( "?_=" ) ); + } + MatcherAssert.assertThat(location, not(containsString("host="))); + MatcherAssert.assertThat(location, not(containsString("port="))); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testHdfsTildeUseCase() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayBasicFuncTest/testHdfsTildeUseCase"; + String username = "hdfs"; + String password = "hdfs-password"; + InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; + + // Attempt to delete the test directory in case a previous run failed. + // Ignore any result. + // Cleanup anything that might have been leftover because the test failed previously. + driver.getMock( "WEBHDFS" ) + .expect() + .method( "DELETE" ) + .from( "testHdfsTildeUseCase" ) + .pathInfo( "/v1/user/hdfs" + root ) + .queryParam( "op", "DELETE" ) + .queryParam( "user.name", username ) + .queryParam( "recursive", "true" ) + .respond() + .status( HttpStatus.SC_OK ); + given() + .log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "DELETE" ) + .queryParam( "recursive", "true" ) + .expect() + .log().all() + .statusCode( HttpStatus.SC_OK ) + .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1/~" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) ); + driver.assertComplete(); + + driver.getMock( "WEBHDFS" ) + .expect() + .method( "PUT" ) + .pathInfo( "/v1/user/hdfs/dir" ) + .queryParam( "op", "MKDIRS" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "webhdfs-success.json" ) ) + .contentType("application/json"); + given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "MKDIRS" ) + .expect() + //.log().all(); + .statusCode( HttpStatus.SC_OK ) + .contentType( "application/json" ) + .content( "boolean", is( true ) ) + .when().put( driver.getUrl( "WEBHDFS" ) + "/v1/~/dir" ); + driver.assertComplete(); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testBasicHdfsUseCase() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayBasicFuncTest/testBasicHdfsUseCase"; + String username = "hdfs"; + String password = "hdfs-password"; + InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; + String gatewayHostName = gatewayAddress.getHostName(); + String gatewayAddrName = InetAddress.getByName( gatewayHostName ).getHostAddress(); + + // Attempt to delete the test directory in case a previous run failed. + // Ignore any result. + // Cleanup anything that might have been leftover because the test failed previously. + driver.getMock( "WEBHDFS" ) + .expect() + .method( "DELETE" ) + .from( "testBasicHdfsUseCase-1" ) + .pathInfo( "/v1" + root ) + .queryParam( "op", "DELETE" ) + .queryParam( "user.name", username ) + .queryParam( "recursive", "true" ) + .respond() + .status( HttpStatus.SC_OK ); + given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "DELETE" ) + .queryParam( "recursive", "true" ) + .expect() + .log().all() + .statusCode( HttpStatus.SC_OK ) + .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) ); + driver.assertComplete(); + + /* Create a directory. + curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]" + + The client receives a respond with a boolean JSON object: + HTTP/1.1 HttpStatus.SC_OK OK + Content-Type: application/json + Transfer-Encoding: chunked + + {"boolean": true} + */ + driver.getMock( "WEBHDFS" ) + .expect() + .method( "PUT" ) + .pathInfo( "/v1" + root + "/dir" ) + .queryParam( "op", "MKDIRS" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "webhdfs-success.json" ) ) + .contentType( "application/json" ); + given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "MKDIRS" ) + .expect() + //.log().all(); + .statusCode( HttpStatus.SC_OK ) + .contentType( "application/json" ) + .content( "boolean", is( true ) ) + .when().put( driver.getUrl( "WEBHDFS" ) + "/v1" + root + "/dir" ); + driver.assertComplete(); + + driver.getMock( "WEBHDFS" ) + .expect() + .method( "GET" ) + .pathInfo( "/v1" + root ) + .queryParam( "op", "LISTSTATUS" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "webhdfs-liststatus-test.json" ) ) + .contentType( "application/json" ); + given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "LISTSTATUS" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_OK ) + .content( "FileStatuses.FileStatus[0].pathSuffix", is( "dir" ) ) + .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root ); + driver.assertComplete(); + + //NEGATIVE: Test a bad password. + given() + //.log().all() + .auth().preemptive().basic( username, "invalid-password" ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "LISTSTATUS" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_UNAUTHORIZED ) + .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root ); + driver.assertComplete(); + + //NEGATIVE: Test a bad user. + given() + //.log().all() + .auth().preemptive().basic( "hdfs-user", "hdfs-password" ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "LISTSTATUS" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_UNAUTHORIZED ) + .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root ); + driver.assertComplete(); + + //NEGATIVE: Test a valid but unauthorized user. + given() + //.log().all() + .auth().preemptive().basic( "mapred-user", "mapred-password" ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "LISTSTATUS" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_UNAUTHORIZED ) + .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root ); + + /* Add a file. + curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE + [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>] + [&permission=<OCTAL>][&buffersize=<INT>]" + + The expect is redirected to a datanode where the file data is to be written: + HTTP/1.1 307 TEMPORARY_REDIRECT + Location: http://<DATANODE>:<PORT>/webhdfs/v1/<PATH>?op=CREATE... + Content-Length: 0 + + Step 2: Submit another HTTP PUT expect using the URL in the Location header with the file data to be written. + curl -i -X PUT -T <LOCAL_FILE> "http://<DATANODE>:<PORT>/webhdfs/v1/<PATH>?op=CREATE..." + + The client receives a HttpStatus.SC_CREATED Created respond with zero content length and the WebHDFS URI of the file in the Location header: + HTTP/1.1 HttpStatus.SC_CREATED Created + Location: webhdfs://<HOST>:<PORT>/<PATH> + Content-Length: 0 + */ + driver.getMock( "WEBHDFS" ) + .expect() + .method( "PUT" ) + .pathInfo( "/v1" + root + "/dir/file" ) + .queryParam( "op", "CREATE" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_TEMPORARY_REDIRECT ) + .header( "Location", driver.getRealUrl( "DATANODE" ) + "/v1" + root + "/dir/file?op=CREATE&user.name=hdfs" ); + driver.getMock( "DATANODE" ) + .expect() + .method( "PUT" ) + .pathInfo( "/v1" + root + "/dir/file" ) + .queryParam( "op", "CREATE" ) + .queryParam( "user.name", username ) + .contentType( "text/plain" ) + .content( driver.getResourceBytes( "test.txt" ) ) + //.content( driver.gerResourceBytes( "hadoop-examples.jar" ) ) + .respond() + .status( HttpStatus.SC_CREATED ) + .header( "Location", "webhdfs://" + driver.getRealAddr( "DATANODE" ) + "/v1" + root + "/dir/file" ); + Response response = given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "CREATE" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_TEMPORARY_REDIRECT ) + .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" ); + String location = response.getHeader( "Location" ); + log.debug( "Redirect location: " + response.getHeader( "Location" ) ); + if( driver.isUseGateway() ) { + MatcherAssert.assertThat( location, anyOf( + startsWith( "http://"; + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ), + startsWith( "http://"; + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) ); + MatcherAssert.assertThat( location, containsString( "?_=" ) ); + } + MatcherAssert.assertThat( location, not( containsString( "host=" ) ) ); + MatcherAssert.assertThat( location, not( containsString( "port=" ) ) ); + response = given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "test.txt" ) ) + .contentType( "text/plain" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_CREATED ) + .when().put( location ); + location = response.getHeader( "Location" ); + log.debug( "Created location: " + location ); + if( driver.isUseGateway() ) { + MatcherAssert.assertThat( location, anyOf( + startsWith( "http://"; + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ), + startsWith( "http://"; + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) ); + } + driver.assertComplete(); + + /* Get the file. + curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN + [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]" + + The expect is redirected to a datanode where the file data can be read: + HTTP/1.1 307 TEMPORARY_REDIRECT + Location: http://<DATANODE>:<PORT>/webhdfs/v1/<PATH>?op=OPEN... + Content-Length: 0 + + The client follows the redirect to the datanode and receives the file data: + HTTP/1.1 HttpStatus.SC_OK OK + Content-Type: application/octet-stream + Content-Length: 22 + + Hello, webhdfs user! + */ + driver.getMock( "WEBHDFS" ) + .expect() + .method( "GET" ) + .pathInfo( "/v1" + root + "/dir/file" ) + .queryParam( "op", "OPEN" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_TEMPORARY_REDIRECT ) + .header( "Location", driver.getRealUrl( "DATANODE" ) + "/v1" + root + "/dir/file?op=OPEN&user.name=hdfs" ); + driver.getMock( "DATANODE" ) + .expect() + .method( "GET" ) + .pathInfo( "/v1" + root + "/dir/file" ) + .queryParam( "op", "OPEN" ) + .queryParam( "user.name", username ) + .respond() + .status( HttpStatus.SC_OK ) + .contentType( "text/plain" ) + .content( driver.getResourceBytes( "test.txt" ) ); + given() + //.log().all() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "OPEN" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_OK ) + .content( is( "TEST" ) ) + .when().get( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" ); + driver.assertComplete(); + + /* Delete the directory. + curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE + [&recursive=<true|false>]" + + The client receives a respond with a boolean JSON object: + HTTP/1.1 HttpStatus.SC_OK OK + Content-Type: application/json + Transfer-Encoding: chunked + + {"boolean": true} + */ + // Mock the interaction with the namenode. + driver.getMock( "WEBHDFS" ) + .expect() + .from( "testBasicHdfsUseCase-1" ) + .method( "DELETE" ) + .pathInfo( "/v1" + root ) + .queryParam( "op", "DELETE" ) + .queryParam( "user.name", username ) + .queryParam( "recursive", "true" ) + .respond() + .status( HttpStatus.SC_OK ); + given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "DELETE" ) + .queryParam( "recursive", "true" ) + .expect() + //.log().ifError() + .statusCode( HttpStatus.SC_OK ) + .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1" + root ); + driver.assertComplete(); + LOG_EXIT(); + } + + // User hdfs in groups hadoop, hdfs + // User mapred in groups hadoop, mapred + // User hcat in group hcat + @Test( timeout = MEDIUM_TIMEOUT ) + public void testPmHdfsM1UseCase() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayBasicFuncTest/testPmHdfdM1UseCase"; + String userA = "hdfs"; + String passA = "hdfs-password"; + String userB = "mapred"; + String passB = "mapred-password"; + String userC = "hcat"; + String passC = "hcat-password"; + String groupA = "hdfs"; + String groupB = "mapred"; + String groupAB = "hadoop"; + String groupC = "hcat"; + + driver.deleteFile( userA, passA, root, "true", 200 ); + + driver.createDir( userA, passA, groupA, root + "/dirA700", "700", 200, 200 ); + driver.createDir( userA, passA, groupA, root + "/dirA770", "770", 200, 200 ); + driver.createDir( userA, passA, groupA, root + "/dirA707", "707", 200, 200 ); + driver.createDir( userA, passA, groupA, root + "/dirA777", "777", 200, 200 ); + driver.createDir( userA, passA, groupAB, root + "/dirAB700", "700", 200, 200 ); + driver.createDir( userA, passA, groupAB, root + "/dirAB770", "770", 200, 200 ); + driver.createDir( userA, passA, groupAB, root + "/dirAB707", "707", 200, 200 ); + driver.createDir( userA, passA, groupAB, root + "/dirAB777", "777", 200, 200 ); + + // CREATE: Files + // userA:groupA + driver.createFile( userA, passA, groupA, root + "/dirA700/fileA700", "700", "text/plain", "small1.txt", 307, 201, 200 ); + driver.createFile( userA, passA, groupA, root + "/dirA770/fileA770", "770", "text/plain", "small1.txt", 307, 201, 200 ); + driver.createFile( userA, passA, groupA, root + "/dirA707/fileA707", "707", "text/plain", "small1.txt", 307, 201, 200 ); + driver.createFile( userA, passA, groupA, root + "/dirA777/fileA777", "777", "text/plain", "small1.txt", 307, 201, 200 ); + // userA:groupAB + driver.createFile( userA, passA, groupAB, root + "/dirAB700/fileAB700", "700", "text/plain", "small1.txt", 307, 201, 200 ); + driver.createFile( userA, passA, groupAB, root + "/dirAB770/fileAB770", "770", "text/plain", "small1.txt", 307, 201, 200 ); + driver.createFile( userA, passA, groupAB, root + "/dirAB707/fileAB707", "707", "text/plain", "small1.txt", 307, 201, 200 ); + driver.createFile( userA, passA, groupAB, root + "/dirAB777/fileAB777", "777", "text/plain", "small1.txt", 307, 201, 200 ); + // userB:groupB + driver.createFile( userB, passB, groupB, root + "/dirA700/fileB700", "700", "text/plain", "small1.txt", 307, 403, 0 ); + driver.createFile( userB, passB, groupB, root + "/dirA770/fileB700", "700", "text/plain", "small1.txt", 307, 403, 0 ); +//kam:20130219[ chmod seems to be broken at least in Sandbox 1.2 +// driver.createFile( userB, passB, groupB, root + "/dirA707/fileB700", "700", "text/plain", "small1.txt", 307, 201, 200 ); +// driver.createFile( userB, passB, groupB, root + "/dirA777/fileB700", "700", "text/plain", "small1.txt", 307, 201, 200 ); +//kam] + // userB:groupAB + driver.createFile( userB, passB, groupAB, root + "/dirA700/fileBA700", "700", "text/plain", "small1.txt", 307, 403, 0 ); + driver.createFile( userB, passB, groupAB, root + "/dirA770/fileBA700", "700", "text/plain", "small1.txt", 307, 403, 0 ); + driver.createFile( userB, passB, groupAB, root + "/dirA707/fileBA700", "700", "text/plain", "small1.txt", 307, 201, 200 ); + driver.createFile( userB, passB, groupAB, root + "/dirA777/fileBA700", "700", "text/plain", "small1.txt", 307, 201, 200 ); + // userC:groupC + driver.createFile( userC, passC, groupC, root + "/dirA700/fileC700", "700", "text/plain", "small1.txt", 307, 403, 0 ); + driver.createFile( userC, passC, groupC, root + "/dirA770/fileC700", "700", "text/plain", "small1.txt", 307, 403, 0 ); +//kam:20130219[ chmod seems to be broken at least in Sandbox 1.2 +// driver.createFile( userC, passC, groupC, root + "/dirA707/fileC700", "700", "text/plain", "small1.txt", 307, 201, 200 ); +// driver.createFile( userC, passC, groupC, root + "/dirA777/fileC700", "700", "text/plain", "small1.txt", 307, 201, 200 ); +//kam] + + // READ + // userA + driver.readFile( userA, passA, root + "/dirA700/fileA700", "text/plain", "small1.txt", HttpStatus.SC_OK ); + driver.readFile( userA, passA, root + "/dirA770/fileA770", "text/plain", "small1.txt", HttpStatus.SC_OK ); + driver.readFile( userA, passA, root + "/dirA707/fileA707", "text/plain", "small1.txt", HttpStatus.SC_OK ); + driver.readFile( userA, passA, root + "/dirA777/fileA777", "text/plain", "small1.txt", HttpStatus.SC_OK ); + // userB:groupB + driver.readFile( userB, passB, root + "/dirA700/fileA700", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN ); + driver.readFile( userB, passB, root + "/dirA770/fileA770", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN ); + driver.readFile( userB, passB, root + "/dirA707/fileA707", "text/plain", "small1.txt", HttpStatus.SC_OK ); + driver.readFile( userB, passB, root + "/dirA777/fileA777", "text/plain", "small1.txt", HttpStatus.SC_OK ); + // userB:groupAB + driver.readFile( userB, passB, root + "/dirAB700/fileAB700", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN ); + driver.readFile( userB, passB, root + "/dirAB770/fileAB770", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN ); + driver.readFile( userB, passB, root + "/dirAB707/fileAB707", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN ); + driver.readFile( userB, passB, root + "/dirAB777/fileAB777", "text/plain", "small1.txt", HttpStatus.SC_OK ); + // userC:groupC + driver.readFile( userC, passC, root + "/dirA700/fileA700", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN ); + driver.readFile( userC, passC, root + "/dirA770/fileA770", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN ); + driver.readFile( userC, passC, root + "/dirA707/fileA707", "text/plain", "small1.txt", HttpStatus.SC_OK ); + driver.readFile( userC, passC, root + "/dirA777/fileA777", "text/plain", "small1.txt", HttpStatus.SC_OK ); + + //NEGATIVE: Test a bad password. + if( driver.isUseGateway() ) { + Response response = given() + //.log().all() + .auth().preemptive().basic( userA, "invalid-password" ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .queryParam( "op", "OPEN" ) + .expect() + //.log().all() + .statusCode( HttpStatus.SC_UNAUTHORIZED ) + .when().get( driver.getUrl("WEBHDFS") + "/v1" + root + "/dirA700/fileA700" ); + } + driver.assertComplete(); + + // UPDATE (Negative First) + driver.updateFile( userC, passC, root + "/dirA700/fileA700", "text/plain", "small2.txt", 307, 403 ); + driver.updateFile( userB, passB, root + "/dirAB700/fileAB700", "text/plain", "small2.txt", 307, 403 ); + driver.updateFile( userB, passB, root + "/dirAB770/fileAB700", "text/plain", "small2.txt", 307, 403 ); + driver.updateFile( userB, passB, root + "/dirAB770/fileAB770", "text/plain", "small2.txt", 307, 403 ); + driver.updateFile( userA, passA, root + "/dirA700/fileA700", "text/plain", "small2.txt", 307, 201 ); + + // DELETE (Negative First) + driver.deleteFile( userC, passC, root + "/dirA700/fileA700", "false", HttpStatus.SC_FORBIDDEN ); + driver.deleteFile( userB, passB, root + "/dirAB700/fileAB700", "false", HttpStatus.SC_FORBIDDEN ); + driver.deleteFile( userB, passB, root + "/dirAB770/fileAB770", "false", HttpStatus.SC_FORBIDDEN ); + driver.deleteFile( userA, passA, root + "/dirA700/fileA700", "false", HttpStatus.SC_OK ); + + // Cleanup anything that might have been leftover because the test failed previously. + driver.deleteFile( userA, passA, root, "true", HttpStatus.SC_OK ); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testJavaMapReduceViaWebHCat() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayBasicFuncTest/testJavaMapReduceViaWebHCat"; + String user = "mapred"; + String pass = "mapred-password"; + String group = "mapred"; +// String user = "hcat"; +// String pass = "hcat-password"; +// String group = "hcat"; + + // Cleanup anything that might have been leftover because the test failed previously. + driver.deleteFile( user, pass, root, "true", HttpStatus.SC_OK ); + + /* Put the mapreduce code into HDFS. (hadoop-examples.jar) + curl -X PUT --data-binary @hadoop-examples.jar 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/hadoop-examples.jar?user.name=hdfs&op=CREATE' + */ + driver.createFile( user, pass, null, root+"/hadoop-examples.jar", "777", "application/octet-stream", findHadoopExamplesJar(), 307, 201, 200 ); + + /* Put the data file into HDFS (changes.txt) + curl -X PUT --data-binary @changes.txt 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input/changes.txt?user.name=hdfs&op=CREATE' + */ + driver.createFile( user, pass, null, root+"/input/changes.txt", "777", "text/plain", "changes.txt", 307, 201, 200 ); + + /* Create the output directory + curl -X PUT 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/output?op=MKDIRS&user.name=hdfs' + */ + driver.createDir( user, pass, null, root+"/output", "777", 200, 200 ); + + /* Submit the job + curl -d user.name=hdfs -d jar=wordcount/hadoop-examples.jar -d class=org.apache.org.apache.hadoop.examples.WordCount -d arg=wordcount/input -d arg=wordcount/output 'http://localhost:8888/org.apache.org.apache.hadoop.gateway/cluster/templeton/v1/mapreduce/jar' + {"id":"job_201210301335_0059"} + */ + String job = driver.submitJava( + user, pass, + root+"/hadoop-examples.jar", "org.apache.org.apache.hadoop.examples.WordCount", + root+"/input", root+"/output", + 200 ); + + /* Get the job status + curl 'http://vm:50111/templeton/v1/queue/:jobid?user.name=hdfs' + */ + driver.queryQueue( user, pass, job ); + + // Can't really check for the output here because the job won't be done. + /* Retrieve results + curl 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input?op=LISTSTATUS' + */ + + if( CLEANUP_TEST ) { + // Cleanup anything that might have been leftover because the test failed previously. + driver.deleteFile( user, pass, root, "true", HttpStatus.SC_OK ); + } + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testPigViaWebHCat() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayWebHCatFuncTest/testPigViaWebHCat"; + String user = "mapred"; + String pass = "mapred-password"; + String group = "mapred"; + + // Cleanup if previous run failed. + driver.deleteFile( user, pass, root, "true", 200, 404 ); + + // Post the data to HDFS + driver.createFile( user, pass, null, root + "/passwd.txt", "777", "text/plain", "passwd.txt", 307, 201, 200 ); + + // Post the script to HDFS + driver.createFile( user, pass, null, root+"/script.pig", "777", "text/plain", "script.pig", 307, 201, 200 ); + + // Create the output directory + driver.createDir( user, pass, null, root + "/output", "777", 200, 200 ); + + // Submit the job + driver.submitPig( user, pass, group, root + "/script.pig", "-v", root + "/output", 200 ); + + // Check job status (if possible) + // Check output (if possible) + + // Cleanup + driver.deleteFile( user, pass, root, "true", 200 ); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testHiveViaWebHCat() throws IOException { + LOG_ENTER(); + String user = "hive"; + String pass = "hive-password"; + String group = "hive"; + String root = "/tmp/GatewayWebHCatFuncTest/testHiveViaWebHCat"; + + // Cleanup if previous run failed. + driver.deleteFile( user, pass, root, "true", 200, 404 ); + + // Post the data to HDFS + + // Post the script to HDFS + driver.createFile(user, pass, null, root + "/script.hive", "777", "text/plain", "script.hive", 307, 201, 200); + + // Submit the job + driver.submitHive(user, pass, group, root + "/script.hive", root + "/output", 200); + + // Check job status (if possible) + // Check output (if possible) + + // Cleanup + driver.deleteFile( user, pass, root, "true", 200 ); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testOozieJobSubmission() throws Exception { + LOG_ENTER(); + String root = "/tmp/GatewayBasicFuncTest/testOozieJobSubmission"; + String user = "hdfs"; + String pass = "hdfs-password"; + String group = "hdfs"; + + // Cleanup anything that might have been leftover because the test failed previously. + driver.deleteFile( user, pass, root, "true", HttpStatus.SC_OK ); + + /* Put the workflow definition into HDFS */ + driver.createFile( user, pass, group, root+"/workflow.xml", "666", "application/octet-stream", "oozie-workflow.xml", 307, 201, 200 ); + + /* Put the mapreduce code into HDFS. (hadoop-examples.jar) + curl -X PUT --data-binary @hadoop-examples.jar 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/hadoop-examples.jar?user.name=hdfs&op=CREATE' + */ + driver.createFile( user, pass, group, root+"/lib/hadoop-examples.jar", "777", "application/octet-stream", findHadoopExamplesJar(), 307, 201, 200 ); + + /* Put the data file into HDFS (changes.txt) + curl -X PUT --data-binary @changes.txt 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input/changes.txt?user.name=hdfs&op=CREATE' + */ + driver.createFile( user, pass, group, root+"/input/changes.txt", "666", "text/plain", "changes.txt", 307, 201, 200 ); + + VelocityEngine velocity = new VelocityEngine(); + velocity.setProperty( RuntimeConstants.RUNTIME_LOG_LOGSYSTEM_CLASS, "org.apache.velocity.runtime.log.NullLogSystem" ); + velocity.setProperty( RuntimeConstants.RESOURCE_LOADER, "classpath" ); + velocity.setProperty( "classpath.resource.loader.class", ClasspathResourceLoader.class.getName() ); + velocity.init(); + + VelocityContext context = new VelocityContext(); + context.put( "userName", user ); + context.put( "nameNode", "hdfs://sandbox:8020" ); + context.put( "jobTracker", "sandbox:50300" ); + //context.put( "appPath", "hdfs://sandbox:8020" + root ); + context.put( "appPath", root ); + context.put( "inputDir", root + "/input" ); + context.put( "outputDir", root + "/output" ); + + //URL url = TestUtils.getResourceUrl( GatewayBasicFuncTest.class, "oozie-jobs-submit-request.xml" ); + //String name = url.toExternalForm(); + String name = TestUtils.getResourceName( this.getClass(), "oozie-jobs-submit-request.xml" ); + Template template = velocity.getTemplate( name ); + StringWriter sw = new StringWriter(); + template.merge( context, sw ); + String request = sw.toString(); + //System.out.println( "REQUEST=" + request ); + + /* Submit the job via Oozie. */ + String id = driver.oozieSubmitJob( user, pass, request, 201 ); + //System.out.println( "ID=" + id ); + + String success = "SUCCEEDED"; + String status = "UNKNOWN"; + long delay = 1000 * 1; // 1 second. + long limit = 1000 * 60; // 60 seconds. + long start = System.currentTimeMillis(); + while( System.currentTimeMillis() <= start+limit ) { + status = driver.oozieQueryJobStatus( user, pass, id, 200 ); + //System.out.println( "Status=" + status ); + if( success.equalsIgnoreCase( status ) ) { + break; + } else { + //System.out.println( "Status=" + status ); + Thread.sleep( delay ); + } + } + //System.out.println( "Status is " + status + " after " + ((System.currentTimeMillis()-start)/1000) + " seconds." ); + MatcherAssert.assertThat( status, is( success ) ); + + if( CLEANUP_TEST ) { + // Cleanup anything that might have been leftover because the test failed previously. + driver.deleteFile( user, pass, root, "true", HttpStatus.SC_OK ); + } + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testBasicHiveJDBCUseCase() throws IOException { + LOG_ENTER(); + String root = "/tmp/GatewayHiveJDBCFuncTest/testBasicHiveUseCase"; + String username = "hive"; + String password = "hive-password"; + InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; + + // This use case emulates simple JDBC scenario which consists of following steps: + // -open connection; + // -configure Hive using 'execute' statements (this also includes execution of 'close operation' requests internally); + // -execution of create table command; + // -execution of select from table command; + // Data insertion is omitted because it causes a lot of additional command during insertion/querying. + // All binary data was intercepted during real scenario and stored into files as array of bytes. + + // open session + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/open-session-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/open-session-result.bin" ) ) + .contentType( "application/x-thrift" ); + Response response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/open-session-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/open-session-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/open-session-result.bin" ) ) ); + + driver.assertComplete(); + + // execute 'set hive.fetch.output.serde=...' (is called internally be JDBC driver) + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-result.bin" ) ) ); + driver.assertComplete(); + + // close operation for execute 'set hive.fetch.output.serde=...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/close-operation-1-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/close-operation-1-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/close-operation-1-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/close-operation-1-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-1-result.bin" ) ) ); + driver.assertComplete(); + + // execute 'set hive.server2.http.path=...' (is called internally be JDBC driver) + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/execute-set-server2-http-path-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/execute-set-server2-http-path-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/execute-set-server2-http-path-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/execute-set-server2-http-path-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-server2-http-path-result.bin" ) ) ); + driver.assertComplete(); + + // close operation for execute 'set hive.server2.http.path=...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/close-operation-2-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/close-operation-2-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/close-operation-2-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/close-operation-2-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-2-result.bin" ) ) ); + driver.assertComplete(); + + // execute 'set hive.server2.servermode=...' (is called internally be JDBC driver) + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/execute-set-server2-servermode-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/execute-set-server2-servermode-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/execute-set-server2-servermode-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/execute-set-server2-servermode-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-server2-servermode-result.bin" ) ) ); + driver.assertComplete(); + + // close operation for execute 'set hive.server2.servermode=...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/close-operation-3-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/close-operation-3-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/close-operation-3-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/close-operation-3-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-3-result.bin" ) ) ); + driver.assertComplete(); + + // execute 'set hive.security.authorization.enabled=...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-result.bin" ) ) ); + driver.assertComplete(); + + // close operation for execute 'set hive.security.authorization.enabled=...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/close-operation-4-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/close-operation-4-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/close-operation-4-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/close-operation-4-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-4-result.bin" ) ) ); + driver.assertComplete(); + + // execute 'create table...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/execute-create-table-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/execute-create-table-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/execute-create-table-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/execute-create-table-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-create-table-result.bin" ) ) ); + driver.assertComplete(); + + // close operation for execute 'create table...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/close-operation-5-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/close-operation-5-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/close-operation-5-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/close-operation-5-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-5-result.bin" ) ) ); + driver.assertComplete(); + + // execute 'select * from...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/execute-select-from-table-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/execute-select-from-table-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/execute-select-from-table-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/execute-select-from-table-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-select-from-table-result.bin" ) ) ); + driver.assertComplete(); + + // execute 'GetResultSetMetadata' (is called internally be JDBC driver) + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/get-result-set-metadata-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/get-result-set-metadata-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/get-result-set-metadata-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/get-result-set-metadata-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/get-result-set-metadata-result.bin" ) ) ); + driver.assertComplete(); + + // execute 'FetchResults' (is called internally be JDBC driver) + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/fetch-results-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/fetch-results-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/fetch-results-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/fetch-results-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/fetch-results-result.bin" ) ) ); + driver.assertComplete(); + + // close operation for execute 'select * from...' + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/close-operation-6-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/close-operation-6-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/close-operation-6-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/close-operation-6-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-6-result.bin" ) ) ); + driver.assertComplete(); + + // close session + driver.getMock( "HIVE" ) + .expect() + .method( "POST" ) + .content( driver.getResourceBytes( "hive/close-session-request.bin" ) ) + .contentType( "application/x-thrift" ) + .respond() + .characterEncoding( "UTF-8" ) + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( "hive/close-session-result.bin" ) ) + .contentType( "application/x-thrift" ); + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .content( driver.getResourceBytes( "hive/close-session-request.bin" ) ) + .contentType( "application/x-thrift" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( "hive/close-session-result.bin" ) ) ) + .contentType( "application/x-thrift" ) + .when().post( driver.getUrl( "HIVE" ) ); + assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-session-result.bin" ) ) ); + driver.assertComplete(); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testHBaseGetTableList() throws IOException { + LOG_ENTER(); + String username = "hbase"; + String password = "hbase-password"; + String resourceName = "hbase/table-list"; + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "GET" ) + .pathInfo( "/" ) + .header( "Accept", ContentType.XML.toString() ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( resourceName + ".xml" ) ) + .contentType( ContentType.XML.toString() ); + + Response response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .header( "Accept", ContentType.XML.toString() ) + .expect() + .statusCode( HttpStatus.SC_OK ) + .contentType( ContentType.XML ) + .when().get( driver.getUrl( "WEBHBASE" ) ); + + MatcherAssert + .assertThat( + the( response.getBody().asString() ), + isEquivalentTo( the( driver.getResourceString( resourceName + ".xml", UTF8 ) ) ) ); + driver.assertComplete(); + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "GET" ) + .pathInfo( "/" ) + .header( "Accept", ContentType.JSON.toString() ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( resourceName + ".json" ) ) + .contentType( ContentType.JSON.toString() ); + + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .header( "Accept", ContentType.JSON.toString() ) + .expect() + .statusCode( HttpStatus.SC_OK ) + .contentType( ContentType.JSON ) + .when().get( driver.getUrl( "WEBHBASE" ) ); + + MatcherAssert + .assertThat( response.getBody().asString(), sameJSONAs( driver.getResourceString( resourceName + ".json", UTF8 ) ) ); + driver.assertComplete(); + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "GET" ) + .pathInfo( "/" ) + .header( "Accept", "application/x-protobuf" ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceString( resourceName + ".protobuf", UTF8 ), UTF8 ) + .contentType( "application/x-protobuf" ); + + given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .header( "Accept", "application/x-protobuf" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + .contentType( "application/x-protobuf" ) + .content( is( driver.getResourceString( resourceName + ".protobuf", UTF8 ) ) ) + .when().get( driver.getUrl( "WEBHBASE" ) ); + driver.assertComplete(); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testHBaseCreateTableAndVerifySchema() throws IOException { + LOG_ENTER(); + String username = "hbase"; + String password = "hbase-password"; + String resourceName = "hbase/table-schema"; + String path = "/table/schema"; + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "PUT" ) + .pathInfo( path ) + .respond() + .status( HttpStatus.SC_CREATED ) + .content( driver.getResourceBytes( resourceName + ".xml" ) ) + .contentType( ContentType.XML.toString() ) + .header( "Location", driver.getRealUrl( "WEBHBASE" ) + path ); + + given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .expect() + .statusCode( HttpStatus.SC_CREATED ) + .contentType( ContentType.XML ) + .header( "Location", startsWith( driver.getUrl( "WEBHBASE" ) + path ) ) + .when().put(driver.getUrl("WEBHBASE") + path); + driver.assertComplete(); + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "PUT" ) + .pathInfo( path ) + .respond() + .status(HttpStatus.SC_CREATED) + .content(driver.getResourceBytes(resourceName + ".json")) + .contentType(ContentType.JSON.toString()) + .header("Location", driver.getRealUrl("WEBHBASE") + path); + + given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .expect() + .statusCode( HttpStatus.SC_CREATED ) + .contentType( ContentType.JSON ) + .header( "Location", startsWith( driver.getUrl( "WEBHBASE" ) + path ) ) + .when().put( driver.getUrl( "WEBHBASE" ) + path ); + driver.assertComplete(); + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "PUT" ) + .pathInfo( path ) + .respond() + .status( HttpStatus.SC_CREATED ) + .content( driver.getResourceBytes( resourceName + ".protobuf" ) ) + .contentType( "application/x-protobuf" ) + .header("Location", driver.getRealUrl("WEBHBASE") + path); + + given() + .auth().preemptive().basic(username, password) + .header("X-XSRF-Header", "jksdhfkhdsf") + .expect() + .statusCode(HttpStatus.SC_CREATED) + .contentType("application/x-protobuf") + .header("Location", startsWith(driver.getUrl("WEBHBASE") + path)) + .when().put(driver.getUrl("WEBHBASE") + path); + driver.assertComplete(); + + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testHBaseGetTableSchema() throws IOException { + LOG_ENTER(); + String username = "hbase"; + String password = "hbase-password"; + String resourceName = "hbase/table-metadata"; + String path = "/table/schema"; + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "GET" ) + .pathInfo( path ) + .header("Accept", ContentType.XML.toString()) + .respond() + .status(HttpStatus.SC_OK) + .content(driver.getResourceBytes(resourceName + ".xml")) + .contentType(ContentType.XML.toString()); + + Response response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .header( "Accept", ContentType.XML.toString() ) + .expect() + .statusCode( HttpStatus.SC_OK ) + .contentType( ContentType.XML ) + .when().get( driver.getUrl( "WEBHBASE" ) + path ); + + MatcherAssert + .assertThat( + the(response.getBody().asString()), + isEquivalentTo(the(driver.getResourceString(resourceName + ".xml", UTF8)))); + driver.assertComplete(); + + driver.getMock("WEBHBASE") + .expect() + .method("GET") + .pathInfo(path) + .header("Accept", ContentType.JSON.toString()) + .respond() + .status(HttpStatus.SC_OK) + .content(driver.getResourceBytes(resourceName + ".json")) + .contentType(ContentType.JSON.toString()); + + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .header( "Accept", ContentType.JSON.toString() ) + .expect() + .statusCode( HttpStatus.SC_OK ) + .contentType( ContentType.JSON ) + .when().get( driver.getUrl( "WEBHBASE" ) + path ); + + MatcherAssert + .assertThat(response.getBody().asString(), sameJSONAs(driver.getResourceString(resourceName + ".json", UTF8))); + driver.assertComplete(); + + driver.getMock( "WEBHBASE" ) + .expect() + .method( "GET" ) + .pathInfo( path ) + .header( "Accept", "application/x-protobuf" ) + .respond() + .status( HttpStatus.SC_OK ) + .content( driver.getResourceBytes( resourceName + ".protobuf" ) ) + .contentType("application/x-protobuf"); + + response = given() + .auth().preemptive().basic( username, password ) + .header("X-XSRF-Header", "jksdhfkhdsf") + .header( "Accept", "application/x-protobuf" ) + .expect() + .statusCode( HttpStatus.SC_OK ) + //.content( is( driver.getResourceBytes( resourceName + ".protobuf" ) ) ) + .contentType( "application/x-protobuf" ) + .when().get( driver.getUrl( "WEBHBASE" ) + path ); + // RestAssured seems to be screwing up the binary comparison so do it explicitly. + assertThat( driver.getResourceBytes( resourceName + ".protobuf" ), is( response.body().asByteArray() ) ); + driver.assertComplete(); + LOG_EXIT(); + } + + @Test( timeout = MEDIUM_TIMEOUT ) + public void testHBaseInsertDataIntoTable() throws IOException { + LOG_ENTER(); + String username = "hbase"; + String password = "hbase-password"; + + String resourceName = "hbase/table-data"; + String singleRowPath = "/table/testrow"; + String multipleRowPath = "/table/false-row-key"; + + //PUT request + + driver.getMoc
<TRUNCATED>
