Repository: nutch Updated Branches: refs/heads/2.x 5012c742c -> 9ecdc9be3
NUTCH-2301 Tests for Security Layer of NutchServer Are Created Project: http://git-wip-us.apache.org/repos/asf/nutch/repo Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/3bc3d81e Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/3bc3d81e Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/3bc3d81e Branch: refs/heads/2.x Commit: 3bc3d81e964aac59f61951740e848bd429a15b3c Parents: 22683a1 Author: Furkan KAMACI <[email protected]> Authored: Tue Aug 23 23:41:43 2016 +0300 Committer: Furkan KAMACI <[email protected]> Committed: Tue Aug 23 23:41:43 2016 +0300 ---------------------------------------------------------------------- src/test/nutch-site.xml | 48 ++++ src/test/nutch-ssl.keystore.jks | Bin 0 -> 2300 bytes .../nutch/api/AbstractNutchAPITestBase.java | 186 +++++++++++++++ src/test/org/apache/nutch/api/TestAPI.java | 225 ------------------- src/test/org/apache/nutch/api/TestNutchAPI.java | 100 +++++++++ 5 files changed, 334 insertions(+), 225 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nutch/blob/3bc3d81e/src/test/nutch-site.xml ---------------------------------------------------------------------- diff --git a/src/test/nutch-site.xml b/src/test/nutch-site.xml index e599547..4dbce0c 100644 --- a/src/test/nutch-site.xml +++ b/src/test/nutch-site.xml @@ -29,4 +29,52 @@ </description> </property> +<property> + <name>restapi.auth</name> + <value>NONE</value> + <description> + Configures authentication type for communicating with RESTAPI. Valid values are BASIC, DIGEST, SSL and NONE. + When no authentication type is defined NONE will be used as default which does not provide security. + Use the restapi.auth.username and restapi.auth.password properties to configure + your credentials if security is used. + </description> +</property> + +<property> + <name>restapi.auth.users</name> + <value>admin|admin|admin,user|user|user</value> + <description> + Username, password and role combination for REST API authentication/authorization. restapi.auth property should be set to either BASIC or DIGEST to use this property. + Username, password and role should be delimited by pipe character (|) Every user should be separated with comma character (,). i.e. admin|admin|admin,user|user|user. + Default is admin|admin|admin,user|user|user + </description> +</property> + +<property> + <name>restapi.auth.ssl.storepath</name> + <value>nutch-ssl.keystore.jks</value> + <description> + Key store path for jks file. restapi.auth property should be set to SSL to use this property. + etc/nutch-ssl.keystore.jks is used for restapi.auth.ssl.storepath as default. + </description> +</property> + +<property> + <name>restapi.auth.ssl.storepass</name> + <value>password</value> + <description> + Key store path for jks file. restapi.auth property should be set to SSL to use this property. + "password" is used for restapi.auth.ssl.storepass as default. + </description> +</property> + +<property> + <name>restapi.auth.ssl.keypass</name> + <value>password</value> + <description> + Key store path for jks file. restapi.auth property should be set to SSL to use this property. + "password" is used for restapi.auth.ssl.keypass as default. + </description> +</property> + </configuration> http://git-wip-us.apache.org/repos/asf/nutch/blob/3bc3d81e/src/test/nutch-ssl.keystore.jks ---------------------------------------------------------------------- diff --git a/src/test/nutch-ssl.keystore.jks b/src/test/nutch-ssl.keystore.jks new file mode 100644 index 0000000..9d0bd01 Binary files /dev/null and b/src/test/nutch-ssl.keystore.jks differ http://git-wip-us.apache.org/repos/asf/nutch/blob/3bc3d81e/src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java b/src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java new file mode 100644 index 0000000..3cdff23 --- /dev/null +++ b/src/test/org/apache/nutch/api/AbstractNutchAPITestBase.java @@ -0,0 +1,186 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.nutch.api; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.api.impl.RAMConfManager; +import org.apache.nutch.api.security.AuthenticationTypeEnum; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.BeforeClass; +import org.restlet.data.ChallengeRequest; +import org.restlet.data.ChallengeResponse; +import org.restlet.data.ChallengeScheme; +import org.restlet.resource.ClientResource; +import org.restlet.resource.ResourceException; + +import static org.junit.Assert.assertEquals; + +/** + * Base class for Nutc REST API tests. + * Default path is /admin and default port is 8081. + */ +public abstract class AbstractNutchAPITestBase { + protected static Configuration conf; + protected static final Integer DEFAULT_PORT = 8081; + protected static final String DEFAULT_PATH = "/admin"; + public NutchServer nutchServer; + + /** + * Creates Nutch configuration + * + * @throws Exception + */ + @BeforeClass + public static void before() throws Exception { + conf = NutchConfiguration.create(); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + */ + public void testRequest(int expectedStatusCode, int port) { + testRequest(expectedStatusCode, port, null, null, DEFAULT_PATH, null); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * with given username and password credentials + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + * @param username username + * @param password password + */ + public void testRequest(int expectedStatusCode, int port, String username, String password) { + testRequest(expectedStatusCode, port, username, password, DEFAULT_PATH, null); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * with given username and password credentials for a {@link ChallengeScheme} + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + * @param username username + * @param password password + * @param challengeScheme challenge scheme + */ + public void testRequest(int expectedStatusCode, int port, String username, String password, ChallengeScheme challengeScheme) { + testRequest(expectedStatusCode, port, username, password, DEFAULT_PATH, challengeScheme); + } + + /** + * Tests a request for given expected status code for {@link NutchServer} from which port it runs + * with given username and password credentials for a {@link ChallengeScheme} + * + * @param expectedStatusCode expected status code + * @param port port that {@link NutchServer} runs at + * @param username username + * @param password password + * @param path path + * @param challengeScheme challenge scheme + */ + public void testRequest(int expectedStatusCode, int port, String username, String password, String path, + ChallengeScheme challengeScheme) { + if (port <= 0) { + port = DEFAULT_PORT; + } + + if (!path.startsWith("/")) { + path += "/"; + } + String protocol = AuthenticationTypeEnum.SSL.toString().equals(conf.get("restapi.auth")) ? "https" : "http"; + ClientResource resource = new ClientResource(protocol + "://localhost:" + port + path); + if (challengeScheme != null) { + resource.setChallengeResponse(challengeScheme, username, password); + } + + try { + resource.get(); + } catch (ResourceException rex) { + //Don't use it. Use the status code to check afterwards + } + + /* + If request was a HTTP Digest request, previous request was 401. Client needs some data sent by the server so + we should complete the ChallengeResponse. + */ + + if (ChallengeScheme.HTTP_DIGEST.equals(challengeScheme)) { + // User server's data to complete the challengeResponse object + ChallengeRequest digestChallengeRequest = retrieveDigestChallengeRequest(resource); + ChallengeResponse challengeResponse = new ChallengeResponse(digestChallengeRequest, resource.getResponse(), + username, password.toCharArray()); + testDigestRequest(expectedStatusCode, resource, challengeResponse); + } + assertEquals(expectedStatusCode, resource.getStatus().getCode()); + } + + private void testDigestRequest(int expectedStatusCode, ClientResource resource, ChallengeResponse challengeResponse){ + resource.setChallengeResponse(challengeResponse); + try { + resource.get(); + } catch (ResourceException rex) { + //Don't use it. Use the status code to check afterwards + } + assertEquals(expectedStatusCode, resource.getStatus().getCode()); + } + + private ChallengeRequest retrieveDigestChallengeRequest (ClientResource resource) { + ChallengeRequest digestChallengeRequest = null; + for (ChallengeRequest cr : resource.getChallengeRequests()) { + if (ChallengeScheme.HTTP_DIGEST.equals(cr.getScheme())) { + digestChallengeRequest = cr; + break; + } + } + return digestChallengeRequest; + } + + private void initializeSSLProperties() { + conf.set("restapi.auth.ssl.storepath", "src/test/nutch-ssl.keystore.jks"); + conf.set("restapi.auth.ssl.storepass", "password"); + conf.set("restapi.auth.ssl.keypass", "password"); + } + + /** + * Starts the server with given authentication type + * + * @param authenticationType authentication type + */ + public void startServer(AuthenticationTypeEnum authenticationType) { + conf.set("restapi.auth", authenticationType.toString()); + if(AuthenticationTypeEnum.SSL.equals(authenticationType)) { + initializeSSLProperties(); + } + + RAMConfManager ramConfManager = new RAMConfManager(NutchConfiguration.getUUID(conf), conf); + nutchServer = new NutchServer(ramConfManager, NutchConfiguration.getUUID(conf)); + nutchServer.start(); + } + + /** + * Stops server + */ + public void stopServer() { + nutchServer.stop(true); + } + +} http://git-wip-us.apache.org/repos/asf/nutch/blob/3bc3d81e/src/test/org/apache/nutch/api/TestAPI.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/nutch/api/TestAPI.java b/src/test/org/apache/nutch/api/TestAPI.java deleted file mode 100644 index 28e999a..0000000 --- a/src/test/org/apache/nutch/api/TestAPI.java +++ /dev/null @@ -1,225 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -// CURRENTLY DISABLED. TESTS ARE FLAPPING FOR NO APPARENT REASON. -// SHALL BE FIXED OR REPLACES BY NEW API IMPLEMENTATION - -package org.apache.nutch.api; - -//import static org.junit.Assert.*; -// -//import java.util.HashMap; -//import java.util.Map; -// -//import org.apache.nutch.api.JobManager.JobType; -//import org.apache.nutch.metadata.Nutch; -//import org.apache.nutch.util.NutchTool; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -//import org.restlet.ext.jackson.JacksonRepresentation; -//import org.restlet.representation.Representation; -//import org.restlet.resource.ClientResource; - -public class TestAPI { - @Test - public void test() throws Exception { - } - // - // private static NutchServer server; - // ClientResource cli; - // - // private static String baseUrl = "http://localhost:8192/nutch/";; - // - // @BeforeClass - // public static void before() throws Exception { - // server = new NutchServer(8192); - // server.start(); - // } - // - // @AfterClass - // public static void after() throws Exception { - // if (!server.stop(false)) { - // for (int i = 1; i < 11; i++) { - // System.err.println("Waiting for jobs to complete - " + i + "s"); - // try { - // Thread.sleep(1000); - // } catch (Exception e) {}; - // server.stop(false); - // if (!server.isRunning()) { - // break; - // } - // } - // } - // if (server.isRunning()) { - // System.err.println("Forcibly stopping server..."); - // server.stop(true); - // } - // } - // - // @Test - // public void testInfoAPI() throws Exception { - // ClientResource cli = new ClientResource(baseUrl); - // String expected = - // "[[\"admin\",\"Service admin actions\"],[\"confs\",\"Configuration manager\"],[\"db\",\"DB data streaming\"],[\"jobs\",\"Job manager\"]]"; - // String got = cli.get().getText(); - // assertEquals(expected, got); - // } - // - // @SuppressWarnings("rawtypes") - // @Test - // public void testConfsAPI() throws Exception { - // ClientResource cli = new ClientResource(baseUrl + ConfResource.PATH); - // assertEquals("[\"default\"]", cli.get().getText()); - // // create - // Map<String,Object> map = new HashMap<String,Object>(); - // map.put(Params.CONF_ID, "test"); - // HashMap<String,String> props = new HashMap<String,String>(); - // props.put("testProp", "blurfl"); - // map.put(Params.PROPS, props); - // JacksonRepresentation<Map<String,Object>> jr = - // new JacksonRepresentation<Map<String,Object>>(map); - // System.out.println(cli.put(jr).getText()); - // assertEquals("[\"default\",\"test\"]", cli.get().getText()); - // cli = new ClientResource(baseUrl + ConfResource.PATH + "/test"); - // Map res = cli.get(Map.class); - // assertEquals("blurfl", res.get("testProp")); - // // delete - // cli.delete(); - // cli = new ClientResource(baseUrl + ConfResource.PATH); - // assertEquals("[\"default\"]", cli.get().getText()); - // } - // - // @SuppressWarnings("rawtypes") - // @Test - // public void testJobsAPI() throws Exception { - // ClientResource cli = new ClientResource(baseUrl + JobResource.PATH); - // assertEquals("[]", cli.get().getText()); - // // create - // Map<String,Object> map = new HashMap<String,Object>(); - // map.put(Params.JOB_TYPE, JobType.READDB.toString()); - // map.put(Params.CONF_ID, "default"); - // Representation r = cli.put(map); - // String jobId = r.getText(); - // assertNotNull(jobId); - // assertTrue(jobId.startsWith("default-READDB-")); - // // list - // Map[] list = cli.get(Map[].class); - // assertEquals(1, list.length); - // String id = (String)list[0].get("id"); - // String state = (String)list[0].get("state"); - // assertEquals(jobId, id); - // assertEquals(state, "RUNNING"); - // int cnt = 10; - // do { - // try { - // Thread.sleep(2000); - // } catch (Exception e) {}; - // list = cli.get(Map[].class); - // state = (String)list[0].get("state"); - // if (!state.equals("RUNNING")) { - // break; - // } - // } while (--cnt > 0); - // assertTrue(cnt > 0); - // if (list == null) return; - // for (Map m : list) { - // System.out.println(m); - // } - // } - // - // @SuppressWarnings("unchecked") - // @Test - // public void testStopKill() throws Exception { - // ClientResource cli = new ClientResource(baseUrl + JobResource.PATH); - // // create - // Map<String,Object> map = new HashMap<String,Object>(); - // map.put(Params.JOB_TYPE, JobType.CLASS.toString()); - // Map<String,Object> args = new HashMap<String,Object>(); - // map.put(Params.ARGS, args); - // args.put(Nutch.ARG_CLASS, SpinningJob.class.getName()); - // map.put(Params.CONF_ID, "default"); - // Representation r = cli.put(map); - // String jobId = r.getText(); - // cli.release(); - // assertNotNull(jobId); - // System.out.println(jobId); - // assertTrue(jobId.startsWith("default-CLASS-")); - // ClientResource stopCli = new ClientResource(baseUrl + JobResource.PATH + - // "?job=" + jobId + "&cmd=stop"); - // r = stopCli.get(); - // assertEquals("true", r.getText()); - // stopCli.release(); - // Thread.sleep(2000); // wait for the job to finish - // ClientResource jobCli = new ClientResource(baseUrl + JobResource.PATH + "/" - // + jobId); - // Map<String,Object> res = jobCli.get(Map.class); - // res = (Map<String,Object>)res.get("result"); - // assertEquals("stopped", res.get("res")); - // jobCli.release(); - // // restart and kill - // r = cli.put(map); - // jobId = r.getText(); - // cli.release(); - // assertNotNull(jobId); - // System.out.println(jobId); - // assertTrue(jobId.startsWith("default-CLASS-")); - // ClientResource killCli = new ClientResource(baseUrl + JobResource.PATH + - // "?job=" + jobId + "&cmd=abort"); - // r = killCli.get(); - // assertEquals("true", r.getText()); - // killCli.release(); - // Thread.sleep(2000); // wait for the job to finish - // jobCli = new ClientResource(baseUrl + JobResource.PATH + "/" + jobId); - // res = jobCli.get(Map.class); - // res = (Map<String,Object>)res.get("result"); - // assertEquals("killed", res.get("res")); - // jobCli.release(); - // } - // - // public static class SpinningJob extends NutchTool { - // volatile boolean shouldStop = false; - // - // @Override - // public Map<String, Object> run(Map<String, Object> args) throws Exception { - // status.put(Nutch.STAT_MESSAGE, "running"); - // int cnt = 60; - // while (!shouldStop && cnt-- > 0) { - // Thread.sleep(1000); - // } - // if (cnt == 0) { - // results.put("res", "failed"); - // } - // return results; - // } - // - // @Override - // public boolean stopJob() throws Exception { - // results.put("res", "stopped"); - // shouldStop = true; - // return true; - // } - // - // @Override - // public boolean killJob() throws Exception { - // results.put("res", "killed"); - // shouldStop = true; - // return true; - // } - // - // } -} http://git-wip-us.apache.org/repos/asf/nutch/blob/3bc3d81e/src/test/org/apache/nutch/api/TestNutchAPI.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/nutch/api/TestNutchAPI.java b/src/test/org/apache/nutch/api/TestNutchAPI.java new file mode 100644 index 0000000..56aed92 --- /dev/null +++ b/src/test/org/apache/nutch/api/TestNutchAPI.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.nutch.api; + +import org.apache.nutch.api.security.AuthenticationTypeEnum; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Test; +import org.restlet.data.ChallengeScheme; + +/** + * Test class for {@link org.apache.nutch.api.NutchServer} + */ +public class TestNutchAPI extends AbstractNutchAPITestBase { + + /** + * Test insecure connection + */ + @Test + public void testInsecure() { + startServer(AuthenticationTypeEnum.NONE); + testRequest(200, 8081); + } + + /** + * Test Basic Authentication for invalid username/password pair, + * authorized username/password pair and insufficient privileged username/password pair + */ + @Test + public void testBasicAuth() { + startServer(AuthenticationTypeEnum.BASIC); + //Check for an invalid username/password pair + testRequest(401, 8081, "xxx", "xxx", ChallengeScheme.HTTP_BASIC); + + //Check for an authorized username/password pair + testRequest(200, 8081, "admin", "admin", ChallengeScheme.HTTP_BASIC); + + //Check for an insufficient privileged username/password pair + testRequest(403, 8081, "user", "user", ChallengeScheme.HTTP_BASIC); + } + + /** + * Test Digest Authentication for invalid username/password pair, + * authorized username/password pair and insufficient privileged username/password pair + */ + @Test + public void testDigestAuth() { + startServer(AuthenticationTypeEnum.DIGEST); + //Check for an invalid username/password pair + testRequest(401, 8081, "xxx", "xxx", ChallengeScheme.HTTP_DIGEST); + + //Check for an authorized username/password pair + testRequest(200, 8081, "admin", "admin", ChallengeScheme.HTTP_DIGEST); + + //Check for an insufficient privileged username/password pair + testRequest(403, 8081, "user", "user", ChallengeScheme.HTTP_DIGEST); + } + + /** + * Test SSL for invalid username/password pair, + * authorized username/password pair and insufficient privileged username/password pair + */ + @Ignore + @Test + public void testSSL() { + startServer(AuthenticationTypeEnum.SSL); + //Check for an invalid username/password pair + testRequest(401, 8081, "xxx", "xxx"); + + //Check for an authorized username/password pair + testRequest(200, 8081, "admin", "admin"); + + //Check for an insufficient privileged username/password pair + testRequest(403, 8081, "user", "user"); + } + + /** + * Stops the {@link NutchServer} + */ + @After + public void tearDown() { + stopServer(); + } + +} +
