Repository: nutch Updated Branches: refs/heads/2.x c210b9f2c -> b7f3fce42
NUTCH-2294 Authorization Support for REST API Project: http://git-wip-us.apache.org/repos/asf/nutch/repo Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/b7f3fce4 Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/b7f3fce4 Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/b7f3fce4 Branch: refs/heads/2.x Commit: b7f3fce42acdba00ca53e3be98acb78fc37bd168 Parents: c210b9f Author: Furkan KAMACI <[email protected]> Authored: Fri Aug 19 23:48:58 2016 +0300 Committer: Furkan KAMACI <[email protected]> Committed: Sat Aug 20 03:58:44 2016 +0300 ---------------------------------------------------------------------- conf/nutch-default.xml | 18 +--- ivy/ivy.xml | 8 +- src/java/org/apache/nutch/api/NutchServer.java | 28 ++--- .../nutch/api/resources/AdminResource.java | 16 ++- .../nutch/api/resources/ConfigResource.java | 9 ++ .../apache/nutch/api/resources/DbResource.java | 7 ++ .../apache/nutch/api/resources/JobResource.java | 7 ++ .../nutch/api/resources/SeedResource.java | 12 ++- .../api/security/AuthorizationRoleEnum.java | 38 +++++++ .../apache/nutch/api/security/SecurityUtil.java | 107 +++++++++++++++++++ 10 files changed, 214 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/conf/nutch-default.xml ---------------------------------------------------------------------- diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml index 10904a2..f1a16fc 100644 --- a/conf/nutch-default.xml +++ b/conf/nutch-default.xml @@ -1447,20 +1447,12 @@ </property> <property> - <name>restapi.auth.username</name> - <value>admin</value> + <name>restapi.auth.users</name> + <value>admin|admin|admin,user|user|user</value> <description> - Username for REST API authentication. restapi.auth property should be set to either BASIC or DIGEST to use this property. - "admin" is used for username as default. - </description> -</property> - -<property> - <name>restapi.auth.password</name> - <value>nutch</value> - <description> - Password for REST API authentication. restapi.auth property should be set to either BASIC or DIGEST to use this property. - "nutch" is used for password as default. + Username, password and role combination for REST API authentication/authorization. restapi.auth property should be set to either BASIC or DIGEST to use this property. + Username, password and role should be delimited by pipe character (|) Every user should be separated with comma character (,). i.e. admin|admin|admin,user|user|user. + Default is admin|admin|admin,user|user|user </description> </property> http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/ivy/ivy.xml ---------------------------------------------------------------------- diff --git a/ivy/ivy.xml b/ivy/ivy.xml index c909323..db42162 100644 --- a/ivy/ivy.xml +++ b/ivy/ivy.xml @@ -76,10 +76,10 @@ <dependency org="com.google.guava" name="guava" rev="11.0.2" /> <dependency org="com.google.code.crawler-commons" name="crawler-commons" rev="0.5" /> - <dependency org="org.restlet.jse" name="org.restlet" rev="2.3.7" conf="*->default" /> - <dependency org="org.restlet.jse" name="org.restlet.ext.jackson" rev="2.3.7" conf="*->default" /> - <dependency org="org.restlet.jse" name="org.restlet.ext.jaxrs" rev="2.3.7" conf="*->default" /> - <dependency org="org.restlet.jee" name="org.restlet.ext.crypto" rev="2.3.7" conf="*->default" /> + <dependency org="org.restlet.jse" name="org.restlet" rev="2.2.3" conf="*->default" /> + <dependency org="org.restlet.jse" name="org.restlet.ext.jackson" rev="2.2.3" conf="*->default" /> + <dependency org="org.restlet.jse" name="org.restlet.ext.jaxrs" rev="2.2.3" conf="*->default" /> + <dependency org="org.restlet.jee" name="org.restlet.ext.crypto" rev="2.2.3" conf="*->default" /> <!--artifacts needed for testing --> <dependency org="junit" name="junit" rev="4.11" conf="*->default" /> http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/NutchServer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/NutchServer.java b/src/java/org/apache/nutch/api/NutchServer.java index b5ca6e8..3bdfc6c 100644 --- a/src/java/org/apache/nutch/api/NutchServer.java +++ b/src/java/org/apache/nutch/api/NutchServer.java @@ -44,6 +44,7 @@ import org.apache.nutch.api.resources.DbResource; import org.apache.nutch.api.resources.JobResource; import org.apache.nutch.api.resources.SeedResource; import org.apache.nutch.api.security.AuthenticationTypeEnum; +import org.apache.nutch.api.security.SecurityUtil; import org.restlet.Component; import org.restlet.Context; import org.restlet.Server; @@ -52,9 +53,10 @@ import org.restlet.data.Protocol; import org.restlet.data.Reference; import org.restlet.ext.jaxrs.JaxRsApplication; import org.restlet.resource.ClientResource; -import org.restlet.security.ChallengeAuthenticator; import org.restlet.ext.crypto.DigestAuthenticator; -import org.restlet.security.MapVerifier; +import org.restlet.security.ChallengeAuthenticator; +import org.restlet.security.LocalVerifier; +import org.restlet.security.MemoryRealm; import org.restlet.util.Series; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,6 +149,7 @@ public class NutchServer extends Application { application.add(this); application.setStatusService(new ErrorStatusService()); childContext.getAttributes().put(NUTCH_SERVER, this); + application.setRoles(SecurityUtil.getRoles(application)); switch (authenticationType) { case NONE: @@ -155,19 +158,27 @@ public class NutchServer extends Application { break; case BASIC: ChallengeAuthenticator challengeGuard = new ChallengeAuthenticator(null, ChallengeScheme.HTTP_BASIC, "Nutch REST API Realm"); - challengeGuard.setVerifier(retrieveServerCredentials()); + //Create in-memory users with roles + MemoryRealm basicAuthRealm = SecurityUtil.constructRealm(application, configManager); + //Attach verifier to check authentication and enroler to determine roles + challengeGuard.setVerifier(basicAuthRealm.getVerifier()); + challengeGuard.setEnroler(basicAuthRealm.getEnroler()); challengeGuard.setNext(application); // Attach the application with HTTP basic authentication security component.getDefaultHost().attach(challengeGuard); break; case DIGEST: DigestAuthenticator digestGuard = new DigestAuthenticator(null, "Nutch REST API Realm", "NutchSecretKey"); - digestGuard.setWrappedVerifier(retrieveServerCredentials()); + //Create in-memory users with roles + MemoryRealm digestAuthRealm = SecurityUtil.constructRealm(application, configManager); + digestGuard.setWrappedVerifier((LocalVerifier) digestAuthRealm.getVerifier()); + digestGuard.setEnroler(digestAuthRealm.getEnroler()); digestGuard.setNext(application); // Attach the application with digest authentication security component.getDefaultHost().attachDefault(digestGuard); break; default: + LOG.error("Unsupported Server Security Type!"); throw new IllegalStateException("Unsupported Server Security Type!"); } @@ -345,13 +356,4 @@ public class NutchServer extends Application { return options; } - private MapVerifier retrieveServerCredentials() { - MapVerifier mapVerifier = new MapVerifier(); - - String username = configManager.get(ConfigResource.DEFAULT).get("restapi.auth.username", "admin"); - String password = configManager.get(ConfigResource.DEFAULT).get("restapi.auth.password", "nutch"); - mapVerifier.getLocalSecrets().put(username, password.toCharArray()); - - return mapVerifier; - } } http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/resources/AdminResource.java b/src/java/org/apache/nutch/api/resources/AdminResource.java index 03ff4b5..58d08b4 100644 --- a/src/java/org/apache/nutch/api/resources/AdminResource.java +++ b/src/java/org/apache/nutch/api/resources/AdminResource.java @@ -22,12 +22,17 @@ import java.util.concurrent.TimeUnit; import javax.ws.rs.GET; import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.SecurityContext; + import org.apache.nutch.api.model.response.NutchStatus; import org.apache.nutch.api.model.response.JobInfo.State; +import org.apache.nutch.api.security.SecurityUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,11 +44,14 @@ public class AdminResource extends AbstractResource { private static final Logger LOG = LoggerFactory .getLogger(AdminResource.class); + @Context + SecurityContext securityContext; + @GET @Path("/") - public NutchStatus getNutchStatus() { + public NutchStatus getNutchStatus(@Context HttpHeaders headers) { + SecurityUtil.allowOnlyAdmin(securityContext); NutchStatus status = new NutchStatus(); - status.setStartDate(new Date(server.getStarted())); status.setConfiguration(configManager.list()); status.setJobs(jobManager.list(null, State.ANY)); @@ -56,6 +64,7 @@ public class AdminResource extends AbstractResource { @Path("/stop") @Produces(MediaType.TEXT_PLAIN) public String stop(@QueryParam("force") boolean force) { + SecurityUtil.allowOnlyAdmin(securityContext); if (!server.canStop(force)) { LOG.info("Command 'stop' denied due to unfinished jobs"); return "Can't stop now. There are jobs running. Try force option."; @@ -66,6 +75,7 @@ public class AdminResource extends AbstractResource { } private void scheduleServerStop() { + SecurityUtil.allowOnlyAdmin(securityContext); LOG.info("Server shutdown scheduled in {} seconds", DELAY_SEC); Thread thread = new Thread() { public void run() { http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/resources/ConfigResource.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/resources/ConfigResource.java b/src/java/org/apache/nutch/api/resources/ConfigResource.java index 0c546f6..bc65826 100644 --- a/src/java/org/apache/nutch/api/resources/ConfigResource.java +++ b/src/java/org/apache/nutch/api/resources/ConfigResource.java @@ -29,16 +29,22 @@ import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import javax.ws.rs.core.SecurityContext; import org.apache.nutch.api.model.request.NutchConfig; +import org.apache.nutch.api.security.SecurityUtil; @Path("/config") public class ConfigResource extends AbstractResource { public static final String DEFAULT = "default"; + @Context + SecurityContext securityContext; + @GET @Path("/") public Set<String> getConfigs() { @@ -62,6 +68,7 @@ public class ConfigResource extends AbstractResource { @DELETE @Path("/{configId}") public void deleteConfig(@PathParam("configId") String configId) { + SecurityUtil.allowOnlyAdmin(securityContext); configManager.delete(configId); } @@ -70,6 +77,7 @@ public class ConfigResource extends AbstractResource { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.TEXT_PLAIN) public String createConfig(NutchConfig newConfig) { + SecurityUtil.allowOnlyAdmin(securityContext); if (newConfig == null) { throw new WebApplicationException(Response.status(Status.BAD_REQUEST) .entity("Nutch configuration cannot be empty!").build()); @@ -81,6 +89,7 @@ public class ConfigResource extends AbstractResource { @Path("/{config}/{property}") public Response update(@PathParam("config") String config, @PathParam("property") String property, @FormParam("value") String value) { + SecurityUtil.allowOnlyAdmin(securityContext); if (value == null) { throwBadRequestException("Missing property value!"); } http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/resources/DbResource.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/resources/DbResource.java b/src/java/org/apache/nutch/api/resources/DbResource.java index 7491327..b137dea 100644 --- a/src/java/org/apache/nutch/api/resources/DbResource.java +++ b/src/java/org/apache/nutch/api/resources/DbResource.java @@ -23,20 +23,27 @@ import java.util.WeakHashMap; import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.SecurityContext; import org.apache.nutch.api.impl.db.DbReader; import org.apache.nutch.api.model.request.DbFilter; import org.apache.nutch.api.model.response.DbQueryResult; +import org.apache.nutch.api.security.SecurityUtil; @Path("/db") public class DbResource extends AbstractResource { private Map<String, DbReader> readers = new WeakHashMap<String, DbReader>(); + @Context + SecurityContext securityContext; + @POST @Consumes(MediaType.APPLICATION_JSON) public DbQueryResult runQuery(DbFilter filter) { + SecurityUtil.allowOnlyAdmin(securityContext); if (filter == null) { throwBadRequestException("Filter cannot be null!"); } http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/resources/JobResource.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/resources/JobResource.java b/src/java/org/apache/nutch/api/resources/JobResource.java index cca5a3c..2ada981 100644 --- a/src/java/org/apache/nutch/api/resources/JobResource.java +++ b/src/java/org/apache/nutch/api/resources/JobResource.java @@ -25,15 +25,21 @@ import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.SecurityContext; import org.apache.nutch.api.model.request.JobConfig; import org.apache.nutch.api.model.response.JobInfo; import org.apache.nutch.api.model.response.JobInfo.State; +import org.apache.nutch.api.security.SecurityUtil; @Path(value = "/job") public class JobResource extends AbstractResource { + @Context + SecurityContext securityContext; + @GET @Path(value = "/") public Collection<JobInfo> getJobs(@QueryParam("crawlId") String crawlId) { @@ -66,6 +72,7 @@ public class JobResource extends AbstractResource { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.TEXT_PLAIN) public String create(JobConfig config) { + SecurityUtil.allowOnlyAdmin(securityContext); if (config == null) { throwBadRequestException("Job configuration is required!"); } http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/resources/SeedResource.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/resources/SeedResource.java b/src/java/org/apache/nutch/api/resources/SeedResource.java index b0bcb70..d7439e0 100644 --- a/src/java/org/apache/nutch/api/resources/SeedResource.java +++ b/src/java/org/apache/nutch/api/resources/SeedResource.java @@ -29,13 +29,16 @@ import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import javax.ws.rs.core.SecurityContext; import org.apache.commons.collections.CollectionUtils; import org.apache.nutch.api.model.request.SeedList; import org.apache.nutch.api.model.request.SeedUrl; +import org.apache.nutch.api.security.SecurityUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,18 +46,21 @@ import com.google.common.io.Files; @Path("/seed") public class SeedResource extends AbstractResource { - private static final Logger log = LoggerFactory - .getLogger(AdminResource.class); + private static final Logger log = LoggerFactory.getLogger(SeedResource.class); + + @Context + SecurityContext securityContext; @POST @Path("/create") @Consumes(MediaType.APPLICATION_JSON) /** - * Method creates seed list file and returns temorary directory path + * Method creates seed list file and returns temporary directory path * @param seedList * @return */ public String createSeedFile(SeedList seedList) { + SecurityUtil.allowOnlyAdmin(securityContext); if (seedList == null) { throw new WebApplicationException(Response.status(Status.BAD_REQUEST) .entity("Seed list cannot be empty!").build()); http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/security/AuthorizationRoleEnum.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/security/AuthorizationRoleEnum.java b/src/java/org/apache/nutch/api/security/AuthorizationRoleEnum.java new file mode 100644 index 0000000..4d2d616 --- /dev/null +++ b/src/java/org/apache/nutch/api/security/AuthorizationRoleEnum.java @@ -0,0 +1,38 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.nutch.api.security; + +/** + * Authorization Roles enum which holds authorization roles for NutchServer REST API. + * Supported roles are user as {@link AuthorizationRoleEnum#USER} and admin as {@link AuthorizationRoleEnum#USER} + */ +public enum AuthorizationRoleEnum { + USER("user"), + ADMIN("admin"); + + private final String value; + + AuthorizationRoleEnum(String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } + +} http://git-wip-us.apache.org/repos/asf/nutch/blob/b7f3fce4/src/java/org/apache/nutch/api/security/SecurityUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/api/security/SecurityUtil.java b/src/java/org/apache/nutch/api/security/SecurityUtil.java new file mode 100644 index 0000000..77688a4 --- /dev/null +++ b/src/java/org/apache/nutch/api/security/SecurityUtil.java @@ -0,0 +1,107 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.nutch.api.security; + +import org.apache.nutch.api.ConfManager; +import org.apache.nutch.api.resources.ConfigResource; +import org.restlet.ext.jaxrs.JaxRsApplication; +import org.restlet.security.MapVerifier; +import org.restlet.security.MemoryRealm; +import org.restlet.security.Role; +import org.restlet.security.User; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import java.util.ArrayList; +import java.util.List; + +/** + * Utility class for security operations for NutchServer REST API. + * + */ +public final class SecurityUtil { + + private static final Logger LOG = LoggerFactory.getLogger(SecurityUtil.class); + + /** + * Private constructor to prevent instantiation + */ + private SecurityUtil() { + } + + /** + * Returns roles defined at {@link org.apache.nutch.api.security.AuthorizationRoleEnum} associated with + * {@link org.restlet.ext.jaxrs.JaxRsApplication} type application + * + * @param application {@link org.restlet.ext.jaxrs.JaxRsApplication} type application + * @return roles associated with given {@link org.restlet.ext.jaxrs.JaxRsApplication} type application + */ + public static List<Role> getRoles(JaxRsApplication application) { + List<Role> roles = new ArrayList<>(); + for (AuthorizationRoleEnum authorizationRole : AuthorizationRoleEnum.values()) { + roles.add(new Role(application, authorizationRole.toString())); + } + return roles; + } + + /** + * Constructs realm + * + * @param application {@link org.restlet.ext.jaxrs.JaxRsApplication }application + * @param configManager {@link org.apache.nutch.api.ConfManager} type config manager + * @return realm + */ + public static MemoryRealm constructRealm(JaxRsApplication application, ConfManager configManager){ + MemoryRealm realm = new MemoryRealm(); + MapVerifier mapVerifier = new MapVerifier(); + String[] users = configManager.get(ConfigResource.DEFAULT).getTrimmedStrings("restapi.auth.users", "admin|admin|admin,user|user|user"); + if (users.length <= 1) { + throw new IllegalStateException("Check users definition of restapi.auth.users at nutch-site.xml "); + } + for (String userconf : users) { + String[] userDetail = userconf.split("\\|"); + if(userDetail.length != 3) { + LOG.error("Check user definition of restapi.auth.users at nutch-site.xml"); + throw new IllegalStateException("Check user definition of restapi.auth.users at nutch-site.xml "); + } + User user = new User(userDetail[0], userDetail[1]); + mapVerifier.getLocalSecrets().put(user.getIdentifier(), user.getSecret()); + realm.getUsers().add(user); + realm.map(user, Role.get(application, userDetail[2])); + LOG.info("User added: {}", userDetail[0]); + } + realm.setVerifier(mapVerifier); + return realm; + } + + /** + * Check for allowing only admin role + * + * @param securityContext to check role of logged-in user + */ + public static void allowOnlyAdmin(SecurityContext securityContext) { + if (securityContext.getAuthenticationScheme() != null + && !securityContext.isUserInRole(AuthorizationRoleEnum.ADMIN.toString())) { + throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN) + .entity("User does not have required " + AuthorizationRoleEnum.ADMIN + " role!").build()); + } + } + +}
