Author: stillalex Date: Wed Nov 22 16:00:40 2017 New Revision: 1816064 URL: http://svn.apache.org/viewvc?rev=1816064&view=rev Log: OAK-6818 TokenAuthentication/TokenProviderImpl: cleanup expired tokens
Added: jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LoginWithTokensTest.java (with props) jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authentication/token/TokenCleanupTest.java (with props) Modified: jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractLoginTest.java jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenConfigurationImpl.java jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenProviderImpl.java jackrabbit/oak/trunk/oak-doc/src/site/markdown/security/authentication/token/default.md Modified: jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractLoginTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractLoginTest.java?rev=1816064&r1=1816063&r2=1816064&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractLoginTest.java (original) +++ jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractLoginTest.java Wed Nov 22 16:00:40 2017 @@ -108,16 +108,32 @@ abstract class AbstractLoginTest extends } } + protected boolean customConfigurationParameters() { + return noIterations != -1 || expiration > 0; + } + + protected ConfigurationParameters prepare(ConfigurationParameters conf) { + return conf; + } + @Override protected Repository[] createRepository(RepositoryFixture fixture) throws Exception { - if (noIterations != -1 || expiration > 0) { + if (customConfigurationParameters()) { if (fixture instanceof OakRepositoryFixture) { return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() { @Override public Jcr customize(Oak oak) { ConfigurationParameters conf; - ConfigurationParameters iterations = ConfigurationParameters.of(UserConstants.PARAM_PASSWORD_HASH_ITERATIONS, noIterations); - ConfigurationParameters cache = ConfigurationParameters.of("cacheExpiration", expiration); + ConfigurationParameters iterations = ConfigurationParameters.EMPTY; + if (noIterations != DEFAULT_ITERATIONS) { + iterations = ConfigurationParameters.of(UserConstants.PARAM_PASSWORD_HASH_ITERATIONS, + noIterations); + } + ConfigurationParameters cache = ConfigurationParameters.EMPTY; + if (expiration > 0) { + cache = ConfigurationParameters.of("cacheExpiration", expiration); + } + if (runWithToken) { conf = ConfigurationParameters.of( TokenConfiguration.NAME, iterations, @@ -126,6 +142,7 @@ abstract class AbstractLoginTest extends conf = ConfigurationParameters.of( UserConfiguration.NAME, ConfigurationParameters.of(iterations, cache)); } + conf = prepare(conf); SecurityProvider sp = new SecurityProviderImpl(conf); return new Jcr(oak).with(sp); } Modified: jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1816064&r1=1816063&r2=1816064&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java (original) +++ jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java Wed Nov 22 16:00:40 2017 @@ -250,6 +250,7 @@ public class BenchmarkRunner { runAsUser.value(options), runWithToken.value(options), noIterations.value(options)), + new LoginWithTokensTest(numberOfUsers.value(options)), new LoginSystemTest(), new LoginImpersonateTest(), new LoginWithMembershipTest( Added: jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LoginWithTokensTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LoginWithTokensTest.java?rev=1816064&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LoginWithTokensTest.java (added) +++ jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LoginWithTokensTest.java Wed Nov 22 16:00:40 2017 @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.benchmark; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import javax.jcr.Node; +import javax.jcr.Repository; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.SimpleCredentials; + +import org.apache.jackrabbit.api.JackrabbitSession; +import org.apache.jackrabbit.api.security.user.Authorizable; +import org.apache.jackrabbit.api.security.user.UserManager; +import org.apache.jackrabbit.oak.spi.security.ConfigurationParameters; +import org.apache.jackrabbit.oak.spi.security.authentication.token.TokenConfiguration; +import org.apache.jackrabbit.oak.spi.security.authentication.token.TokenProvider; +import org.apache.jackrabbit.oak.spi.security.principal.PrincipalImpl; +import org.apache.jackrabbit.util.Text; + +/** + * Measure impact of synchronous token cleanup on the repository login with + * tokens over multiple users. Concurrency can be set via the benchmark runner. + * + * Default expiration time login tokens is 2 hours, this benchmark uses 15 + * seconds to allow for cleanup during the benchmark. + * + */ +public class LoginWithTokensTest extends AbstractLoginTest { + + private static final String REL_TEST_PATH = "testPath"; + private static final String USER = "user"; + private final Random r = new Random(); + + // defaults to 10k + private final int numberOfUsers; + + private final long tknExpy = TimeUnit.SECONDS.toMillis(15); + + private final long cleanupThreshold = 100; + + public LoginWithTokensTest(int numberOfUsers) { + super("admin", true, DEFAULT_ITERATIONS); + this.numberOfUsers = numberOfUsers; + } + + @Override + protected boolean customConfigurationParameters() { + return true; + } + + @Override + protected ConfigurationParameters prepare(ConfigurationParameters conf) { + ConfigurationParameters tkns = ConfigurationParameters.of(TokenProvider.PARAM_TOKEN_EXPIRATION, tknExpy, + "tokenCleanupThreshold", cleanupThreshold); + + ConfigurationParameters tokenConfig = ConfigurationParameters.of(TokenConfiguration.NAME, tkns); + return ConfigurationParameters.of(conf, tokenConfig); + } + + @Override + public void beforeSuite() throws Exception { + super.beforeSuite(); + + Session s = loginAdministrative(); + try { + UserManager userManager = ((JackrabbitSession) s).getUserManager(); + + for (int i = 0; i < numberOfUsers; i++) { + String id = USER + i; + userManager.createUser(id, id, new PrincipalImpl(id), REL_TEST_PATH); + } + + } finally { + s.save(); + s.logout(); + } + System.out.println("setup done, created " + numberOfUsers + " users."); + } + + @Override + public void afterSuite() throws Exception { + Session s = loginAdministrative(); + try { + Authorizable authorizable = ((JackrabbitSession) s).getUserManager().getAuthorizable(USER + "0"); + if (authorizable != null) { + Node n = s.getNode(Text.getRelativeParent(authorizable.getPath(), 1)); + n.remove(); + } + + s.save(); + } finally { + s.logout(); + } + } + + @Override + public void runTest() throws RepositoryException { + Repository repository = getRepository(); + String t = USER + r.nextInt(numberOfUsers); + SimpleCredentials creds = new SimpleCredentials(t, t.toCharArray()); + creds.setAttribute(".token", ""); + repository.login(creds).logout(); + } + +} Propchange: jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LoginWithTokensTest.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenConfigurationImpl.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenConfigurationImpl.java?rev=1816064&r1=1816063&r2=1816064&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenConfigurationImpl.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenConfigurationImpl.java Wed Nov 22 16:00:40 2017 @@ -82,6 +82,13 @@ public class TokenConfigurationImpl exte boolean tokenRefresh() default true; @AttributeDefinition( + name = "Token Cleanup Threshold", + description = "Setting this option to a value > 0 will trigger a cleanup upon token creation: " + + "if the number of existing token matches/exceeds the " + + "configured value an attempt is made to removed expired tokens.") + long tokenCleanupThreshold() default TokenProviderImpl.NO_TOKEN_CLEANUP; + + @AttributeDefinition( name = "Hash Algorithm", description = "Name of the algorithm to hash the token.") String passwordHashAlgorithm() default PasswordUtil.DEFAULT_ALGORITHM; Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenProviderImpl.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenProviderImpl.java?rev=1816064&r1=1816063&r2=1816064&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenProviderImpl.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/security/authentication/token/TokenProviderImpl.java Wed Nov 22 16:00:40 2017 @@ -98,6 +98,18 @@ class TokenProviderImpl implements Token private static final Logger log = LoggerFactory.getLogger(TokenProviderImpl.class); /** + * Optional configuration parameter to define the number of token nodes that + * when exceeded will trigger a cleanup of expired tokens upon creation. + */ + static final String PARAM_TOKEN_CLEANUP_THRESHOLD = "tokenCleanupThreshold"; + + /** + * Default value indicating that tokens should never be cleaned up (i.e. + * backwards compatible behavior). + */ + static final long NO_TOKEN_CLEANUP = 0; + + /** * Default expiration time in ms for login tokens is 2 hours. */ static final long DEFAULT_TOKEN_EXPIRATION = 2 * 3600 * 1000; @@ -112,6 +124,7 @@ class TokenProviderImpl implements Token private final long tokenExpiration; private final UserManager userManager; private final IdentifierManager identifierManager; + private final long cleanupThreshold; TokenProviderImpl(@Nonnull Root root, @Nonnull ConfigurationParameters options, @Nonnull UserConfiguration userConfiguration) { this(root, options, userConfiguration, SimpleCredentialsSupport.getInstance()); @@ -125,6 +138,7 @@ class TokenProviderImpl implements Token this.tokenExpiration = options.getConfigValue(PARAM_TOKEN_EXPIRATION, DEFAULT_TOKEN_EXPIRATION); this.userManager = userConfiguration.getUserManager(root, NamePathMapper.DEFAULT); this.identifierManager = new IdentifierManager(root); + this.cleanupThreshold = options.getConfigValue(PARAM_TOKEN_CLEANUP_THRESHOLD, NO_TOKEN_CLEANUP); } //------------------------------------------------------< TokenProvider >--- @@ -224,6 +238,7 @@ class TokenProviderImpl implements Token tokenInfo = createTokenNode(tokenParent, UUID.randomUUID().toString(), expTime, uuid, id, attributes); root.commit(CommitMarker.asCommitAttributes()); } + cleanupExpired(userId, tokenParent, creationTime, tokenInfo.getToken()); return tokenInfo; } catch (NoSuchAlgorithmException | UnsupportedEncodingException e) { // error while generating login token @@ -276,6 +291,10 @@ class TokenProviderImpl implements Token return TreeUtil.getLong(tokenTree, TOKEN_ATTRIBUTE_EXPIRY, defaultValue); } + private static boolean isExpired(long expirationTime, long loginTime) { + return expirationTime < loginTime; + } + private static void setExpirationTime(@Nonnull Tree tree, long time) { Calendar calendar = Calendar.getInstance(); calendar.setTimeInMillis(time); @@ -424,7 +443,64 @@ class TokenProviderImpl implements Token return new TokenInfoImpl(tokenNode, token, id, null); } - //-------------------------------------------------------------------------- + /** + * Remove expired token nodes if the configured threshold (i.e. number of + * token nodes) is matched/exceeded. By default (i.e. unless configured with + * a value bigger than {@link #NO_TOKEN_CLEANUP}) no cleanup is performed + * and this method returns without looking at the token nodes; this makes + * this addition optional and will not affect existing configurations. + * + * @param parent + * The token parent node. + * @param currentTime + * The time to be used for analysing expiration of existing + * tokens. + * @param token + * The token info used as random data to skip cleanup. + */ + private void cleanupExpired(@Nonnull String userId, @Nonnull Tree parent, long currentTime, @Nonnull String token) { + if (cleanupThreshold > NO_TOKEN_CLEANUP && shouldRunCleanup(token)) { + long start = System.currentTimeMillis(); + long active = 0; + long expired = 0; + try { + if (parent.getChildrenCount(cleanupThreshold) >= cleanupThreshold) { + for (Tree child : parent.getChildren()) { + if (isExpired(getExpirationTime(child, Long.MIN_VALUE), currentTime)) { + expired++; + child.remove(); + } else { + active++; + } + } + } + if (root.hasPendingChanges()) { + root.commit(CommitMarker.asCommitAttributes()); + } + } catch (CommitFailedException e) { + log.debug("Failed to cleanup expired token nodes", e); + root.refresh(); + } finally { + if (log.isDebugEnabled() && active + expired > 0) { + log.debug("Token cleanup completed in {} ms: removed {}/{} tokens for {}.", + System.currentTimeMillis() - start, expired, active + expired, userId); + } + } + } + } + + /** + * Method that determines if the cleanup should run or not based on the + * randomly generated token's first char, this decreases the chances to 1/8. + * + * @param tkn + * @return true if the cleanup should run + */ + static boolean shouldRunCleanup(@Nonnull String token) { + return token.charAt(0) < '2'; + } + + // -------------------------------------------------------------------------- /** * TokenInfo @@ -489,7 +565,7 @@ class TokenProviderImpl implements Token @Override public boolean isExpired(long loginTime) { - return expirationTime < loginTime; + return TokenProviderImpl.isExpired(expirationTime, loginTime); } @Override Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authentication/token/TokenCleanupTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authentication/token/TokenCleanupTest.java?rev=1816064&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authentication/token/TokenCleanupTest.java (added) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authentication/token/TokenCleanupTest.java Wed Nov 22 16:00:40 2017 @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.security.authentication.token; + +import javax.annotation.Nonnull; + +import com.google.common.collect.ImmutableMap; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.spi.security.ConfigurationParameters; +import org.apache.jackrabbit.oak.spi.security.authentication.token.TokenConstants; +import org.apache.jackrabbit.oak.spi.security.authentication.token.TokenInfo; +import org.apache.jackrabbit.oak.spi.security.authentication.token.TokenProvider; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TokenCleanupTest extends AbstractTokenTest { + + private String userId; + + @Override + public void before() throws Exception { + super.before(); + userId = getTestUser().getID(); + } + + @Override + ConfigurationParameters getTokenConfig() { + return ConfigurationParameters.of(TokenProviderImpl.PARAM_TOKEN_CLEANUP_THRESHOLD, 5); + } + + private void assertTokenNodes(int expectedNumber) throws Exception { + Tree tokenParent = root.getTree(getTestUser().getPath() + '/' + TokenConstants.TOKENS_NODE_NAME); + assertEquals(expectedNumber, tokenParent.getChildrenCount(expectedNumber*2)); + } + + private void createExpiredTokens(int numberOfTokens) throws Exception { + for (int i = 0; i < numberOfTokens; i++) { + TokenInfo tokenInfo = tokenProvider.createToken(userId, ImmutableMap.of(TokenProvider.PARAM_TOKEN_EXPIRATION, 2)); + // wait until the info created has expired + if (tokenInfo != null) { + waitUntilExpired(tokenInfo); + } + } + } + + private int createTokensUntilCleanup() throws Exception { + int tkn = 0; + boolean clean = false; + while (!clean && tkn < 50) { + TokenInfo tokenInfo = tokenProvider.createToken(userId, ImmutableMap.of()); + clean = TokenProviderImpl.shouldRunCleanup(tokenInfo.getToken()); + tkn++; + } + return tkn; + } + + private void waitUntilExpired(@Nonnull TokenInfo info) { + long now = System.currentTimeMillis(); + while (!info.isExpired(now)) { + now = waitForSystemTimeIncrement(now); + } + } + + @Test + public void testExpiredBelowThreshold() throws Exception { + createExpiredTokens(4); + assertTokenNodes(4); + } + + @Test + public void testAllExpiredReachingThreshold() throws Exception { + createExpiredTokens(5); + int extras = createTokensUntilCleanup(); + assertTokenNodes(extras); + } + + @Test + public void testSomeExpiredReachingThreshold() throws Exception { + createExpiredTokens(3); + tokenProvider.createToken(userId, ImmutableMap.of()); + + assertTokenNodes(4); + + int extras = createTokensUntilCleanup(); + assertTokenNodes(1 + extras); + } + + @Test + public void testNotExpiredReachingThreshold() throws Exception { + for (int i = 0; i < 10; i++) { + tokenProvider.createToken(userId, ImmutableMap.of()); + } + assertTokenNodes(10); + } +} \ No newline at end of file Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authentication/token/TokenCleanupTest.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-doc/src/site/markdown/security/authentication/token/default.md URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-doc/src/site/markdown/security/authentication/token/default.md?rev=1816064&r1=1816063&r2=1816064&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-doc/src/site/markdown/security/authentication/token/default.md (original) +++ jackrabbit/oak/trunk/oak-doc/src/site/markdown/security/authentication/token/default.md Wed Nov 22 16:00:40 2017 @@ -97,6 +97,15 @@ that the expiration time has not been re and the user will need to login again using the configured login mechanism (e.g. using the credentials support for token creation). +#### Token Cleanup + +Automatic token cleanup can be enabled by setting the `tokenCleanupThreshold` parameter +to a value larger than `0` (`0` means disabled). This will trigger a cleanup call if +the number of tokens under a user exceeds this value. (As an implementation detail a +throttling method was introduced to only allow the call to go through 1/8 times). + +This is available with Oak 1.7.12 on, see also [OAK-6818]for additional information. + <a name="representation"/> ### Representation in the Repository @@ -195,7 +204,7 @@ all of type `Constraint` with the follow <a name="configuration"/> ### Configuration -The default Oak [TokenConfiguration] allows to define the following configuration +The default Oak `TokenConfiguration` allows to define the following configuration options for the `TokenProvider`: #### Configuration Parameters @@ -208,6 +217,7 @@ options for the `TokenProvider`: | PARAM_PASSWORD_HASH_ALGORITHM | String | SHA-256 | | PARAM_PASSWORD_HASH_ITERATIONS | int | 1000 | | PARAM_PASSWORD_SALT_SIZE | int | 8 | +| PARAM_TOKEN_CLEANUP_THRESHOLD | long | 0 (no cleanup) | | | | |