This is an automated email from the ASF dual-hosted git repository.
smolnar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new afdb4cc3f KNOX-2990 - Using DerbyDatabaseTSS instead of AliasBasedTSS
by default (#826)
afdb4cc3f is described below
commit afdb4cc3f20d4c295b58eb3709343ed4fe47d6b6
Author: Sandor Molnar <[email protected]>
AuthorDate: Fri Feb 2 08:29:32 2024 +0100
KNOX-2990 - Using DerbyDatabaseTSS instead of AliasBasedTSS by default
(#826)
In addition to the new implementation I deprecated the AliasBased,
Zookeeper and JournalBased TSS implementations in 2.1.0.
---
gateway-server/pom.xml | 7 +-
.../gateway/config/impl/GatewayConfigImpl.java | 57 +++--
.../knox/gateway/services/CLIGatewayServices.java | 2 +
.../services/factory/TokenStateServiceFactory.java | 28 ++-
.../token/impl/AliasBasedTokenStateService.java | 9 +-
.../token/impl/DefaultTokenStateService.java | 1 +
.../token/impl/DerbyDBTokenStateService.java | 99 +++++++++
.../services/token/impl/JDBCTokenStateService.java | 35 +++-
.../token/impl/JournalBasedTokenStateService.java | 4 +
.../token/impl/TokenStateServiceMessages.java | 10 +
.../token/impl/ZookeeperTokenStateService.java | 3 +
.../org/apache/knox/gateway/util/JDBCUtils.java | 21 +-
.../java/org/apache/knox/gateway/util/KnoxCLI.java | 58 +++++-
.../knox/gateway/util/TokenMigrationTool.java | 232 +++++++++++++++++++++
.../services/factory/ServiceFactoryTest.java | 66 +++++-
.../factory/TokenStateServiceFactoryTest.java | 49 +++--
.../apache/knox/gateway/util/JDBCUtilsTest.java | 10 +-
.../org/apache/knox/gateway/util/KnoxCLITest.java | 7 +-
.../gateway/service/knoxtoken/TokenResource.java | 4 +-
.../org/apache/knox/gateway/GatewayTestConfig.java | 31 ++-
.../apache/knox/gateway/config/GatewayConfig.java | 31 +++
.../services/security/impl/CMFMasterService.java | 30 +--
.../security/token/TokenMigrationTarget.java | 26 +++
.../org/apache/knox/gateway/util/FileUtils.java | 58 ++++++
gateway-test-release/pom.xml | 1 +
gateway-test-release/webhdfs-kerb-test/pom.xml | 1 +
gateway-test-release/webhdfs-test/pom.xml | 1 +
gateway-test/pom.xml | 7 +
.../apache/knox/gateway/GatewayBasicFuncTest.java | 23 +-
.../app/token-generation.component.ts | 6 +-
pom.xml | 4 +
31 files changed, 814 insertions(+), 107 deletions(-)
diff --git a/gateway-server/pom.xml b/gateway-server/pom.xml
index 67e79dc93..f1444ba3e 100644
--- a/gateway-server/pom.xml
+++ b/gateway-server/pom.xml
@@ -215,6 +215,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
+ </dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
@@ -467,19 +471,16 @@
<dependency>
<groupId>org.apache.knox</groupId>
<artifactId>gateway-shell</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derbynet</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
index f924d9773..2501010b4 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
@@ -280,8 +280,14 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
/* property that specifies list of services for which we need to append
service name to the X-Forward-Context header */
public static final String X_FORWARD_CONTEXT_HEADER_APPEND_SERVICES =
GATEWAY_CONFIG_FILE_PREFIX + ".xforwarded.header.context.append.servicename";
- private static final String TOKEN_STATE_SERVER_MANAGED =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.exp.server-managed";
- private static final String USERS_CAN_SEE_ALL_TOKENS =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.management.users.can.see.all.tokens";
+ private static final String KNOX_TOKEN_PREFIX = GATEWAY_CONFIG_FILE_PREFIX +
".knox.token";
+ private static final String TOKEN_STATE_SERVER_MANAGED = KNOX_TOKEN_PREFIX +
".exp.server-managed";
+ private static final String USERS_CAN_SEE_ALL_TOKENS = KNOX_TOKEN_PREFIX +
".management.users.can.see.all.tokens";
+ private static final String SKIP_TOKEN_MIGRATION= KNOX_TOKEN_PREFIX +
".migration.skip";
+ private static final String ARCHIVE_MIGRATED_TOKENS= KNOX_TOKEN_PREFIX +
".migration.archive.tokens";
+ private static final String MIGRATE_EXPIRED_TOKENS= KNOX_TOKEN_PREFIX +
".migration.include.expired.tokens";
+ private static final String TOKEN_MIGRATION_PRINTS_VERBOSE_MESSAGES=
KNOX_TOKEN_PREFIX + ".migration.verbose";
+ private static final String TOKEN_MIGRATION_PROGRESS_COUNT=
KNOX_TOKEN_PREFIX + ".migration.progress.count";
private static final String CLOUDERA_MANAGER_DESCRIPTORS_MONITOR_INTERVAL =
GATEWAY_CONFIG_FILE_PREFIX + ".cloudera.manager.descriptors.monitor.interval";
private static final String
CLOUDERA_MANAGER_ADVANCED_SERVICE_DISCOVERY_CONF_MONITOR_INTERVAL =
GATEWAY_CONFIG_FILE_PREFIX +
".cloudera.manager.advanced.service.discovery.config.monitor.interval";
@@ -295,12 +301,12 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
private static final long
CLOUDERA_MANAGER_SERVICE_DISCOVERY_READ_TIMEOUT_DEFAULT = 10000;
private static final long
CLOUDERA_MANAGER_SERVICE_DISCOVERY_WRITE_TIMEOUT_DEFAULT = 10000;
- private static final String KNOX_TOKEN_EVICTION_INTERVAL =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.eviction.interval";
- private static final String KNOX_TOKEN_EVICTION_GRACE_PERIOD =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.eviction.grace.period";
- private static final String KNOX_TOKEN_ALIAS_PERSISTENCE_INTERVAL =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.state.alias.persistence.interval";
- private static final String KNOX_TOKEN_PERMISSIVE_VALIDATION_ENABLED =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.permissive.validation";
- private static final String KNOX_TOKEN_HASH_ALGORITHM =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.hash.algorithm";
- public static final String KNOX_TOKEN_USER_LIMIT =
GATEWAY_CONFIG_FILE_PREFIX + ".knox.token.limit.per.user";
+ private static final String KNOX_TOKEN_EVICTION_INTERVAL = KNOX_TOKEN_PREFIX
+ ".eviction.interval";
+ private static final String KNOX_TOKEN_EVICTION_GRACE_PERIOD =
KNOX_TOKEN_PREFIX + ".eviction.grace.period";
+ private static final String KNOX_TOKEN_ALIAS_PERSISTENCE_INTERVAL =
KNOX_TOKEN_PREFIX + ".state.alias.persistence.interval";
+ private static final String KNOX_TOKEN_PERMISSIVE_VALIDATION_ENABLED =
KNOX_TOKEN_PREFIX + ".permissive.validation";
+ private static final String KNOX_TOKEN_HASH_ALGORITHM = KNOX_TOKEN_PREFIX +
".hash.algorithm";
+ public static final String KNOX_TOKEN_USER_LIMIT = KNOX_TOKEN_PREFIX +
".limit.per.user";
private static final long KNOX_TOKEN_EVICTION_INTERVAL_DEFAULT =
TimeUnit.MINUTES.toSeconds(5);
private static final long KNOX_TOKEN_EVICTION_GRACE_PERIOD_DEFAULT =
TimeUnit.HOURS.toSeconds(24);
private static final long KNOX_TOKEN_ALIAS_PERSISTENCE_INTERVAL_DEFAULT =
TimeUnit.SECONDS.toSeconds(15);
@@ -318,11 +324,11 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
private static final String KNOX_INCOMING_XFORWARDED_ENABLED =
"gateway.incoming.xforwarded.enabled";
//Gateway Database related properties
- private static final String GATEWAY_DATABASE_TYPE =
GATEWAY_CONFIG_FILE_PREFIX + ".database.type";
- private static final String GATEWAY_DATABASE_CONN_URL =
GATEWAY_CONFIG_FILE_PREFIX + ".database.connection.url";
- private static final String GATEWAY_DATABASE_HOST =
GATEWAY_CONFIG_FILE_PREFIX + ".database.host";
- private static final String GATEWAY_DATABASE_PORT =
GATEWAY_CONFIG_FILE_PREFIX + ".database.port";
- private static final String GATEWAY_DATABASE_NAME =
GATEWAY_CONFIG_FILE_PREFIX + ".database.name";
+ public static final String GATEWAY_DATABASE_TYPE =
GATEWAY_CONFIG_FILE_PREFIX + ".database.type";
+ public static final String GATEWAY_DATABASE_CONN_URL =
GATEWAY_CONFIG_FILE_PREFIX + ".database.connection.url";
+ public static final String GATEWAY_DATABASE_HOST =
GATEWAY_CONFIG_FILE_PREFIX + ".database.host";
+ public static final String GATEWAY_DATABASE_PORT =
GATEWAY_CONFIG_FILE_PREFIX + ".database.port";
+ public static final String GATEWAY_DATABASE_NAME =
GATEWAY_CONFIG_FILE_PREFIX + ".database.name";
private static final String GATEWAY_DATABASE_SSL_ENABLED =
GATEWAY_CONFIG_FILE_PREFIX + ".database.ssl.enabled";
private static final String GATEWAY_DATABASE_VERIFY_SERVER_CERT =
GATEWAY_CONFIG_FILE_PREFIX + ".database.ssl.verify.server.cert";
private static final String GATEWAY_DATABASE_TRUSTSTORE_FILE =
GATEWAY_CONFIG_FILE_PREFIX + ".database.ssl.truststore.file";
@@ -1534,4 +1540,29 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
return pathAliases;
}
+ @Override
+ public boolean skipTokenMigration() {
+ return getBoolean(SKIP_TOKEN_MIGRATION, false);
+ }
+
+ @Override
+ public boolean archiveMigratedTokens() {
+ return getBoolean(ARCHIVE_MIGRATED_TOKENS, false);
+ }
+
+ @Override
+ public boolean migrateExpiredTokens() {
+ return getBoolean(MIGRATE_EXPIRED_TOKENS, false);
+ }
+
+ @Override
+ public boolean printVerboseTokenMigrationMessages() {
+ return getBoolean(TOKEN_MIGRATION_PRINTS_VERBOSE_MESSAGES, true);
+ }
+
+ @Override
+ public int getTokenMigrationProgressCount() {
+ return getInt(TOKEN_MIGRATION_PROGRESS_COUNT, 10);
+ }
+
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/CLIGatewayServices.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/CLIGatewayServices.java
index 4fdbf5548..872578716 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/CLIGatewayServices.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/CLIGatewayServices.java
@@ -53,6 +53,8 @@ public class CLIGatewayServices extends
AbstractGatewayServices {
addService(ServiceType.CRYPTO_SERVICE, gatewayServiceFactory.create(this,
ServiceType.CRYPTO_SERVICE, config, options));
addService(ServiceType.TOPOLOGY_SERVICE,
gatewayServiceFactory.create(this, ServiceType.TOPOLOGY_SERVICE, config,
options));
+
+ addService(ServiceType.TOKEN_STATE_SERVICE,
gatewayServiceFactory.create(this, ServiceType.TOKEN_STATE_SERVICE, config,
options));
}
@Override
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactory.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactory.java
index 30a1f0602..4934c1f5f 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactory.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactory.java
@@ -32,6 +32,7 @@ import
org.apache.knox.gateway.services.ServiceLifecycleException;
import org.apache.knox.gateway.services.ServiceType;
import org.apache.knox.gateway.services.token.impl.AliasBasedTokenStateService;
import org.apache.knox.gateway.services.token.impl.DefaultTokenStateService;
+import org.apache.knox.gateway.services.token.impl.DerbyDBTokenStateService;
import org.apache.knox.gateway.services.token.impl.JDBCTokenStateService;
import
org.apache.knox.gateway.services.token.impl.JournalBasedTokenStateService;
import org.apache.knox.gateway.services.token.impl.ZookeeperTokenStateService;
@@ -45,9 +46,11 @@ public class TokenStateServiceFactory extends
AbstractServiceFactory {
throws ServiceLifecycleException {
Service service = null;
if (shouldCreateService(implementation)) {
- if (matchesImplementation(implementation,
DefaultTokenStateService.class)) {
+ if (matchesImplementation(implementation,
DerbyDBTokenStateService.class, true)) {
+ service = useDerbyDatabaseTokenStateService(gatewayServices,
gatewayConfig, options);
+ } else if (matchesImplementation(implementation,
DefaultTokenStateService.class)) {
service = new DefaultTokenStateService();
- } else if (matchesImplementation(implementation,
AliasBasedTokenStateService.class, true)) {
+ } else if (matchesImplementation(implementation,
AliasBasedTokenStateService.class)) {
service = new AliasBasedTokenStateService();
((AliasBasedTokenStateService)
service).setAliasService(getAliasService(gatewayServices));
} else if (matchesImplementation(implementation,
JournalBasedTokenStateService.class)) {
@@ -61,17 +64,30 @@ public class TokenStateServiceFactory extends
AbstractServiceFactory {
service.init(gatewayConfig, options);
} catch (ServiceLifecycleException e) {
LOG.errorInitializingService(implementation, e.getMessage(), e);
- service = new AliasBasedTokenStateService();
- ((AliasBasedTokenStateService)
service).setAliasService(getAliasService(gatewayServices));
+ service = useDerbyDatabaseTokenStateService(gatewayServices,
gatewayConfig, options);
}
}
- logServiceUsage(isEmptyDefaultImplementation(implementation) ?
AliasBasedTokenStateService.class.getName() : implementation, serviceType);
+ logServiceUsage(service.getClass().getName(), serviceType);
}
return service;
}
+ private Service useDerbyDatabaseTokenStateService(GatewayServices
gatewayServices, GatewayConfig gatewayConfig, Map<String, String> options) {
+ Service service;
+ try {
+ service = new DerbyDBTokenStateService();
+ ((DerbyDBTokenStateService)
service).setAliasService(getAliasService(gatewayServices));
+ ((DerbyDBTokenStateService)
service).setMasterService(getMasterService(gatewayServices));
+ service.init(gatewayConfig, options);
+ } catch (ServiceLifecycleException e) {
+ LOG.errorInitializingService(DerbyDBTokenStateService.class.getName(),
e.getMessage(), e);
+ service = new DefaultTokenStateService();
+ }
+ return service;
+ }
+
@Override
protected ServiceType getServiceType() {
return ServiceType.TOKEN_STATE_SERVICE;
@@ -80,6 +96,6 @@ public class TokenStateServiceFactory extends
AbstractServiceFactory {
@Override
protected Collection<String> getKnownImplementations() {
return unmodifiableList(asList(DefaultTokenStateService.class.getName(),
AliasBasedTokenStateService.class.getName(),
JournalBasedTokenStateService.class.getName(),
- ZookeeperTokenStateService.class.getName(),
JDBCTokenStateService.class.getName()));
+ ZookeeperTokenStateService.class.getName(),
JDBCTokenStateService.class.getName(),
DerbyDBTokenStateService.class.getName()));
}
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/AliasBasedTokenStateService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/AliasBasedTokenStateService.java
index a05dff08a..18902eed8 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/AliasBasedTokenStateService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/AliasBasedTokenStateService.java
@@ -52,13 +52,15 @@ import org.apache.knox.gateway.util.Tokens;
/**
* A TokenStateService implementation based on the AliasService.
+ *
+ * @deprecated Since 2.1.0
*/
public class AliasBasedTokenStateService extends
AbstractPersistentTokenStateService implements
TokenStatePeristerMonitorListener {
static final String TOKEN_ALIAS_SUFFIX_DELIM = "--";
- static final String TOKEN_ISSUE_TIME_POSTFIX = TOKEN_ALIAS_SUFFIX_DELIM +
"iss";
- static final String TOKEN_MAX_LIFETIME_POSTFIX = TOKEN_ALIAS_SUFFIX_DELIM +
"max";
- static final String TOKEN_META_POSTFIX = TOKEN_ALIAS_SUFFIX_DELIM +
"meta";
+ public static final String TOKEN_ISSUE_TIME_POSTFIX =
TOKEN_ALIAS_SUFFIX_DELIM + "iss";
+ public static final String TOKEN_MAX_LIFETIME_POSTFIX =
TOKEN_ALIAS_SUFFIX_DELIM + "max";
+ public static final String TOKEN_META_POSTFIX =
TOKEN_ALIAS_SUFFIX_DELIM + "meta";
protected AliasService aliasService;
@@ -80,6 +82,7 @@ public class AliasBasedTokenStateService extends
AbstractPersistentTokenStateSer
@Override
public void init(final GatewayConfig config, final Map<String, String>
options) throws ServiceLifecycleException {
+ log.deprecatedServiceUsage(this.getClass().getCanonicalName());
super.init(config, options);
if (aliasService == null) {
throw new ServiceLifecycleException("The required AliasService reference
has not been set.");
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DefaultTokenStateService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DefaultTokenStateService.java
index cbb783b38..1676e14fa 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DefaultTokenStateService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DefaultTokenStateService.java
@@ -457,4 +457,5 @@ public class DefaultTokenStateService implements
TokenStateService {
});
return tokens;
}
+
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DerbyDBTokenStateService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DerbyDBTokenStateService.java
new file mode 100644
index 000000000..48525fa86
--- /dev/null
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/DerbyDBTokenStateService.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.services.token.impl;
+
+import static
org.apache.knox.gateway.config.impl.GatewayConfigImpl.GATEWAY_DATABASE_NAME;
+import static
org.apache.knox.gateway.config.impl.GatewayConfigImpl.GATEWAY_DATABASE_TYPE;
+import static
org.apache.knox.gateway.services.security.AliasService.NO_CLUSTER_NAME;
+import static
org.apache.knox.gateway.util.JDBCUtils.DATABASE_PASSWORD_ALIAS_NAME;
+import static org.apache.knox.gateway.util.JDBCUtils.DATABASE_USER_ALIAS_NAME;
+import static org.apache.knox.gateway.util.JDBCUtils.DERBY_DB_TYPE;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.knox.gateway.config.GatewayConfig;
+import org.apache.knox.gateway.services.ServiceLifecycleException;
+import org.apache.knox.gateway.services.security.MasterService;
+import org.apache.knox.gateway.shell.jdbc.derby.DerbyDatabase;
+import org.apache.knox.gateway.util.FileUtils;
+
+public class DerbyDBTokenStateService extends JDBCTokenStateService {
+
+ public static final String DEFAULT_TOKEN_DB_USER_NAME = "knox";
+ public static final String DB_NAME = "tokens";
+
+ private DerbyDatabase derbyDatabase;
+ private Path derbyDatabaseFolder;
+ private MasterService masterService;
+
+ public void setMasterService(MasterService masterService) {
+ this.masterService = masterService;
+ }
+
+ @Override
+ public void init(GatewayConfig config, Map<String, String> options) throws
ServiceLifecycleException {
+ try {
+ derbyDatabaseFolder = Paths.get(config.getGatewaySecurityDir(), DB_NAME);
+ startDerby();
+ ((Configuration) config).set(GATEWAY_DATABASE_TYPE, DERBY_DB_TYPE);
+ ((Configuration) config).set(GATEWAY_DATABASE_NAME,
derbyDatabaseFolder.toString());
+ getAliasService().addAliasForCluster(NO_CLUSTER_NAME,
DATABASE_USER_ALIAS_NAME, getDatabaseUserName());
+ getAliasService().addAliasForCluster(NO_CLUSTER_NAME,
DATABASE_PASSWORD_ALIAS_NAME, getDatabasePassword());
+ super.init(config, options);
+
+ // we need the "x" permission too to be able to browse that folder (600
is not enough)
+ if (Files.exists(derbyDatabaseFolder)) {
+ FileUtils.chmod("700", derbyDatabaseFolder.toFile());
+ }
+ } catch (Exception e) {
+ throw new ServiceLifecycleException("Error while initiating
DerbyDBTokenStateService: " + e, e);
+ }
+ }
+
+ private void startDerby() throws Exception {
+ derbyDatabase = new DerbyDatabase(derbyDatabaseFolder.toString());
+ derbyDatabase.create();
+ TimeUnit.SECONDS.sleep(1); // give a bit of time for the server to start
+ }
+
+ private String getDatabasePassword() throws Exception {
+ final char[] dbPasswordAliasValue =
getAliasService().getPasswordFromAliasForGateway(DATABASE_PASSWORD_ALIAS_NAME);
+ return dbPasswordAliasValue != null ? new String(dbPasswordAliasValue) :
new String(masterService.getMasterSecret());
+ }
+
+ private String getDatabaseUserName() throws Exception {
+ final char[] dbUserAliasValue =
getAliasService().getPasswordFromAliasForGateway(DATABASE_USER_ALIAS_NAME);
+ return dbUserAliasValue != null ? new String(dbUserAliasValue) :
DEFAULT_TOKEN_DB_USER_NAME;
+ }
+
+ @Override
+ public void stop() throws ServiceLifecycleException {
+ try {
+ if (derbyDatabase != null) {
+ derbyDatabase.shutdown();
+ }
+ } catch (Exception e) {
+ throw new ServiceLifecycleException("Error while shutting down Derby
Database", e);
+ }
+ }
+
+}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateService.java
index 7110325af..3887b117a 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateService.java
@@ -34,22 +34,34 @@ import
org.apache.knox.gateway.services.ServiceLifecycleException;
import org.apache.knox.gateway.services.security.AliasService;
import org.apache.knox.gateway.services.security.token.KnoxToken;
import org.apache.knox.gateway.services.security.token.TokenMetadata;
+import org.apache.knox.gateway.services.security.token.TokenMigrationTarget;
import
org.apache.knox.gateway.services.security.token.TokenStateServiceException;
import org.apache.knox.gateway.services.security.token.UnknownTokenException;
import org.apache.knox.gateway.util.JDBCUtils;
+import org.apache.knox.gateway.util.TokenMigrationTool;
import org.apache.knox.gateway.util.Tokens;
-public class JDBCTokenStateService extends AbstractPersistentTokenStateService
{
+public class JDBCTokenStateService extends AbstractPersistentTokenStateService
implements TokenMigrationTarget {
private AliasService aliasService; // connection username/pw and passcode
HMAC secret are stored here
private TokenStateDatabase tokenDatabase;
private AtomicBoolean initialized = new AtomicBoolean(false);
private Lock initLock = new ReentrantLock(true);
private Lock addMetadataLock = new ReentrantLock(true);
+ private boolean skipTokenMigration;
+ private boolean archiveMigratedTokens;
+ private boolean migrateExpiredTokens;
+ private boolean verboseTokenMigration;
+ private int tokenMigrationProgressCount;
+
public void setAliasService(AliasService aliasService) {
this.aliasService = aliasService;
}
+ protected AliasService getAliasService() {
+ return aliasService;
+ }
+
@Override
public void init(GatewayConfig config, Map<String, String> options) throws
ServiceLifecycleException {
if (!initialized.get()) {
@@ -65,12 +77,33 @@ public class JDBCTokenStateService extends
AbstractPersistentTokenStateService {
} catch (Exception e) {
throw new ServiceLifecycleException("Error while initiating
JDBCTokenStateService: " + e, e);
}
+
+ this.skipTokenMigration = config.skipTokenMigration();
+ this.archiveMigratedTokens = config.archiveMigratedTokens();
+ this.migrateExpiredTokens = config.migrateExpiredTokens();
+ this.verboseTokenMigration =
config.printVerboseTokenMigrationMessages();
+ this.tokenMigrationProgressCount =
config.getTokenMigrationProgressCount();
} finally {
initLock.unlock();
}
}
}
+ @Override
+ public void start() throws ServiceLifecycleException {
+ super.start();
+ if (skipTokenMigration) {
+ log.skipTokenMigration();
+ } else {
+ final TokenMigrationTool tokenMigrationTool = new
TokenMigrationTool(aliasService, this, null);
+ tokenMigrationTool.setArchiveMigratedTokens(archiveMigratedTokens);
+ tokenMigrationTool.setProgressCount(tokenMigrationProgressCount);
+ tokenMigrationTool.setVerbose(verboseTokenMigration);
+ tokenMigrationTool.setMigrateExpiredTokens(migrateExpiredTokens);
+ tokenMigrationTool.migrateTokensFromGatewayCredentialStore();
+ }
+ }
+
@Override
public void addToken(String tokenId, long issueTime, long expiration, long
maxLifetimeDuration) {
try {
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JournalBasedTokenStateService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JournalBasedTokenStateService.java
index 6487bd1dd..e21852b14 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JournalBasedTokenStateService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/JournalBasedTokenStateService.java
@@ -32,12 +32,16 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+/**
+ * @deprecated Since 2.1.0
+ */
public class JournalBasedTokenStateService extends
AbstractPersistentTokenStateService {
private TokenStateJournal journal;
@Override
public void init(final GatewayConfig config, final Map<String, String>
options) throws ServiceLifecycleException {
+ log.deprecatedServiceUsage(this.getClass().getCanonicalName());
super.init(config, options);
try {
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateServiceMessages.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateServiceMessages.java
index 166e89ae7..8ca485cbe 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateServiceMessages.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateServiceMessages.java
@@ -261,4 +261,14 @@ public interface TokenStateServiceMessages {
@Message(level = MessageLevel.ERROR, text = "An error occurred while
fetching impersonation tokens for user {0} from the database : {1}")
void errorFetchingDoAsTokensForUserFromDatabase(String userName, String
errorMessage, @StackTrace(level = MessageLevel.DEBUG) Exception e);
+
+ @Message(level = MessageLevel.WARN, text = "The configured TokenStateService
implementation, {0}, is deprecated!")
+ void deprecatedServiceUsage(String className);
+
+ @Message(level = MessageLevel.INFO, text = "Skipping token migration!")
+ void skipTokenMigration();
+
+ @Message(level = MessageLevel.INFO, text = "{0}")
+ void info(String message);
+
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/ZookeeperTokenStateService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/ZookeeperTokenStateService.java
index 1fce481f4..70aad418f 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/ZookeeperTokenStateService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/ZookeeperTokenStateService.java
@@ -38,6 +38,8 @@ import org.apache.knox.gateway.util.Tokens;
* A Zookeeper Token State Service is actually an Alias based TSS where the
'alias service' happens to be the 'zookeeper' implementation.
* This means the only important thing that should be overridden here is the
init method where the underlying alias service is configured
* properly.
+ *
+ * @deprecated Since 2.1.0
*/
public class ZookeeperTokenStateService extends AliasBasedTokenStateService
implements RemoteTokenStateChangeListener {
@@ -55,6 +57,7 @@ public class ZookeeperTokenStateService extends
AliasBasedTokenStateService impl
@Override
public void init(GatewayConfig config, Map<String, String> options) throws
ServiceLifecycleException {
+ log.deprecatedServiceUsage(this.getClass().getCanonicalName());
final ZookeeperRemoteAliasService zookeeperAliasService =
(ZookeeperRemoteAliasService) aliasServiceFactory.create(gatewayServices,
ALIAS_SERVICE, config, options,
ZookeeperRemoteAliasService.class.getName());
options.put(ZookeeperRemoteAliasService.OPTION_NAME_SHOULD_CREATE_TOKENS_SUB_NODE,
"true");
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
b/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
index 8253f4c5b..efb2c3ea2 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
@@ -31,7 +31,7 @@ import javax.sql.DataSource;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.derby.jdbc.ClientDataSource;
+import org.apache.derby.jdbc.EmbeddedDataSource;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.services.security.AliasService;
import org.apache.knox.gateway.services.security.AliasServiceException;
@@ -111,13 +111,11 @@ public class JDBCUtils {
}
private static DataSource createDerbyDatasource(GatewayConfig gatewayConfig,
AliasService aliasService) throws AliasServiceException {
- final ClientDataSource derbyDatasource = new ClientDataSource();
- derbyDatasource.setDatabaseName(gatewayConfig.getDatabaseName());
- derbyDatasource.setServerName(gatewayConfig.getDatabaseHost());
- derbyDatasource.setPortNumber(gatewayConfig.getDatabasePort());
- derbyDatasource.setUser(getDatabaseUser(aliasService));
- derbyDatasource.setPassword(getDatabasePassword(aliasService));
- return derbyDatasource;
+ final EmbeddedDataSource embeddedDataSource = new EmbeddedDataSource();
+ embeddedDataSource.setDatabaseName(gatewayConfig.getDatabaseName());
+ embeddedDataSource.setUser(getDatabaseUser(aliasService));
+ embeddedDataSource.setPassword(getDatabasePassword(aliasService));
+ return embeddedDataSource;
}
@@ -195,8 +193,15 @@ public class JDBCUtils {
public static void createTable(String createSqlFileName, DataSource
dataSource, ClassLoader classLoader) throws Exception {
final InputStream is = classLoader.getResourceAsStream(createSqlFileName);
String createTableSql = IOUtils.toString(is, UTF_8);
+ if (isDerbyDatasource(dataSource)) {
+ createTableSql = createTableSql.replaceAll("IF NOT EXISTS ", "");
+ }
try (Connection connection = dataSource.getConnection(); Statement
createTableStatment = connection.createStatement();) {
createTableStatment.execute(createTableSql);
}
}
+
+ private static boolean isDerbyDatasource(DataSource dataSource) {
+ return dataSource.getClass().getName().contains("derby");
+ }
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
b/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
index d2412fb8c..71fde73fa 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java
@@ -78,6 +78,8 @@ import
org.apache.knox.gateway.services.security.AliasServiceException;
import org.apache.knox.gateway.services.security.KeystoreService;
import org.apache.knox.gateway.services.security.KeystoreServiceException;
import org.apache.knox.gateway.services.security.MasterService;
+import org.apache.knox.gateway.services.security.token.TokenMigrationTarget;
+import org.apache.knox.gateway.services.security.token.TokenStateService;
import org.apache.knox.gateway.services.topology.TopologyService;
import org.apache.knox.gateway.topology.Provider;
import org.apache.knox.gateway.topology.Topology;
@@ -131,7 +133,8 @@ public class KnoxCLI extends Configured implements Tool {
" [" + RemoteRegistryDeleteDescriptorCommand.USAGE + "]\n" +
" [" + RemoteRegistryGetACLCommand.USAGE + "]\n" +
" [" + TopologyConverter.USAGE + "]\n" +
- " [" + JWKGenerator.USAGE + "]\n";
+ " [" + JWKGenerator.USAGE + "]\n" +
+ " [" + TokenMigration.USAGE + "]\n";
/** allows stdout to be captured if necessary */
public PrintStream out = System.out;
@@ -152,6 +155,10 @@ public class KnoxCLI extends Configured implements Tool {
private String pass;
private boolean groups;
private JWSAlgorithm jwsAlgorithm = JWSAlgorithm.HS256;
+ private int progressCount = 10;
+ private boolean archiveMigratedTokens;
+ private boolean migrateExpiredTokens;
+ private boolean verbose;
private String alias;
private String remoteRegistryClient;
@@ -529,6 +536,16 @@ public class KnoxCLI extends Configured implements Tool {
}
} else if (args[i].equalsIgnoreCase("--saveAlias")) {
alias = args[++i];
+ } else if (args[i].equalsIgnoreCase("migrate-tokens") ) {
+ command = new TokenMigration();
+ } else if (args[i].equalsIgnoreCase("--progressCount") ) {
+ progressCount = Integer.parseInt(args[++i]);
+ } else if (args[i].equalsIgnoreCase("--archiveMigrated") ) {
+ archiveMigratedTokens = Boolean.parseBoolean(args[++i]);
+ } else if (args[i].equalsIgnoreCase("--migrateExpiredTokens") ) {
+ migrateExpiredTokens = Boolean.parseBoolean(args[++i]);
+ } else if (args[i].equalsIgnoreCase("--verbose") ) {
+ verbose = Boolean.parseBoolean(args[++i]);
} else {
printKnoxShellUsage();
return -1;
@@ -2429,6 +2446,45 @@ public class KnoxCLI extends Configured implements Tool {
}
}
+ public class TokenMigration extends Command {
+
+ static final String USAGE = "migrate-tokens [--progressCount num]
[--verbose true|false] [--archivedMigrated true|false] [--migrateExpiredTokens
true|false]";
+ static final String DESC =
+ "Migrates previously created Knox Tokens from the Gateway credential
store into the configured JDBC TokenStateService backend.\n"
+ + "Options are as follows: \n"
+ + "--progressCount (optional) indicates the number of tokens after
this tool displays progress on the standard output. Defaults to 10.\n"
+ + "--archiveMigrated (optional) a boolean flag indicating if
migrated tokens should not be removed completely. "
+ + "Instead, tokens are going to be archived in a separate keystore
called __tokens-credentials.jceks. Defaults to false\n"
+ + "--verbose (optional) a boolean flag that controls of a more
verbose output on the STDOUT when processing tokens. Defaults to false.\n"
+ + "--migrateExpiredTokens (optional) a boolean flag indicating
whether already expired tokens should be migrated into the configure TSS
backend. Defaults to false";
+
+ @Override
+ public void execute() throws Exception {
+ final TokenStateService tokenStateService =
services.getService(ServiceType.TOKEN_STATE_SERVICE);
+ if (isTokenMigrationTarget(tokenStateService)) {
+ out.println("Migrating tokens from __gateway credential store into the
configured TokenStateService backend...");
+ final TokenMigrationTool tokenMigrationTool = new
TokenMigrationTool(getAliasService(), tokenStateService, out);
+ tokenMigrationTool.setArchiveMigratedTokens(archiveMigratedTokens);
+ tokenMigrationTool.setMigrateExpiredTokens(migrateExpiredTokens);
+ tokenMigrationTool.setProgressCount(progressCount);
+ tokenMigrationTool.setVerbose(verbose);
+ tokenMigrationTool.migrateTokensFromGatewayCredentialStore();
+ } else {
+ out.println("This tool is meant to migrate tokens into a JDBC
TokenStateService backend. However, the currently configured one ("
+ + tokenStateService.getClass().getCanonicalName() + ") does not
fulfill this requirement!");
+ }
+ }
+
+ private boolean isTokenMigrationTarget(TokenStateService
tokenStateService) {
+ return tokenStateService instanceof TokenMigrationTarget;
+ }
+
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+ }
+
private static Properties loadBuildProperties() {
Properties properties = new Properties();
try(InputStream inputStream =
KnoxCLI.class.getClassLoader().getResourceAsStream( "build.properties" )) {
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/util/TokenMigrationTool.java
b/gateway-server/src/main/java/org/apache/knox/gateway/util/TokenMigrationTool.java
new file mode 100644
index 000000000..472bdede9
--- /dev/null
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/util/TokenMigrationTool.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.util;
+
+import static
org.apache.knox.gateway.services.token.impl.AliasBasedTokenStateService.TOKEN_ISSUE_TIME_POSTFIX;
+import static
org.apache.knox.gateway.services.token.impl.AliasBasedTokenStateService.TOKEN_MAX_LIFETIME_POSTFIX;
+import static
org.apache.knox.gateway.services.token.impl.AliasBasedTokenStateService.TOKEN_META_POSTFIX;
+
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+import org.apache.knox.gateway.services.security.AliasService;
+import org.apache.knox.gateway.services.security.AliasServiceException;
+import org.apache.knox.gateway.services.security.token.TokenMetadata;
+import org.apache.knox.gateway.services.security.token.TokenStateService;
+import org.apache.knox.gateway.services.token.impl.TokenStateServiceMessages;
+
+public class TokenMigrationTool {
+
+ private static final TokenStateServiceMessages LOG =
MessagesFactory.get(TokenStateServiceMessages.class);
+
+ private final AliasService aliasService;
+ private final TokenStateService tokenStateService;
+ private final PrintStream out;
+
+ private int progressCount = 10;
+ private boolean archiveMigratedTokens;
+ private boolean migrateExpiredTokens;
+ private boolean verbose;
+
+ public TokenMigrationTool(AliasService aliasService, TokenStateService
tokenStateService, PrintStream out) {
+ this.aliasService = aliasService;
+ this.tokenStateService = tokenStateService;
+ this.out = out;
+ }
+
+ public void setProgressCount(int progressCount) {
+ this.progressCount = progressCount;
+ }
+
+ public void setArchiveMigratedTokens(boolean archiveMigratedTokens) {
+ this.archiveMigratedTokens = archiveMigratedTokens;
+ }
+
+ public void setMigrateExpiredTokens(boolean migrateExpiredTokens) {
+ this.migrateExpiredTokens = migrateExpiredTokens;
+ }
+
+ public void setVerbose(boolean verbose) {
+ this.verbose = verbose;
+ }
+
+ public void migrateTokensFromGatewayCredentialStore() {
+ try {
+ final Map<String, TokenData> tokenDataMap = new ConcurrentHashMap<>();
+ final long start = System.currentTimeMillis();
+ String logMessage = "Loading token aliases from the __gateway credential
store. This could take a while.";
+ log(logMessage);
+ final Map<String, char[]> passwordAliasMap =
aliasService.getPasswordsForGateway();
+ log("Token aliases loaded in " + (System.currentTimeMillis() - start) +
" milliseconds");
+ String alias;
+ for (Map.Entry<String, char[]> passwordAliasMapEntry :
passwordAliasMap.entrySet()) {
+ alias = passwordAliasMapEntry.getKey();
+ processAlias(passwordAliasMap, passwordAliasMapEntry, alias,
tokenDataMap);
+ }
+
+ final long migrationStart = System.currentTimeMillis();
+ final AtomicInteger count = new AtomicInteger(0);
+ tokenDataMap.entrySet().forEach(entry -> {
+ int loggedCount = 0;
+ saveTokenIfComplete(tokenStateService, entry.getKey(),
entry.getValue());
+ count.incrementAndGet();
+ // log some progress (it's very useful in case a huge amount of token
related
+ // aliases in __gateway-credentials.jceks)
+ if (count.intValue() > 0 && (count.intValue() % progressCount == 0) &&
loggedCount != count.intValue()) {
+ loggedCount = count.intValue();
+ logProgress(count.intValue(), System.currentTimeMillis() -
migrationStart);
+ }
+ });
+
+ logProgress(count.intValue(), System.currentTimeMillis() -
migrationStart);
+
+ archiveTokens(tokenDataMap);
+
+ removeTokenAliasesFromGatewayCredentialStore(tokenDataMap);
+ } catch (AliasServiceException e) {
+ throw new RuntimeException("Error while migrating tokens from __gateway
credential store: " + e.getMessage(), e);
+ }
+ }
+
+ private void log(String message) {
+ LOG.info(message);
+ if (out != null) {
+ out.println(message);
+ }
+ }
+
+ private void logProgress(int count, long duration) {
+ log(String.format(Locale.getDefault(), "Processed %d tokens in %d
milliseconds", count, duration));
+ }
+
+ /*
+ *
+ * The AliasBasedTSS implementation persists 4 aliases in
__gateway-credentials.jceks:
+ * - an alias which maps a token ID to its expiration time
+ * - an alias with '--max' postfix which maps the maximum lifetime of the
token identified by the 1st alias
+ * - an alias with '--iss' postfix which maps the issue time of the token
+ * - an alias with '-meta' postfix which maps an arbitrary metadata of the
token
+ *
+ */
+ private void processAlias(final Map<String, char[]> passwordAliasMap,
Map.Entry<String, char[]> passwordAliasMapEntry, String alias,
+ Map<String, TokenData> tokenDataMap) {
+ String tokenId = null;
+ long expiration, maxLifeTime;
+ if (alias.endsWith(TOKEN_MAX_LIFETIME_POSTFIX)) {
+ tokenId = alias.substring(0, alias.indexOf(TOKEN_MAX_LIFETIME_POSTFIX));
+ tokenDataMap.putIfAbsent(tokenId, new TokenData());
+ expiration = convertCharArrayToLong(passwordAliasMap.get(tokenId));
+ maxLifeTime = convertCharArrayToLong(passwordAliasMapEntry.getValue());
+ tokenDataMap.get(tokenId).expiration = expiration;
+ tokenDataMap.get(tokenId).maxLifeTime = maxLifeTime;
+ } else if (alias.endsWith(TOKEN_META_POSTFIX)) {
+ tokenId = alias.substring(0, alias.indexOf(TOKEN_META_POSTFIX));
+ tokenDataMap.putIfAbsent(tokenId, new TokenData());
+ tokenDataMap.get(tokenId).metadata = TokenMetadata.fromJSON(new
String(passwordAliasMapEntry.getValue()));
+ } else if (alias.endsWith(TOKEN_ISSUE_TIME_POSTFIX)) {
+ tokenId = alias.substring(0, alias.indexOf(TOKEN_ISSUE_TIME_POSTFIX));
+ tokenDataMap.putIfAbsent(tokenId, new TokenData());
+ tokenDataMap.get(tokenId).issueTime =
convertCharArrayToLong(passwordAliasMapEntry.getValue());
+ }
+ }
+
+ private long convertCharArrayToLong(char[] charArray) {
+ return Long.parseLong(new String(charArray));
+ }
+
+ private void saveTokenIfComplete(TokenStateService tokenStateService, String
tokenId, TokenData tokenData) {
+ if (tokenId != null && tokenData.isComplete() && !tokenData.isProcessed())
{
+ if (migrateToken(tokenData)) {
+ tokenStateService.addToken(tokenId, tokenData.issueTime,
tokenData.expiration, tokenData.maxLifeTime);
+ tokenStateService.addMetadata(tokenId, tokenData.metadata);
+ if (verbose) {
+ log("Migrated token " + Tokens.getTokenIDDisplayText(tokenId) + "
into the configured TokenStateService backend.");
+ }
+ } else {
+ if (verbose) {
+ log("Skipping the migration of expired token with ID = " +
Tokens.getTokenIDDisplayText(tokenId));
+ }
+ }
+ }
+ tokenData.processed = true;
+ }
+
+ private boolean migrateToken(TokenData tokenData) {
+ return tokenData.isExpired() ? migrateExpiredTokens : true;
+ }
+
+ private void archiveTokens(Map<String, TokenData> tokenDataMap) throws
AliasServiceException {
+ if (archiveMigratedTokens) {
+ final String cluster = "__tokens";
+ log("Archiving token aliases in the " + cluster + " credential
store...");
+ final long start = System.currentTimeMillis();
+ final Map<String, String> tokenAliasesToArchive = new HashMap<>();
+ tokenDataMap.entrySet().forEach(tokenDataMapEntry -> {
+ String tokenId = tokenDataMapEntry.getKey();
+ tokenDataMapEntry.getValue();
+ tokenAliasesToArchive.put(tokenId,
String.valueOf(tokenDataMapEntry.getValue().expiration));
+ tokenAliasesToArchive.put(tokenId + TOKEN_MAX_LIFETIME_POSTFIX,
String.valueOf(tokenDataMapEntry.getValue().maxLifeTime));
+ tokenAliasesToArchive.put(tokenId + TOKEN_ISSUE_TIME_POSTFIX,
String.valueOf(tokenDataMapEntry.getValue().issueTime));
+ tokenAliasesToArchive.put(tokenId + TOKEN_META_POSTFIX,
tokenDataMapEntry.getValue().metadata.toJSON());
+ });
+ aliasService.addAliasesForCluster(cluster, tokenAliasesToArchive);
+ log("Archived token related aliases in the " + cluster + " credential
store in " + (System.currentTimeMillis() - start) + " millsieconds ");
+ }
+ }
+
+ private void removeTokenAliasesFromGatewayCredentialStore(Map<String,
TokenData> tokenDataMap) throws AliasServiceException {
+ log("Removing token aliases from the __gateway credential store...");
+ final long start = System.currentTimeMillis();
+ final Set<String> tokenAliases = new HashSet<>();
+ tokenDataMap.entrySet().forEach(tokenDataMapEntry -> {
+ String tokenId = tokenDataMapEntry.getKey();
+ tokenAliases.addAll(Arrays.asList(tokenId, tokenId +
TOKEN_MAX_LIFETIME_POSTFIX, tokenId + TOKEN_ISSUE_TIME_POSTFIX, tokenId +
TOKEN_META_POSTFIX));
+ });
+ aliasService.removeAliasesForCluster(AliasService.NO_CLUSTER_NAME,
tokenAliases);
+ log("Removed token related aliases from the __gateway credential store in
" + (System.currentTimeMillis() - start) + " milliseconds");
+ }
+
+ private class TokenData {
+ boolean processed;
+ long issueTime = -1;
+ long maxLifeTime = -1;
+ long expiration = -2; // can be set to '-1' meaning it never expires
+ TokenMetadata metadata;
+
+ boolean isComplete() {
+ return issueTime != -1 && maxLifeTime != -1 && expiration != -2 &&
metadata != null;
+ }
+
+ boolean isProcessed() {
+ return processed;
+ }
+
+ boolean isExpired() {
+ return expiration == -1 ? false : expiration <
System.currentTimeMillis();
+ }
+ }
+
+}
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/ServiceFactoryTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/ServiceFactoryTest.java
index 8a89a2e69..6c1fa6894 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/ServiceFactoryTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/ServiceFactoryTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.knox.gateway.services.factory;
+import static
org.apache.knox.gateway.services.security.AliasService.NO_CLUSTER_NAME;
import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
@@ -25,13 +26,17 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.io.File;
+import java.io.IOException;
import java.lang.reflect.Field;
+import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
-import org.apache.knox.gateway.config.GatewayConfig;
+import org.apache.knox.gateway.config.impl.GatewayConfigImpl;
import org.apache.knox.gateway.services.GatewayServices;
import org.apache.knox.gateway.services.Service;
import org.apache.knox.gateway.services.ServiceLifecycleException;
@@ -39,9 +44,14 @@ import org.apache.knox.gateway.services.ServiceType;
import org.apache.knox.gateway.services.TestService;
import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClientService;
import org.apache.knox.gateway.services.security.AliasService;
+import org.apache.knox.gateway.services.security.AliasServiceException;
import org.apache.knox.gateway.services.security.KeystoreService;
import org.apache.knox.gateway.services.security.MasterService;
+import org.apache.knox.gateway.services.token.impl.DerbyDBTokenStateService;
+import org.apache.knox.gateway.util.JDBCUtils;
+import org.apache.knox.test.TestUtils;
import org.easymock.EasyMock;
+import org.junit.After;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
@@ -52,15 +62,42 @@ class ServiceFactoryTest {
public ExpectedException expectedException = ExpectedException.none();
protected final GatewayServices gatewayServices =
EasyMock.createNiceMock(GatewayServices.class);
- protected final GatewayConfig gatewayConfig =
EasyMock.createNiceMock(GatewayConfig.class);
+ protected final GatewayConfigImpl gatewayConfig =
EasyMock.createNiceMock(GatewayConfigImpl.class);
protected final Map<String, String> options = new HashMap<>();
+ protected File tempDbFolder;
protected void initConfig() {
+ initConfig(false);
+ }
+
+ protected void initConfig(boolean expectDbCredentialLookup) {
+ final String masterSecret = "M4st3RSecret!";
final MasterService masterService =
EasyMock.createNiceMock(MasterService.class);
+
expect(masterService.getMasterSecret()).andReturn(masterSecret.toCharArray()).anyTimes();
+ replay(masterService);
expect(gatewayServices.getService(ServiceType.MASTER_SERVICE)).andReturn(masterService).anyTimes();
final KeystoreService keystoreservice =
EasyMock.createNiceMock(KeystoreService.class);
expect(gatewayServices.getService(ServiceType.KEYSTORE_SERVICE)).andReturn(keystoreservice).anyTimes();
final AliasService aliasService =
EasyMock.createNiceMock(AliasService.class);
+ if (expectDbCredentialLookup) {
+ try {
+ aliasService.addAliasForCluster(NO_CLUSTER_NAME,
JDBCUtils.DATABASE_USER_ALIAS_NAME,
DerbyDBTokenStateService.DEFAULT_TOKEN_DB_USER_NAME);
+ EasyMock.expectLastCall().anyTimes();
+ aliasService.addAliasForCluster(NO_CLUSTER_NAME,
JDBCUtils.DATABASE_PASSWORD_ALIAS_NAME, masterSecret);
+ EasyMock.expectLastCall().anyTimes();
+
expect(aliasService.getPasswordFromAliasForGateway(JDBCUtils.DATABASE_USER_ALIAS_NAME)).andReturn(DerbyDBTokenStateService.DEFAULT_TOKEN_DB_USER_NAME.toCharArray()).anyTimes();
+
expect(aliasService.getPasswordFromAliasForGateway(JDBCUtils.DATABASE_PASSWORD_ALIAS_NAME)).andReturn(masterSecret.toCharArray()).anyTimes();
+
+ // prepare GatewayConfig
+
expect(gatewayConfig.getDatabaseType()).andReturn(JDBCUtils.DERBY_DB_TYPE).anyTimes();
+ tempDbFolder = TestUtils.createTempDir(this.getClass().getName());
+
expect(gatewayConfig.getGatewaySecurityDir()).andReturn(tempDbFolder.getAbsolutePath()).anyTimes();
+
expect(gatewayConfig.getDatabaseName()).andReturn(Paths.get(tempDbFolder.getAbsolutePath(),
DerbyDBTokenStateService.DB_NAME).toString()).anyTimes();
+ } catch (AliasServiceException | IOException e) {
+ // NOP
+ }
+ }
+ replay(aliasService);
expect(gatewayServices.getService(ServiceType.ALIAS_SERVICE)).andReturn(aliasService).anyTimes();
final RemoteConfigurationRegistryClientService registryClientService =
EasyMock.createNiceMock(RemoteConfigurationRegistryClientService.class);
expect(gatewayServices.getService(ServiceType.REMOTE_REGISTRY_CLIENT_SERVICE)).andReturn(registryClientService).anyTimes();
@@ -69,6 +106,13 @@ class ServiceFactoryTest {
replay(gatewayConfig);
}
+ @After
+ public void tearDown() throws IOException {
+ if (tempDbFolder != null) {
+ FileUtils.forceDelete(tempDbFolder);
+ }
+ }
+
protected void testBasics(AbstractServiceFactory serviceFactory, ServiceType
nonMatchingServiceType, ServiceType matchingServiceType) throws Exception {
shouldReturnCorrectServiceType(serviceFactory, matchingServiceType);
shouldReturnNullForNonMatchingServiceType(serviceFactory,
nonMatchingServiceType);
@@ -110,9 +154,21 @@ class ServiceFactoryTest {
return isServiceSet(serviceToCheck, "aliasService");
}
+ protected boolean isAliasServiceSetOnParent(Service serviceToCheck) throws
Exception {
+ return isServiceSetOnParent(serviceToCheck, "aliasService");
+ }
+
private boolean isServiceSet(Service serviceToCheck, String
expectedServiceName) throws Exception {
- final Field aliasServiceField =
FieldUtils.getDeclaredField(serviceToCheck.getClass(), expectedServiceName,
true);
- final Object aliasServiceValue = aliasServiceField.get(serviceToCheck);
- return aliasServiceValue != null;
+ return isServiceSet(serviceToCheck.getClass(), serviceToCheck,
expectedServiceName);
+ }
+
+ private boolean isServiceSetOnParent(Service serviceToCheck, String
expectedServiceName) throws Exception {
+ return isServiceSet(serviceToCheck.getClass().getSuperclass(),
serviceToCheck, expectedServiceName);
+ }
+
+ private boolean isServiceSet(Class<?> clazz, Service serviceToCheck, String
expectedServiceName) throws Exception {
+ final Field expectedServiceField = FieldUtils.getDeclaredField(clazz,
expectedServiceName, true);
+ final Object expectedServiceValue =
expectedServiceField.get(serviceToCheck);
+ return expectedServiceValue != null;
}
}
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactoryTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactoryTest.java
index 5a3675e53..c33ecd22a 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactoryTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/services/factory/TokenStateServiceFactoryTest.java
@@ -23,56 +23,81 @@ import org.apache.knox.gateway.services.ServiceType;
import org.apache.knox.gateway.services.security.token.TokenStateService;
import org.apache.knox.gateway.services.token.impl.AliasBasedTokenStateService;
import org.apache.knox.gateway.services.token.impl.DefaultTokenStateService;
+import org.apache.knox.gateway.services.token.impl.DerbyDBTokenStateService;
import
org.apache.knox.gateway.services.token.impl.JournalBasedTokenStateService;
import org.apache.knox.gateway.services.token.impl.ZookeeperTokenStateService;
-import org.junit.Before;
import org.junit.Test;
public class TokenStateServiceFactoryTest extends ServiceFactoryTest {
private final TokenStateServiceFactory serviceFactory = new
TokenStateServiceFactory();
- @Before
- public void setUp() throws Exception {
- initConfig();
- }
-
@Test
public void testBasics() throws Exception {
+ initConfig();
super.testBasics(serviceFactory, ServiceType.MASTER_SERVICE,
ServiceType.TOKEN_STATE_SERVICE);
}
@Test
public void shouldReturnDefaultTokenStateService() throws Exception {
+ initConfig();
TokenStateService tokenStateService = (TokenStateService)
serviceFactory.create(gatewayServices, ServiceType.TOKEN_STATE_SERVICE,
gatewayConfig, options,
DefaultTokenStateService.class.getName());
assertTrue(tokenStateService instanceof DefaultTokenStateService);
}
@Test
- public void shouldReturnAliasBasedTokenStateServiceByDefault() throws
Exception {
- TokenStateService tokenStateService = (TokenStateService)
serviceFactory.create(gatewayServices, ServiceType.TOKEN_STATE_SERVICE,
gatewayConfig, options, "");
- assertTrue(tokenStateService instanceof AliasBasedTokenStateService);
- assertTrue(isAliasServiceSet(tokenStateService));
+ public void shouldReturnDerbyDBTokenStateServiceByDefault() throws Exception
{
+ TokenStateService tokenStateService = null;
+ try {
+ initConfig(true);
+ tokenStateService = (TokenStateService)
serviceFactory.create(gatewayServices, ServiceType.TOKEN_STATE_SERVICE,
gatewayConfig, options, "");
+ assertTrue(tokenStateService instanceof DerbyDBTokenStateService);
+ } finally {
+ if (tokenStateService != null) {
+ tokenStateService.stop();
+ }
+ }
}
@Test
public void shouldReturnAliasBasedTokenStateService() throws Exception {
- final TokenStateService tokenStateService = (TokenStateService)
serviceFactory.create(gatewayServices, ServiceType.TOKEN_STATE_SERVICE,
gatewayConfig, options,
- AliasBasedTokenStateService.class.getName());
+ initConfig();
+ final TokenStateService tokenStateService = (TokenStateService)
serviceFactory.create(gatewayServices, ServiceType.TOKEN_STATE_SERVICE,
gatewayConfig,
+ options, AliasBasedTokenStateService.class.getName());
assertTrue(tokenStateService instanceof AliasBasedTokenStateService);
assertTrue(isAliasServiceSet(tokenStateService));
}
@Test
public void shouldReturnJournalTokenStateService() throws Exception {
+ initConfig();
assertTrue(serviceFactory.create(gatewayServices,
ServiceType.TOKEN_STATE_SERVICE, gatewayConfig, options,
JournalBasedTokenStateService.class.getName()) instanceof
JournalBasedTokenStateService);
}
@Test
public void shouldReturnZookeeperTokenStateService() throws Exception {
+ initConfig();
assertTrue(serviceFactory.create(gatewayServices,
ServiceType.TOKEN_STATE_SERVICE, gatewayConfig, options,
ZookeeperTokenStateService.class.getName()) instanceof
ZookeeperTokenStateService);
}
+
+ @Test
+ public void shouldReturnDerbyDatabaseTokenStateService() throws Exception {
+ TokenStateService tokenStateService = null;
+ try {
+ initConfig(true);
+ tokenStateService = (TokenStateService)
serviceFactory.create(gatewayServices, ServiceType.TOKEN_STATE_SERVICE,
gatewayConfig, options,
+ DerbyDBTokenStateService.class.getName());
+ assertTrue(tokenStateService instanceof DerbyDBTokenStateService);
+ assertTrue(isAliasServiceSetOnParent(tokenStateService));
+ assertTrue(isMasterServiceSet(tokenStateService));
+ } finally {
+ if (tokenStateService != null) {
+ tokenStateService.stop();
+ }
+ }
+
+ }
}
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/util/JDBCUtilsTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/util/JDBCUtilsTest.java
index f6bca1622..730827ef5 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/util/JDBCUtilsTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/util/JDBCUtilsTest.java
@@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue;
import com.mysql.cj.jdbc.MysqlDataSource;
-import org.apache.derby.jdbc.ClientDataSource;
+import org.apache.derby.jdbc.EmbeddedDataSource;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.services.security.AliasService;
import org.apache.knox.gateway.services.security.AliasServiceException;
@@ -135,23 +135,19 @@ public class JDBCUtilsTest {
final AliasService aliasService =
EasyMock.createNiceMock(AliasService.class);
EasyMock.expect(aliasService.getPasswordFromAliasForGateway(EasyMock.anyString())).andReturn(null).anyTimes();
EasyMock.replay(gatewayConfig, aliasService);
- assertTrue(JDBCUtils.getDataSource(gatewayConfig, aliasService) instanceof
ClientDataSource);
+ assertTrue(JDBCUtils.getDataSource(gatewayConfig, aliasService) instanceof
EmbeddedDataSource);
}
@Test
public void derbyDataSourceShouldHaveProperConnectionProperties() throws
Exception {
final GatewayConfig gatewayConfig =
EasyMock.createNiceMock(GatewayConfig.class);
EasyMock.expect(gatewayConfig.getDatabaseType()).andReturn(JDBCUtils.DERBY_DB_TYPE).anyTimes();
-
EasyMock.expect(gatewayConfig.getDatabaseHost()).andReturn("localhost").anyTimes();
-
EasyMock.expect(gatewayConfig.getDatabasePort()).andReturn(1527).anyTimes();
EasyMock.expect(gatewayConfig.getDatabaseName()).andReturn("sampleDatabase");
final AliasService aliasService =
EasyMock.createNiceMock(AliasService.class);
EasyMock.expect(aliasService.getPasswordFromAliasForGateway(JDBCUtils.DATABASE_USER_ALIAS_NAME)).andReturn("user".toCharArray()).anyTimes();
EasyMock.expect(aliasService.getPasswordFromAliasForGateway(JDBCUtils.DATABASE_PASSWORD_ALIAS_NAME)).andReturn("password".toCharArray()).anyTimes();
EasyMock.replay(gatewayConfig, aliasService);
- final ClientDataSource dataSource = (ClientDataSource)
JDBCUtils.getDataSource(gatewayConfig, aliasService);
- assertEquals("localhost", dataSource.getServerName());
- assertEquals(1527, dataSource.getPortNumber());
+ final EmbeddedDataSource dataSource = (EmbeddedDataSource)
JDBCUtils.getDataSource(gatewayConfig, aliasService);
assertEquals("sampleDatabase", dataSource.getDatabaseName());
assertEquals("user", dataSource.getUser());
assertEquals("password", dataSource.getPassword());
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/util/KnoxCLITest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/util/KnoxCLITest.java
index ca577fac6..a72489eed 100644
--- a/gateway-server/src/test/java/org/apache/knox/gateway/util/KnoxCLITest.java
+++ b/gateway-server/src/test/java/org/apache/knox/gateway/util/KnoxCLITest.java
@@ -884,6 +884,11 @@ public class KnoxCLITest {
assertThat( master2, not( is( master ) ) );
assertThat( rc, is( 0 ) );
assertThat(outContent.toString(StandardCharsets.UTF_8.name()),
containsString("Master secret has been persisted to disk."));
+
+ // Need to delete the master file so that the it will not interfere with
other tests
+ if( masterFile.exists() ) {
+ assertThat( "Failed to delete existing master file.",
masterFile.delete(), is( true ) );
+ }
}
@Test
@@ -1320,7 +1325,7 @@ public class KnoxCLITest {
} else {
assertThat(commandOutput, containsString(alias + " has been successfully
created."));
- final AliasService aliasService =
KnoxCLI.getGatewayServices().getService(ServiceType.ALIAS_SERVICE);
+ final AliasService aliasService =
cli.getGatewayServices().getService(ServiceType.ALIAS_SERVICE);
assertNotNull(new
String(aliasService.getPasswordFromAliasForGateway(alias)));
}
diff --git
a/gateway-service-knoxtoken/src/main/java/org/apache/knox/gateway/service/knoxtoken/TokenResource.java
b/gateway-service-knoxtoken/src/main/java/org/apache/knox/gateway/service/knoxtoken/TokenResource.java
index bc614eb2c..5c98d9fba 100644
---
a/gateway-service-knoxtoken/src/main/java/org/apache/knox/gateway/service/knoxtoken/TokenResource.java
+++
b/gateway-service-knoxtoken/src/main/java/org/apache/knox/gateway/service/knoxtoken/TokenResource.java
@@ -393,8 +393,8 @@ public class TokenResource {
}
}
} else {
- //if there is no custom configuration in the topology, then we allow
keystore and DB back-end for the tokengen application
- if ("AliasBasedTokenStateService".equals(actualTokenServiceName) ||
"JDBCTokenStateService".equals(actualTokenServiceName)) {
+ //if there is no custom configuration in the topology, then we allow
DerbyDB and custom DB back-ends for the tokengen application
+ if ("DerbyDBTokenStateService".equals(actualTokenServiceName) ||
"JDBCTokenStateService".equals(actualTokenServiceName)) {
tokenStateServiceStatusMap.put(TSS_ALLOWED_BACKEND_FOR_TOKENGEN,
"true");
}
}
diff --git
a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
index bf53c50da..d8bf0c663 100644
---
a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
+++
b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
@@ -961,7 +961,7 @@ public class GatewayTestConfig extends Configuration
implements GatewayConfig {
@Override
public String getDatabaseType() {
- return null;
+ return "derbydb";
}
@Override
@@ -981,10 +981,9 @@ public class GatewayTestConfig extends Configuration
implements GatewayConfig {
@Override
public String getDatabaseName() {
- return null;
+ return Paths.get(getGatewaySecurityDir(), "tokens").toString();
}
-
@Override
public boolean isDatabaseSslEnabled() {
return false;
@@ -1081,4 +1080,30 @@ public class GatewayTestConfig extends Configuration
implements GatewayConfig {
public long getServiceDiscoveryWriteTimeoutMillis() {
return -1;
}
+
+ @Override
+ public boolean skipTokenMigration() {
+ return true;
+ }
+
+ @Override
+ public boolean archiveMigratedTokens() {
+ return false;
+ }
+
+ @Override
+ public boolean migrateExpiredTokens() {
+ return false;
+ }
+
+ @Override
+ public boolean printVerboseTokenMigrationMessages() {
+ return false;
+ }
+
+ @Override
+ public int getTokenMigrationProgressCount() {
+ return 1;
+ }
+
}
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
index 7159ebf27..9096894b7 100644
---
a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
+++
b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
@@ -903,4 +903,35 @@ public interface GatewayConfig {
long getServiceDiscoveryReadTimeoutMillis();
long getServiceDiscoveryWriteTimeoutMillis();
+
+ /**
+ * @return <code>true</code> if token migration must be skipped when a
+ * JDBC-based TSS starts; <code>false</code> otherwise
+ */
+ boolean skipTokenMigration();
+
+ /**
+ * @return <code>true</code> if migrated tokens must be archived when a
+ * JDBC-based starts; <code>false</code> otherwise
+ */
+ boolean archiveMigratedTokens();
+
+ /**
+ * @return <code>true</code> if expired tokens must be migrated when a
+ * JDBC-based starts; <code>false</code> otherwise
+ */
+ boolean migrateExpiredTokens();
+
+ /**
+ * @return <code>true</code> if the token migration tool should print verbose
+ * messages when a JDBC-based starts; <code>false</code> otherwise
+ */
+ boolean printVerboseTokenMigrationMessages();
+
+ /**
+ * @return the number of tokens after the token migration tool displays
progress
+ * in the logs when a JDBC-based TSS starts.
+ */
+ int getTokenMigrationProgressCount();
+
}
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/services/security/impl/CMFMasterService.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/services/security/impl/CMFMasterService.java
index 0aba570b3..b10c246a5 100644
---
a/gateway-spi/src/main/java/org/apache/knox/gateway/services/security/impl/CMFMasterService.java
+++
b/gateway-spi/src/main/java/org/apache/knox/gateway/services/security/impl/CMFMasterService.java
@@ -17,7 +17,6 @@
*/
package org.apache.knox.gateway.services.security.impl;
-import de.thetaphi.forbiddenapis.SuppressForbidden;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.apache.commons.net.ntp.TimeStamp;
@@ -146,7 +145,7 @@ public class CMFMasterService {
FileUtils.writeLines(masterFile, StandardCharsets.UTF_8.name(), lines);
// restrict os permissions to only the user running this process
- chmod("600", masterFile);
+ org.apache.knox.gateway.util.FileUtils.chmod("600", masterFile);
} catch (IOException e) {
LOG.failedToPersistMasterSecret(e);
}
@@ -176,31 +175,4 @@ public class CMFMasterService {
throw e;
}
}
-
- @SuppressForbidden
- private void chmod(String args, File file) throws IOException {
- // TODO: move to Java 7 NIO support to add windows as well
- // TODO: look into the following for Windows:
Runtime.getRuntime().exec("attrib -r myFile");
- if (isUnixEnv()) {
- //args and file should never be null.
- if (args == null || file == null) {
- throw new IllegalArgumentException("nullArg");
- }
- if (!file.exists()) {
- throw new IOException("fileNotFound");
- }
-
- // " +" regular expression for 1 or more spaces
- final String[] argsString = args.split(" +");
- List<String> cmdList = new ArrayList<>();
- cmdList.add("/bin/chmod");
- cmdList.addAll(Arrays.asList(argsString));
- cmdList.add(file.getAbsolutePath());
- new ProcessBuilder(cmdList).start();
- }
- }
-
- private boolean isUnixEnv() {
- return (File.separatorChar == '/');
- }
}
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/services/security/token/TokenMigrationTarget.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/services/security/token/TokenMigrationTarget.java
new file mode 100644
index 000000000..70c61a719
--- /dev/null
+++
b/gateway-spi/src/main/java/org/apache/knox/gateway/services/security/token/TokenMigrationTarget.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.services.security.token;
+
+/**
+ * The implementations of this marker interface can be used as token migration
+ * targets in KnoxCLI's migrate-token command.
+ *
+ */
+public interface TokenMigrationTarget {
+
+}
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/util/FileUtils.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/util/FileUtils.java
new file mode 100644
index 000000000..3ef28c0eb
--- /dev/null
+++ b/gateway-spi/src/main/java/org/apache/knox/gateway/util/FileUtils.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import de.thetaphi.forbiddenapis.SuppressForbidden;
+
+public class FileUtils {
+
+ @SuppressForbidden
+ public static void chmod(String args, File file) throws IOException {
+ // TODO: move to Java 7 NIO support to add windows as well
+ // TODO: look into the following for Windows:
Runtime.getRuntime().exec("attrib
+ // -r myFile");
+ if (isUnixEnv()) {
+ // args and file should never be null.
+ if (args == null || file == null) {
+ throw new IllegalArgumentException("nullArg");
+ }
+ if (!file.exists()) {
+ throw new IOException("fileNotFound");
+ }
+
+ // " +" regular expression for 1 or more spaces
+ final String[] argsString = args.split(" +");
+ List<String> cmdList = new ArrayList<>();
+ cmdList.add("/bin/chmod");
+ cmdList.addAll(Arrays.asList(argsString));
+ cmdList.add(file.getAbsolutePath());
+ new ProcessBuilder(cmdList).start();
+ }
+ }
+
+ private static boolean isUnixEnv() {
+ return File.separatorChar == '/';
+ }
+
+}
diff --git a/gateway-test-release/pom.xml b/gateway-test-release/pom.xml
index 59973a7f0..abbdb9cc0 100644
--- a/gateway-test-release/pom.xml
+++ b/gateway-test-release/pom.xml
@@ -345,6 +345,7 @@
<reuseForks>false</reuseForks>
<systemPropertyVariables>
<project.version>${project.version}</project.version>
+
<derby.stream.error.file>/dev/null</derby.stream.error.file>
</systemPropertyVariables>
</configuration>
</plugin>
diff --git a/gateway-test-release/webhdfs-kerb-test/pom.xml
b/gateway-test-release/webhdfs-kerb-test/pom.xml
index c25482911..2c4a46f48 100644
--- a/gateway-test-release/webhdfs-kerb-test/pom.xml
+++ b/gateway-test-release/webhdfs-kerb-test/pom.xml
@@ -110,6 +110,7 @@
<reuseForks>false</reuseForks>
<systemPropertyVariables>
<project.version>${project.version}</project.version>
+
<derby.stream.error.file>/dev/null</derby.stream.error.file>
</systemPropertyVariables>
</configuration>
</plugin>
diff --git a/gateway-test-release/webhdfs-test/pom.xml
b/gateway-test-release/webhdfs-test/pom.xml
index b809987e6..9f29e3e3a 100644
--- a/gateway-test-release/webhdfs-test/pom.xml
+++ b/gateway-test-release/webhdfs-test/pom.xml
@@ -102,6 +102,7 @@
<reuseForks>false</reuseForks>
<systemPropertyVariables>
<project.version>${project.version}</project.version>
+
<derby.stream.error.file>/dev/null</derby.stream.error.file>
</systemPropertyVariables>
</configuration>
</plugin>
diff --git a/gateway-test/pom.xml b/gateway-test/pom.xml
index 2d14fb3a0..c1ead63fa 100644
--- a/gateway-test/pom.xml
+++ b/gateway-test/pom.xml
@@ -245,6 +245,12 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
@@ -327,6 +333,7 @@
<reuseForks>false</reuseForks>
<systemPropertyVariables>
<project.version>${project.version}</project.version>
+
<derby.stream.error.file>/dev/null</derby.stream.error.file>
</systemPropertyVariables>
</configuration>
</plugin>
diff --git
a/gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
b/gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
index e62b5496f..e457c4627 100644
---
a/gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
+++
b/gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
@@ -28,6 +28,7 @@ import io.restassured.response.Response;
import io.restassured.specification.ResponseSpecification;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
@@ -149,6 +150,7 @@ public class GatewayBasicFuncTest {
public static void setUpBeforeClass() throws Exception {
LOG_ENTER();
GatewayTestConfig config = new GatewayTestConfig();
+ driver.config = config;
driver.setResourceBase(GatewayBasicFuncTest.class);
driver.setupLdap(0);
driver.setupService("WEBHDFS", "http://"; + TEST_HOST + ":50070/webhdfs",
"/cluster/webhdfs", USE_MOCK_SERVICES);
@@ -3710,8 +3712,7 @@ public class GatewayBasicFuncTest {
String[] args = {"service-test", "--master", "knox", "--cluster",
driver.clusterName, "--hostname", gatewayAddress.getHostName(),
"--port", gatewayPort, "--u", "kminder","--p", "kminder-password" };
- KnoxCLI cli = new KnoxCLI();
- cli.run(args);
+ getKnoxCli().run(args);
Assume.assumeTrue("Gateway port should not contain status code",
!gatewayPort.contains("404") && !gatewayPort.contains("403"));
@@ -3724,8 +3725,7 @@ public class GatewayBasicFuncTest {
String[] args2 = {"service-test", "--master", "knox", "--cluster",
driver.clusterName, "--hostname", gatewayAddress.getHostName(),
"--port", gatewayPort};
- cli = new KnoxCLI();
- cli.run(args2);
+ getKnoxCli().run(args2);
assertThat(outContent.toString(StandardCharsets.UTF_8.name()),
(containsString("Username and/or password not supplied. Expect HTTP 401
Unauthorized responses.")));
outContent.reset();
@@ -3733,8 +3733,7 @@ public class GatewayBasicFuncTest {
String[] args3 = {"service-test", "--master", "knox", "--cluster",
driver.clusterName, "--hostname", "bad-host",
"--port", "0", "--u", "guest", "--p", "guest-password" };
- cli = new KnoxCLI();
- cli.run(args3);
+ getKnoxCli().run(args3);
assertThat(outContent.toString(StandardCharsets.UTF_8.name()).toLowerCase(Locale.ROOT),
either(containsString("nodename nor servname
provided")).or(containsString("name or service not known"))
.or(containsString("//bad-host:0/")));
@@ -3743,8 +3742,7 @@ public class GatewayBasicFuncTest {
String[] args4 = {"service-test", "--master", "knox", "--cluster",
driver.clusterName, "--hostname", gatewayAddress.getHostName(),
"--port", "543", "--u", "mapred", "--p", "mapred-password" };
- cli = new KnoxCLI();
- cli.run(args4);
+ getKnoxCli().run(args4);
assertThat(outContent.toString(StandardCharsets.UTF_8.name()),
containsString("failed: Connection refused"));
outContent.reset();
@@ -3752,8 +3750,7 @@ public class GatewayBasicFuncTest {
String[] args5 = {"service-test", "--master", "knox", "--hostname",
gatewayAddress.getHostName(),
"--port", "543", "--u", "mapred", "--p", "mapred-password" };
- cli = new KnoxCLI();
- cli.run(args5);
+ getKnoxCli().run(args5);
assertThat(outContent.toString(StandardCharsets.UTF_8.name()),
containsString("--cluster argument is required"));
outContent.reset();
@@ -3762,6 +3759,12 @@ public class GatewayBasicFuncTest {
LOG_EXIT();
}
+ private KnoxCLI getKnoxCli() {
+ final KnoxCLI cli = new KnoxCLI();
+ cli.setConf((Configuration) driver.config);
+ return cli;
+ }
+
@Test( timeout = TestUtils.MEDIUM_TIMEOUT )
public void testSolrRESTAPI() throws Exception {
LOG_ENTER();
diff --git
a/knox-token-generation-ui/token-generation/app/token-generation.component.ts
b/knox-token-generation-ui/token-generation/app/token-generation.component.ts
index b3fecc411..e155fc931 100644
---
a/knox-token-generation-ui/token-generation/app/token-generation.component.ts
+++
b/knox-token-generation-ui/token-generation/app/token-generation.component.ts
@@ -188,9 +188,9 @@ export class TokenGenerationComponent implements OnInit {
private decideTssMessage() {
if (this.tssStatus.tokenManagementEnabled) {
if (this.tssStatus.allowedTssForTokengen) {
- if (this.tssStatus.actualTssBackend === 'AliasBasedTokenStateService')
{
- this.setTssMessage('warning', `Token management backend is
configured to store tokens in keystores.
- This is only valid non-HA environments!`);
+ if (this.tssStatus.actualTssBackend === 'DerbyDBTokenStateService') {
+ this.setTssMessage('warning', `Token management backend is
configured to store tokens in a Derby DB on local file system.
+ This is only valid in non-HA environments!`);
} else {
this.setTssMessage('info', 'Token management backend is properly
configured for HA and production deployments.');
}
diff --git a/pom.xml b/pom.xml
index a8b331784..8b0c51611 100644
--- a/pom.xml
+++ b/pom.xml
@@ -539,6 +539,7 @@
</excludedGroups>
<systemPropertyVariables>
<project.version>${project.version}</project.version>
+
<derby.stream.error.file>/dev/null</derby.stream.error.file>
</systemPropertyVariables>
</configuration>
</plugin>
@@ -558,6 +559,9 @@
<includes>
<include>**/*.java</include>
</includes>
+ <systemPropertyVariables>
+
<derby.stream.error.file>/dev/null</derby.stream.error.file>
+ </systemPropertyVariables>
</configuration>
</execution>
</executions>
