This is an automated email from the ASF dual-hosted git repository.
amagyar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new 6d3f960b5 KNOX-2835 - SQL DB based topology monitor
6d3f960b5 is described below
commit 6d3f960b54fee520b2311f614a073cc8c2592403
Author: Attila Magyar <[email protected]>
AuthorDate: Tue Nov 15 11:24:01 2022 +0100
KNOX-2835 - SQL DB based topology monitor
---
.../org/apache/knox/gateway/GatewayMessages.java | 40 ++++
.../gateway/config/impl/GatewayConfigImpl.java | 8 +
.../services/token/impl/TokenStateDatabase.java | 30 +--
.../topology/impl/DefaultTopologyService.java | 65 ++-----
.../monitor/RemoteConfigurationMonitorFactory.java | 73 -------
.../RemoteConfigurationMonitorServiceFactory.java | 75 ++++++++
...va => ZkRemoteConfigurationMonitorService.java} | 90 +++++++--
.../db/DbRemoteConfigurationMonitorService.java | 149 ++++++++++++++
.../topology/monitor/db/LocalDirectory.java | 85 ++++++++
.../RemoteConfig.java} | 30 ++-
.../topology/monitor/db/RemoteConfigDatabase.java | 150 +++++++++++++++
.../org/apache/knox/gateway/util/JDBCUtils.java | 28 +++
...logy.monitor.RemoteConfigurationMonitorProvider | 19 --
.../main/resources/createKnoxProvidersTable.sql | 21 ++
.../resources/createKnoxTokenDescriptorsTable.sql | 21 ++
.../services/AbstractGatewayServicesTest.java | 3 +-
.../token/impl/JDBCTokenStateServiceTest.java | 13 +-
...> ZkRemoteConfigurationMonitorServiceTest.java} | 4 +-
.../monitor/ZooKeeperConfigurationMonitorTest.java | 2 +-
.../DbRemoteConfigurationMonitorServiceTest.java | 213 +++++++++++++++++++++
.../topology/monitor/db/LocalDirectoryTest.java | 106 ++++++++++
.../monitor/db/RemoteConfigDatabaseTest.java | 141 ++++++++++++++
.../org/apache/knox/gateway/GatewayTestConfig.java | 5 +
.../apache/knox/gateway/config/GatewayConfig.java | 2 +
.../apache/knox/gateway/services/ServiceType.java | 3 +-
.../monitor/RemoteConfigurationMonitor.java | 15 +-
.../RemoteConfigurationMonitorProvider.java | 34 ----
.../monitor/RemoteConfigurationMonitorTest.java | 21 +-
28 files changed, 1183 insertions(+), 263 deletions(-)
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
index aa2130fbc..428f40f8e 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
@@ -19,6 +19,7 @@ package org.apache.knox.gateway;
import java.io.File;
import java.net.URI;
+import java.time.Instant;
import java.util.Date;
import java.util.Map;
import java.util.Set;
@@ -29,6 +30,9 @@ import org.apache.knox.gateway.i18n.messages.MessageLevel;
import org.apache.knox.gateway.i18n.messages.Messages;
import org.apache.knox.gateway.i18n.messages.StackTrace;
import org.apache.knox.gateway.services.security.KeystoreServiceException;
+import org.apache.knox.gateway.topology.monitor.db.LocalDirectory;
+
+import java.io.IOException;
@Messages(logger="org.apache.knox.gateway")
public interface GatewayMessages {
@@ -727,4 +731,40 @@ public interface GatewayMessages {
@Message(level = MessageLevel.WARN, text =
"InMemoryConcurrentSessionVerifier is used and privileged user group is not
configured! Non-privileged limit applies to all users (except the unlimited
group).")
void privilegedUserGroupIsNotConfigured();
+
+ @Message(level = MessageLevel.WARN,
+ text = "Could not create local provider/descriptor config at {0},
cause: {1}")
+ void errorSynchronizingLocalProviderDescriptor(LocalDirectory dir, Exception
cause);
+
+ @Message(level = MessageLevel.INFO,
+ text = "Downloading provider/descriptor {0} to {1} from database")
+ void downloadingProviderDescriptor(String name, LocalDirectory localDir);
+
+ @Message(level = MessageLevel.INFO,
+ text = "Deleting provider/descriptor {0} from directory: {1}")
+ void deletingProviderDescriptor(String name, LocalDirectory localDir);
+
+ @Message(level = MessageLevel.WARN,
+ text = "Cannot create local directory: {0} cause: {1}")
+ void cannotCreateLocalDirectory(File base, IOException e);
+
+ @Message(level = MessageLevel.INFO,
+ text = "DB remote configuration monitor. Interval = {0} seconds")
+ void startingDbRemoteConfigurationMonitor(long intervalSeconds);
+
+ @Message(level = MessageLevel.WARN,
+ text = "Can not sync local file system from DB, cause = {0}")
+ void errorWhileSyncingLocalFileSystem(Exception e);
+
+ @Message(level = MessageLevel.DEBUG,
+ text = "Remote configuration sync completed at: {0}")
+ void remoteConfigurationSyncCompleted(Instant lastSyncTime);
+
+ @Message(level = MessageLevel.DEBUG,
+ text = "Deleting local {0} with name {1}")
+ void deletingLocalDescriptorProvider(String type, String name);
+
+ @Message(level = MessageLevel.DEBUG,
+ text = "Creating local {0} with name {1}")
+ void creatingLocalDescriptorProvider(String type, String name);
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
index 43c92351c..71e839dc3 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
@@ -242,6 +242,9 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
static final String REMOTE_CONFIG_MONITOR_CLIENT_ALLOW_READ_ACCESS =
REMOTE_CONFIG_MONITOR_CLIENT_NAME + ".allowUnauthenticatedReadAccess";
+ private static final String
REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS = GATEWAY_CONFIG_FILE_PREFIX
+ ".remote.config.monitor.db.poll.interval.seconds";
+ private static final long
REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS_DEFAULT = 15;
+
/* @since 1.1.0 Default discovery configuration */
static final String DEFAULT_DISCOVERY_ADDRESS = GATEWAY_CONFIG_FILE_PREFIX +
".discovery.default.address";
static final String DEFAULT_DISCOVERY_CLUSTER = GATEWAY_CONFIG_FILE_PREFIX +
".discovery.default.cluster";
@@ -1426,6 +1429,11 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
return nonPrivilegedUsers == null ? Collections.emptySet() : new
HashSet<>(nonPrivilegedUsers);
}
+ @Override
+ public long getDbRemoteConfigMonitorPollingInterval() {
+ return getLong(REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS,
REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS_DEFAULT);
+ }
+
@Override
public long getConcurrentSessionVerifierExpiredTokensCleaningPeriod() {
return
getLong(GATEWAY_SESSION_VERIFICATION_EXPIRED_TOKENS_CLEANING_PERIOD,
GATEWAY_SESSION_VERIFICATION_EXPIRED_TOKENS_CLEANING_PERIOD_DEFAULT);
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateDatabase.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateDatabase.java
index 0b705141c..b5f5f0abd 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateDatabase.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/token/impl/TokenStateDatabase.java
@@ -19,26 +19,22 @@ package org.apache.knox.gateway.services.token.impl;
import static java.nio.charset.StandardCharsets.UTF_8;
-import java.io.InputStream;
import java.sql.Connection;
-import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
-import java.util.Locale;
import java.util.Map;
import java.util.Set;
import javax.sql.DataSource;
import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.io.IOUtils;
import org.apache.knox.gateway.services.security.token.KnoxToken;
import org.apache.knox.gateway.services.security.token.TokenMetadata;
+import org.apache.knox.gateway.util.JDBCUtils;
public class TokenStateDatabase {
private static final String TOKENS_TABLE_CREATE_SQL_FILE_NAME =
"createKnoxTokenDatabaseTable.sql";
@@ -72,28 +68,8 @@ public class TokenStateDatabase {
}
private void createTableIfNotExists(String tableName, String
createSqlFileName) throws Exception {
- if (!isTableExists(tableName)) {
- createTable(createSqlFileName);
- }
- }
-
- private boolean isTableExists(String tableName) throws SQLException {
- boolean exists = false;
- try (Connection connection = dataSource.getConnection()) {
- final DatabaseMetaData dbMetadata = connection.getMetaData();
- final String tableNameToCheck = dbMetadata.storesUpperCaseIdentifiers()
? tableName : tableName.toLowerCase(Locale.ROOT);
- try (ResultSet tables = dbMetadata.getTables(connection.getCatalog(),
null, tableNameToCheck, null)) {
- exists = tables.next();
- }
- }
- return exists;
- }
-
- private void createTable(String createSqlFileName) throws Exception {
- final InputStream is =
TokenStateDatabase.class.getClassLoader().getResourceAsStream(createSqlFileName);
- final String createTableSql = IOUtils.toString(is, UTF_8);
- try (Connection connection = dataSource.getConnection(); Statement
createTableStatment = connection.createStatement();) {
- createTableStatment.execute(createTableSql);
+ if (!JDBCUtils.isTableExists(tableName, dataSource)) {
+ JDBCUtils.createTable(createSqlFileName, dataSource,
TokenStateDatabase.class.getClassLoader());
}
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
index 23598170a..223fdd4d4 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
@@ -39,7 +39,6 @@ import
org.apache.knox.gateway.service.definition.ServiceDefinitionChangeListene
import org.apache.knox.gateway.services.GatewayServices;
import org.apache.knox.gateway.services.ServiceLifecycleException;
import org.apache.knox.gateway.services.ServiceType;
-import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClient;
import org.apache.knox.gateway.services.security.AliasService;
import org.apache.knox.gateway.services.topology.TopologyService;
import org.apache.knox.gateway.services.topology.monitor.DescriptorsMonitor;
@@ -54,8 +53,8 @@ import org.apache.knox.gateway.topology.TopologyProvider;
import org.apache.knox.gateway.topology.Version;
import org.apache.knox.gateway.topology.discovery.ClusterConfigurationMonitor;
import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
+import
org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitorServiceFactory;
import org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitor;
-import
org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitorFactory;
import org.apache.knox.gateway.topology.simple.SimpleDescriptor;
import org.apache.knox.gateway.topology.simple.SimpleDescriptorFactory;
import org.apache.knox.gateway.topology.validation.TopologyValidator;
@@ -419,14 +418,8 @@ public class DefaultTopologyService extends
FileAlterationListenerAdaptor implem
// If the remote configuration registry is being employed, persist it
there also
if (remoteMonitor != null) {
- RemoteConfigurationRegistryClient client = remoteMonitor.getClient();
- if (client != null) {
- String entryPath = "/knox/config/shared-providers/" + name;
- client.createEntry(entryPath, content);
- result = (client.getEntryData(entryPath) != null);
- }
+ remoteMonitor.createProvider(name, content);
}
-
return result;
}
@@ -465,7 +458,10 @@ public class DefaultTopologyService extends
FileAlterationListenerAdaptor implem
// If the remote config monitor is configured, attempt to delete the
provider configuration from the remote
// registry, even if it does not exist locally.
- deleteRemoteEntry("/knox/config/shared-providers", name);
+ if (remoteMonitor != null) {
+ // If the remote config monitor is configured, delete the descriptor
from the remote registry
+ remoteMonitor.deleteProvider(providerConfig.getName()); // use file
name with extension
+ }
// Whether the remote configuration registry is being employed or not,
delete the local file if it exists
result = providerConfig == null || !providerConfig.exists() ||
providerConfig.delete();
@@ -486,12 +482,7 @@ public class DefaultTopologyService extends
FileAlterationListenerAdaptor implem
// If the remote configuration registry is being employed, persist it
there also
if (remoteMonitor != null) {
- RemoteConfigurationRegistryClient client = remoteMonitor.getClient();
- if (client != null) {
- String entryPath = "/knox/config/descriptors/" + name;
- client.createEntry(entryPath, content);
- result = (client.getEntryData(entryPath) != null);
- }
+ return remoteMonitor.createDescriptor(name, content);
}
return result;
@@ -512,8 +503,10 @@ public class DefaultTopologyService extends
FileAlterationListenerAdaptor implem
public boolean deleteDescriptor(String name) {
boolean result;
- // If the remote config monitor is configured, delete the descriptor from
the remote registry
- deleteRemoteEntry("/knox/config/descriptors", name);
+ if (remoteMonitor != null) {
+ // If the remote config monitor is configured, delete the descriptor
from the remote registry
+ remoteMonitor.deleteDescriptor(name);
+ }
// Whether the remote configuration registry is being employed or not,
delete the local file
File descriptor = getExistingFile(descriptorsDirectory, name);
@@ -661,44 +654,14 @@ public class DefaultTopologyService extends
FileAlterationListenerAdaptor implem
log.configuredMonitoringProviderConfigChangesInDirectory(sharedProvidersDirectory.getAbsolutePath());
// Initialize the remote configuration monitor, if it has been configured
- remoteMonitor = RemoteConfigurationMonitorFactory.get(config);
+ RemoteConfigurationMonitorServiceFactory provider = new
RemoteConfigurationMonitorServiceFactory();
+ remoteMonitor = (RemoteConfigurationMonitor) provider.create(gwServices,
ServiceType.REMOTE_CONFIGURATION_MONITOR, config, Collections.emptyMap());
+
} catch (Exception e) {
throw new ServiceLifecycleException(e.getMessage(), e);
}
}
- /**
- * Delete the entry in the remote configuration registry, which matches the
specified resource name.
- *
- * @param entryParent The remote registry path in which the entry exists.
- * @param name The name of the entry (typically without any file
extension).
- *
- * @return true, if the entry is deleted, or did not exist; otherwise, false.
- */
- private boolean deleteRemoteEntry(String entryParent, String name) {
- boolean result = true;
-
- if (remoteMonitor != null) {
- RemoteConfigurationRegistryClient client = remoteMonitor.getClient();
- if (client != null) {
- List<String> existingProviderConfigs =
client.listChildEntries(entryParent);
- for (String entryName : existingProviderConfigs) {
- if (FilenameUtils.getBaseName(entryName).equals(name)) {
- String entryPath = entryParent + "/" + entryName;
- client.deleteEntry(entryPath);
- result = !client.entryExists(entryPath);
- if (!result) {
- log.failedToDeletedRemoteConfigFile("descriptor", name);
- }
- break;
- }
- }
- }
- }
-
- return result;
- }
-
/**
* Utility method for listing the files in the specified directory.
* This method is "nicer" than the File#listFiles() because it will not
return null.
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorFactory.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorFactory.java
deleted file mode 100644
index 15ab3f202..000000000
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorFactory.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-package org.apache.knox.gateway.topology.monitor;
-
-import org.apache.knox.gateway.GatewayMessages;
-import org.apache.knox.gateway.GatewayServer;
-import org.apache.knox.gateway.config.GatewayConfig;
-import org.apache.knox.gateway.i18n.messages.MessagesFactory;
-import org.apache.knox.gateway.services.ServiceType;
-import org.apache.knox.gateway.services.GatewayServices;
-import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClientService;
-
-import java.util.ServiceLoader;
-
-public class RemoteConfigurationMonitorFactory {
- private static final GatewayMessages log =
MessagesFactory.get(GatewayMessages.class);
-
- private static RemoteConfigurationRegistryClientService
remoteConfigRegistryClientService;
-
- static synchronized void
setClientService(RemoteConfigurationRegistryClientService clientService) {
- remoteConfigRegistryClientService = clientService;
- }
-
- private static synchronized RemoteConfigurationRegistryClientService
getClientService() {
- if (remoteConfigRegistryClientService == null) {
- GatewayServices services = GatewayServer.getGatewayServices();
- if (services != null) {
- remoteConfigRegistryClientService =
services.getService(ServiceType.REMOTE_REGISTRY_CLIENT_SERVICE);
- }
- }
-
- return remoteConfigRegistryClientService;
- }
-
- /**
- *
- * @param config The GatewayConfig
- *
- * @return The first RemoteConfigurationMonitor extension that is found.
- */
- public static RemoteConfigurationMonitor get(GatewayConfig config) {
- RemoteConfigurationMonitor rcm = null;
-
- ServiceLoader<RemoteConfigurationMonitorProvider> providers =
-
ServiceLoader.load(RemoteConfigurationMonitorProvider.class);
- for (RemoteConfigurationMonitorProvider provider : providers) {
- try {
- rcm = provider.newInstance(config, getClientService());
- if (rcm != null) {
- break;
- }
- } catch (Exception e) {
-
log.remoteConfigurationMonitorInitFailure(e.getLocalizedMessage(), e);
- }
- }
-
- return rcm;
- }
-}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorServiceFactory.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorServiceFactory.java
new file mode 100644
index 000000000..80bb5f9ca
--- /dev/null
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorServiceFactory.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.monitor;
+
+import java.io.File;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.knox.gateway.config.GatewayConfig;
+import org.apache.knox.gateway.services.GatewayServices;
+import org.apache.knox.gateway.services.ServiceLifecycleException;
+import org.apache.knox.gateway.services.ServiceType;
+import org.apache.knox.gateway.services.factory.AbstractServiceFactory;
+import org.apache.knox.gateway.services.security.AliasService;
+import org.apache.knox.gateway.services.security.AliasServiceException;
+import
org.apache.knox.gateway.topology.monitor.db.DbRemoteConfigurationMonitorService;
+import org.apache.knox.gateway.topology.monitor.db.LocalDirectory;
+import org.apache.knox.gateway.topology.monitor.db.RemoteConfigDatabase;
+import org.apache.knox.gateway.util.JDBCUtils;
+
+
+public class RemoteConfigurationMonitorServiceFactory extends
AbstractServiceFactory {
+ @Override
+ protected RemoteConfigurationMonitor createService(GatewayServices
gatewayServices,
+ ServiceType serviceType,
+ GatewayConfig gatewayConfig,
+ Map<String, String> options,
+ String implementation) throws
ServiceLifecycleException {
+ RemoteConfigurationMonitor service = null;
+ if (matchesImplementation(implementation,
ZkRemoteConfigurationMonitorService.class)) {
+ service = new ZkRemoteConfigurationMonitorService(gatewayConfig,
gatewayServices.getService(ServiceType.REMOTE_REGISTRY_CLIENT_SERVICE));
+ } else if (matchesImplementation(implementation,
DbRemoteConfigurationMonitorService.class)) {
+ service = createDbBasedMonitor(gatewayConfig,
getAliasService(gatewayServices));
+ }
+ return service;
+ }
+
+ private DbRemoteConfigurationMonitorService
createDbBasedMonitor(GatewayConfig config, AliasService aliasService) throws
ServiceLifecycleException {
+ try {
+ RemoteConfigDatabase db = new
RemoteConfigDatabase(JDBCUtils.getDataSource(config, aliasService));
+ LocalDirectory descriptorDir = new LocalDirectory(new
File(config.getGatewayDescriptorsDir()));
+ LocalDirectory providerDir = new LocalDirectory(new
File(config.getGatewayProvidersConfigDir()));
+ return new DbRemoteConfigurationMonitorService(
+ db, providerDir, descriptorDir,
config.getDbRemoteConfigMonitorPollingInterval());
+ } catch (SQLException | AliasServiceException e) {
+ throw new ServiceLifecycleException("Cannot create
DbRemoteConfigurationMonitor", e);
+ }
+ }
+
+ @Override
+ protected ServiceType getServiceType() {
+ return ServiceType.REMOTE_CONFIGURATION_MONITOR;
+ }
+
+ @Override
+ protected Collection<String> getKnownImplementations() {
+ return
Arrays.asList(ZkRemoteConfigurationMonitorService.class.getName(),
DbRemoteConfigurationMonitorService.class.getName());
+ }
+}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorService.java
similarity index 74%
rename from
gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java
rename to
gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorService.java
index 90663fc0a..29a0b6c44 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorService.java
@@ -17,9 +17,11 @@
package org.apache.knox.gateway.topology.monitor;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
import org.apache.knox.gateway.GatewayMessages;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+import org.apache.knox.gateway.services.ServiceLifecycleException;
import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClient;
import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClient.ChildEntryListener;
import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClient.EntryListener;
@@ -32,8 +34,9 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
-class DefaultRemoteConfigurationMonitor implements RemoteConfigurationMonitor {
+class ZkRemoteConfigurationMonitorService implements
RemoteConfigurationMonitor {
private static final String NODE_KNOX = "/knox";
private static final String NODE_KNOX_CONFIG = NODE_KNOX + "/config";
@@ -114,8 +117,8 @@ class DefaultRemoteConfigurationMonitor implements
RemoteConfigurationMonitor {
* @param config The gateway configuration
* @param registryClientService The service from which the remote registry
client should be acquired.
*/
- DefaultRemoteConfigurationMonitor(GatewayConfig
config,
- RemoteConfigurationRegistryClientService
registryClientService) {
+ ZkRemoteConfigurationMonitorService(GatewayConfig
config,
+
RemoteConfigurationRegistryClientService registryClientService) {
this.providersDir = new File(config.getGatewayProvidersConfigDir());
this.descriptorsDir = new File(config.getGatewayDescriptorsDir());
@@ -138,12 +141,10 @@ class DefaultRemoteConfigurationMonitor implements
RemoteConfigurationMonitor {
}
@Override
- public RemoteConfigurationRegistryClient getClient() {
- return client;
- }
+ public void init(GatewayConfig config, Map<String, String> options) throws
ServiceLifecycleException {}
@Override
- public void start() throws Exception {
+ public void start() throws ServiceLifecycleException {
if (client == null) {
throw new IllegalStateException("Failed to acquire a remote
configuration registry client.");
}
@@ -166,10 +167,14 @@ class DefaultRemoteConfigurationMonitor implements
RemoteConfigurationMonitor {
for (String providerConfig : providerConfigs) {
File localFile = new File(providersDir, providerConfig);
- byte[] remoteContent = client.getEntryData(NODE_KNOX_PROVIDERS
+ "/" + providerConfig).getBytes(StandardCharsets.UTF_8);
- if (!localFile.exists() || !Arrays.equals(remoteContent,
FileUtils.readFileToByteArray(localFile))) {
- FileUtils.writeByteArrayToFile(localFile, remoteContent);
- log.downloadedRemoteConfigFile(providersDir.getName(),
providerConfig);
+ try {
+ byte[] remoteContent =
client.getEntryData(NODE_KNOX_PROVIDERS + "/" +
providerConfig).getBytes(StandardCharsets.UTF_8);
+ if (!localFile.exists() || !Arrays.equals(remoteContent,
FileUtils.readFileToByteArray(localFile))) {
+ FileUtils.writeByteArrayToFile(localFile,
remoteContent);
+ log.downloadedRemoteConfigFile(providersDir.getName(),
providerConfig);
+ }
+ } catch (IOException e) {
+ throw new ServiceLifecycleException("Exception while
downloading remote configs from zookeeper", e);
}
}
}
@@ -181,20 +186,69 @@ class DefaultRemoteConfigurationMonitor implements
RemoteConfigurationMonitor {
throw new IllegalStateException("Unable to access remote path: " +
NODE_KNOX_DESCRIPTORS);
}
- // Register a listener for provider config znode additions/removals
- client.addChildEntryListener(NODE_KNOX_PROVIDERS, new
ConfigDirChildEntryListener(providersDir));
+ try {
+ // Register a listener for provider config znode additions/removals
+ client.addChildEntryListener(NODE_KNOX_PROVIDERS, new
ConfigDirChildEntryListener(providersDir));
- // Register a listener for descriptor znode additions/removals
- client.addChildEntryListener(NODE_KNOX_DESCRIPTORS, new
ConfigDirChildEntryListener(descriptorsDir));
+ // Register a listener for descriptor znode additions/removals
+ client.addChildEntryListener(NODE_KNOX_DESCRIPTORS, new
ConfigDirChildEntryListener(descriptorsDir));
+ } catch (Exception e) {
+ throw new ServiceLifecycleException("Exception while registering
provider/descriptor znode listeners", e);
+ }
log.monitoringRemoteConfigurationSource(monitorSource);
}
@Override
- public void stop() throws Exception {
- client.removeEntryListener(NODE_KNOX_PROVIDERS);
- client.removeEntryListener(NODE_KNOX_DESCRIPTORS);
+ public void stop() throws ServiceLifecycleException {
+ try {
+ client.removeEntryListener(NODE_KNOX_PROVIDERS);
+ client.removeEntryListener(NODE_KNOX_DESCRIPTORS);
+ } catch (Exception e) {
+ throw new ServiceLifecycleException("Exception while stopping: " +
getClass().getName(), e);
+ }
+ }
+
+ @Override
+ public boolean createProvider(String name, String content) {
+ String entryPath = "/knox/config/shared-providers/" + name;
+ client.createEntry(entryPath, content);
+ return (client.getEntryData(entryPath) != null);
+ }
+
+ @Override
+ public boolean createDescriptor(String name, String content) {
+ String entryPath = "/knox/config/descriptors/" + name;
+ client.createEntry(entryPath, content);
+ return (client.getEntryData(entryPath) != null);
+ }
+
+ @Override
+ public boolean deleteProvider(String name) {
+ return deleteEntry("/knox/config/descriptors", name);
+ }
+
+ @Override
+ public boolean deleteDescriptor(String name) {
+ return deleteEntry("/knox/config/shared-providers", name);
+ }
+
+ private boolean deleteEntry(String entryParent, String name) {
+ boolean result = false;
+ List<String> existingProviderConfigs =
client.listChildEntries(entryParent);
+ for (String entryName : existingProviderConfigs) {
+ if (FilenameUtils.getBaseName(entryName).equals(name)) {
+ String entryPath = entryParent + "/" + entryName;
+ client.deleteEntry(entryPath);
+ result = !client.entryExists(entryPath);
+ if (!result) {
+ log.failedToDeletedRemoteConfigFile("descriptor", name);
+ }
+ break;
+ }
+ }
+ return result;
}
private void ensureEntries() {
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorService.java
new file mode 100644
index 000000000..d48ded957
--- /dev/null
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorService.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.monitor.db;
+
+import static java.util.stream.Collectors.toSet;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.knox.gateway.GatewayMessages;
+import org.apache.knox.gateway.config.GatewayConfig;
+import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+import org.apache.knox.gateway.services.ServiceLifecycleException;
+import org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitor;
+
+public class DbRemoteConfigurationMonitorService implements
RemoteConfigurationMonitor {
+ private static final GatewayMessages LOG =
MessagesFactory.get(GatewayMessages.class);
+ public static final int OFFSET_SECONDS = 5;
+ private final RemoteConfigDatabase db;
+ private final LocalDirectory providersDir;
+ private final LocalDirectory descriptorsDir;
+ private long intervalSeconds;
+ private final ScheduledExecutorService executor;
+ private Instant lastSyncTime;
+
+ public DbRemoteConfigurationMonitorService(RemoteConfigDatabase db,
LocalDirectory providersDir, LocalDirectory descriptorsDir, long
intervalSeconds) {
+ this.db = db;
+ this.providersDir = providersDir;
+ this.descriptorsDir = descriptorsDir;
+ this.executor = Executors.newSingleThreadScheduledExecutor();
+ this.intervalSeconds = intervalSeconds;
+ }
+
+ @Override
+ public void init(GatewayConfig config, Map<String, String> options) throws
ServiceLifecycleException {}
+
+ @Override
+ public void start() throws ServiceLifecycleException {
+ LOG.startingDbRemoteConfigurationMonitor(intervalSeconds);
+ executor.scheduleWithFixedDelay(this::sync, intervalSeconds,
intervalSeconds, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void stop() throws ServiceLifecycleException {
+ executor.shutdown();
+ }
+
+ @Override
+ public boolean createProvider(String name, String content) {
+ LOG.creatingLocalDescriptorProvider("provider", name);
+ return db.putProvider(name, content);
+ }
+
+ @Override
+ public boolean createDescriptor(String name, String content) {
+ LOG.creatingLocalDescriptorProvider("descriptor", name);
+ return db.putDescriptor(name, content);
+ }
+
+ @Override
+ public boolean deleteProvider(String name) {
+ LOG.deletingLocalDescriptorProvider("provider", name);
+ return db.deleteProvider(name);
+ }
+
+ @Override
+ public boolean deleteDescriptor(String name) {
+ LOG.deletingLocalDescriptorProvider("descriptor", name);
+ return db.deleteDescriptor(name);
+ }
+
+ public void sync() {
+ try {
+ syncLocalWithRemote(db.selectProviders(), providersDir);
+ syncLocalWithRemote(db.selectDescriptors(), descriptorsDir);
+ lastSyncTime = Instant.now();
+ LOG.remoteConfigurationSyncCompleted(lastSyncTime);
+ } catch (Exception e) {
+ LOG.errorWhileSyncingLocalFileSystem(e);
+ }
+ }
+
+ private void syncLocalWithRemote(List<RemoteConfig> remoteConfigs,
LocalDirectory localDir) {
+ createOrUpdateLocalFiles(remoteConfigs, localDir);
+ deleteLocalFiles(remoteConfigs, localDir);
+ }
+
+ private void createOrUpdateLocalFiles(List<RemoteConfig> remoteConfigs,
LocalDirectory localDir) {
+ Set<String> localFiles = localDir.list();
+ for (RemoteConfig remoteConfig : remoteConfigs) {
+ try {
+ String remoteContent = remoteConfig.getContent();
+ if (!localFiles.contains(remoteConfig.getName())) {
+ // if file does not exist locally, create it
+ LOG.downloadingProviderDescriptor(remoteConfig.getName(), localDir);
+ localDir.writeFile(remoteConfig.getName(), remoteContent);
+ } else if (shouldUpdateContent(remoteConfig, localDir)) {
+ // exists locally, overwrite content only if necessary
+ LOG.downloadingProviderDescriptor(remoteConfig.getName(),
localDir);
+ localDir.writeFile(remoteConfig.getName(), remoteContent);
+ }
+ } catch (IOException e) {
+ LOG.errorSynchronizingLocalProviderDescriptor(localDir, e);
+ }
+ }
+ }
+
+ private boolean shouldUpdateContent(RemoteConfig remoteConfig,
LocalDirectory localDir) throws IOException {
+ if (lastSyncTime == null || remoteConfig.getLastModified() // remote
changed since last sync?
+ .isAfter(lastSyncTime.minusSeconds(OFFSET_SECONDS))) {
+ // Change in remote config can happen during a sync.
+ // We apply an offset on lastSyncTime to make sure the changes are
picked up at the next sync.
+ // If a remote change happened after (lastSync-offset) we'll still sync.
If it happened before (lastSync-offset) we won't.
+ return
!remoteConfig.getContent().equals(localDir.fileContent(remoteConfig.getName()));
+ } else {
+ return false;
+ }
+ }
+
+ private void deleteLocalFiles(List<RemoteConfig> remoteConfigs,
LocalDirectory localDir) {
+ Set<String> remoteConfigNames =
remoteConfigs.stream().map(RemoteConfig::getName).collect(toSet());
+ for (String localFileName : localDir.list()) {
+ if (!remoteConfigNames.contains(localFileName)) {
+ LOG.deletingProviderDescriptor(localFileName, localDir);
+ localDir.deleteFile(localFileName);
+ }
+ }
+ }
+}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/LocalDirectory.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/LocalDirectory.java
new file mode 100644
index 000000000..c5a2c903e
--- /dev/null
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/LocalDirectory.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.monitor.db;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.knox.gateway.GatewayMessages;
+import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+
+public class LocalDirectory {
+ private static final GatewayMessages LOG =
MessagesFactory.get(GatewayMessages.class);
+ public static final Charset CHARSET = StandardCharsets.UTF_8;
+ private final File base;
+
+ public LocalDirectory(File base) {
+ this.base = base;
+ ensureExists(base);
+ }
+
+ private void ensureExists(File base) {
+ try {
+ if (!base.exists()) {
+ Files.createDirectories(Paths.get(base.getAbsolutePath()));
+ }
+ } catch (IOException e) {
+ LOG.cannotCreateLocalDirectory(base, e);
+ }
+ }
+
+ public void writeFile(String name, String content) throws IOException {
+ FileUtils.writeStringToFile(file(name), content, CHARSET);
+ }
+
+ public boolean deleteFile(String name) {
+ return FileUtils.deleteQuietly(file(name));
+ }
+
+ public String fileContent(String name) throws IOException {
+ try {
+ return FileUtils.readFileToString(file(name), CHARSET);
+ } catch (FileNotFoundException e) {
+ return null;
+ }
+ }
+
+ public Set<String> list() {
+ return FileUtils.listFiles(base, null, false).stream()
+ .map(File::getName)
+ .collect(Collectors.toSet());
+ }
+
+ private File file(String name) {
+ return new File(base, name);
+ }
+
+ @Override
+ public String toString() {
+ return "LocalDirectory{" +
+ "base='" + base + '\'' +
+ '}';
+ }
+}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/DefaultConfigurationMonitorProvider.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfig.java
similarity index 56%
rename from
gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/DefaultConfigurationMonitorProvider.java
rename to
gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfig.java
index 723c9f60f..87c9f87d5 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/DefaultConfigurationMonitorProvider.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfig.java
@@ -14,18 +14,30 @@
* License for the specific language governing permissions and limitations
under
* the License.
*/
-package org.apache.knox.gateway.topology.monitor;
+package org.apache.knox.gateway.topology.monitor.db;
-import org.apache.knox.gateway.config.GatewayConfig;
-import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClientService;
+import java.time.Instant;
+public class RemoteConfig {
+ private final String name;
+ private final String content;
+ private final Instant lastModified;
-public class DefaultConfigurationMonitorProvider implements
RemoteConfigurationMonitorProvider {
+ public RemoteConfig(String name, String content, Instant lastModified) {
+ this.name = name;
+ this.content = content;
+ this.lastModified = lastModified;
+ }
- @Override
- public RemoteConfigurationMonitor newInstance(final GatewayConfig
config,
- final
RemoteConfigurationRegistryClientService clientService) {
- return new DefaultRemoteConfigurationMonitor(config, clientService);
- }
+ public String getName() {
+ return name;
+ }
+ public String getContent() {
+ return content;
+ }
+
+ public Instant getLastModified() {
+ return lastModified;
+ }
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabase.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabase.java
new file mode 100644
index 000000000..891c526b0
--- /dev/null
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabase.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.monitor.db;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import javax.sql.DataSource;
+
+import org.apache.knox.gateway.util.JDBCUtils;
+
+public class RemoteConfigDatabase {
+ private static final String KNOX_PROVIDERS_TABLE_CREATE_SQL_FILE_NAME =
"createKnoxProvidersTable.sql";
+ private static final String KNOX_DESCRIPTORS_TABLE_CREATE_SQL_FILE_NAME =
"createKnoxTokenDescriptorsTable.sql";
+ private static final String KNOX_PROVIDERS_TABLE_NAME = "KNOX_PROVIDERS";
+ private static final String KNOX_DESCRIPTORS_TABLE_NAME = "KNOX_DESCRIPTORS";
+ private final DataSource dataSource;
+
+ public RemoteConfigDatabase(DataSource dataSource) {
+ this.dataSource = dataSource;
+ ensureTablesExist();
+ }
+
+ private void ensureTablesExist() {
+ try {
+ createTableIfNotExists(KNOX_PROVIDERS_TABLE_NAME,
KNOX_PROVIDERS_TABLE_CREATE_SQL_FILE_NAME);
+ createTableIfNotExists(KNOX_DESCRIPTORS_TABLE_NAME,
KNOX_DESCRIPTORS_TABLE_CREATE_SQL_FILE_NAME);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void createTableIfNotExists(String tableName, String
createSqlFileName) throws Exception {
+ if (!JDBCUtils.isTableExists(tableName, dataSource)) {
+ JDBCUtils.createTable(createSqlFileName, dataSource,
this.getClass().getClassLoader());
+ }
+ }
+
+ public List<RemoteConfig> selectProviders() {
+ return selectFrom(KNOX_PROVIDERS_TABLE_NAME);
+ }
+
+ public List<RemoteConfig> selectDescriptors() {
+ return selectFrom(KNOX_DESCRIPTORS_TABLE_NAME);
+ }
+
+ private List<RemoteConfig> selectFrom(String tableName) {
+ List<RemoteConfig> result = new ArrayList<>();
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement("SELECT
name, content, last_modified_time FROM " + tableName)) {
+ try (ResultSet rs = statement.executeQuery()) {
+ while(rs.next()) {
+ result.add(new RemoteConfig(rs.getString(1), rs.getString(2),
rs.getTimestamp(3).toInstant()));
+ }
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ return result;
+ }
+
+ /**
+ * Save provider config to DB, overwrite if exists
+ */
+ public boolean putProvider(String name, String content) {
+ return insert(name, content, KNOX_PROVIDERS_TABLE_NAME);
+ }
+
+ /**
+ * Save descriptor config to DB, overwrite if exists
+ */
+ public boolean putDescriptor(String name, String content) {
+ return insert(name, content, KNOX_DESCRIPTORS_TABLE_NAME);
+ }
+
+ private boolean insert(String name, String content, String tableName) {
+ try {
+ if (exists(name, tableName)) {
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement(
+ "UPDATE " + tableName + " SET content = ?,
last_modified_time = ? WHERE name = ?")) {
+ statement.setString(1, content);
+ statement.setTimestamp(2, Timestamp.from(Instant.now()));
+ statement.setString(3, name);
+ return statement.executeUpdate() == 1;
+ }
+ } else {
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement(
+ "INSERT INTO " + tableName + " (name, content,
last_modified_time) VALUES(?,?,?)")) {
+ statement.setString(1, name);
+ statement.setString(2, content);
+ statement.setTimestamp(3, Timestamp.from(Instant.now()));
+ return statement.executeUpdate() == 1;
+ }
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private boolean exists(String name, String tableName) throws SQLException {
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement(
+ "SELECT name FROM " + tableName + " WHERE name = ?")) {
+ statement.setString(1, name);
+ try (ResultSet resultSet = statement.executeQuery()) {
+ return resultSet.next();
+ }
+ }
+ }
+
+ public boolean deleteProvider(String name) {
+ return delete(name, KNOX_PROVIDERS_TABLE_NAME);
+ }
+
+ public boolean deleteDescriptor(String name) {
+ return delete(name, KNOX_DESCRIPTORS_TABLE_NAME);
+ }
+
+ private boolean delete(String name, String tableName) {
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement(
+ "DELETE FROM " + tableName + " WHERE name = ?")) {
+ statement.setString(1, name);
+ return statement.executeUpdate() == 1;
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
b/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
index d01fc2032..7933a6740 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/util/JDBCUtils.java
@@ -17,8 +17,11 @@
*/
package org.apache.knox.gateway.util;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import com.mysql.cj.conf.PropertyDefinitions;
import com.mysql.cj.jdbc.MysqlDataSource;
+import org.apache.commons.io.IOUtils;
import org.apache.derby.jdbc.ClientDataSource;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.services.security.AliasService;
@@ -29,7 +32,13 @@ import org.postgresql.jdbc.SslMode;
import org.postgresql.ssl.NonValidatingFactory;
import javax.sql.DataSource;
+import java.io.InputStream;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
public class JDBCUtils {
public static final String POSTGRESQL_DB_TYPE = "postgresql";
@@ -143,4 +152,23 @@ public class JDBCUtils {
return value == null ? null : new String(value);
}
+ public static boolean isTableExists(String tableName, DataSource dataSource)
throws SQLException {
+ boolean exists = false;
+ try (Connection connection = dataSource.getConnection()) {
+ final DatabaseMetaData dbMetadata = connection.getMetaData();
+ final String tableNameToCheck = dbMetadata.storesUpperCaseIdentifiers()
? tableName : tableName.toLowerCase(Locale.ROOT);
+ try (ResultSet tables = dbMetadata.getTables(connection.getCatalog(),
null, tableNameToCheck, null)) {
+ exists = tables.next();
+ }
+ }
+ return exists;
+ }
+
+ public static void createTable(String createSqlFileName, DataSource
dataSource, ClassLoader classLoader) throws Exception {
+ final InputStream is = classLoader.getResourceAsStream(createSqlFileName);
+ String createTableSql = IOUtils.toString(is, UTF_8);
+ try (Connection connection = dataSource.getConnection(); Statement
createTableStatment = connection.createStatement();) {
+ createTableStatment.execute(createTableSql);
+ }
+ }
}
diff --git
a/gateway-server/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitorProvider
b/gateway-server/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitorProvider
deleted file mode 100644
index 63f438acc..000000000
---
a/gateway-server/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitorProvider
+++ /dev/null
@@ -1,19 +0,0 @@
-##########################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##########################################################################
-
-org.apache.knox.gateway.topology.monitor.DefaultConfigurationMonitorProvider
diff --git a/gateway-server/src/main/resources/createKnoxProvidersTable.sql
b/gateway-server/src/main/resources/createKnoxProvidersTable.sql
new file mode 100644
index 000000000..da4577a5c
--- /dev/null
+++ b/gateway-server/src/main/resources/createKnoxProvidersTable.sql
@@ -0,0 +1,21 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with this
+-- work for additional information regarding copyright ownership. The ASF
+-- licenses this file to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance with the
License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+-- WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+-- License for the specific language governing permissions and limitations
under
+-- the License.
+
+CREATE TABLE IF NOT EXISTS KNOX_PROVIDERS ( -- IF NOT EXISTS syntax is not
supported by Derby
+ name varchar(256) NOT NULL,
+ content TEXT NOT NULL,
+ last_modified_time TIMESTAMP NOT NULL,
+ PRIMARY KEY (name)
+)
\ No newline at end of file
diff --git
a/gateway-server/src/main/resources/createKnoxTokenDescriptorsTable.sql
b/gateway-server/src/main/resources/createKnoxTokenDescriptorsTable.sql
new file mode 100644
index 000000000..1887d1695
--- /dev/null
+++ b/gateway-server/src/main/resources/createKnoxTokenDescriptorsTable.sql
@@ -0,0 +1,21 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with this
+-- work for additional information regarding copyright ownership. The ASF
+-- licenses this file to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance with the
License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+-- WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+-- License for the specific language governing permissions and limitations
under
+-- the License.
+
+CREATE TABLE IF NOT EXISTS KNOX_DESCRIPTORS ( -- IF NOT EXISTS syntax is not
supported by Derby
+ name varchar(256) NOT NULL,
+ content TEXT NOT NULL,
+ last_modified_time TIMESTAMP NOT NULL,
+ PRIMARY KEY (name)
+)
\ No newline at end of file
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/services/AbstractGatewayServicesTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/services/AbstractGatewayServicesTest.java
index 8ecb72684..fb1ce553d 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/services/AbstractGatewayServicesTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/services/AbstractGatewayServicesTest.java
@@ -63,7 +63,8 @@ public class AbstractGatewayServicesTest {
ServiceType.HOST_MAPPING_SERVICE,
ServiceType.SERVICE_DEFINITION_REGISTRY,
ServiceType.SERVICE_REGISTRY_SERVICE,
- ServiceType.CONCURRENT_SESSION_VERIFIER
+ ServiceType.CONCURRENT_SESSION_VERIFIER,
+ ServiceType.REMOTE_CONFIGURATION_MONITOR,
};
assertNotEquals(ServiceType.values(), orderedServiceTypes);
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateServiceTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateServiceTest.java
index d3f33df79..024025282 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateServiceTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/services/token/impl/JDBCTokenStateServiceTest.java
@@ -28,6 +28,7 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -48,6 +49,7 @@ import
org.apache.knox.gateway.services.security.token.UnknownTokenException;
import org.apache.knox.gateway.services.security.token.impl.TokenMAC;
import org.apache.knox.gateway.util.JDBCUtils;
import org.easymock.EasyMock;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
@@ -85,6 +87,15 @@ public class JDBCTokenStateServiceTest {
tokenMAC = new TokenMAC(HmacAlgorithms.HMAC_SHA_256.getName(),
"sPj8FCgQhCEi6G18kBfpswxYSki33plbelGLs0hMSbk".toCharArray());
}
+ @SuppressWarnings("PMD.JUnit4TestShouldUseAfterAnnotation")
+ @AfterClass
+ public static void tearDown() throws Exception {
+ try (Connection connection = getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("SHUTDOWN");
+ }
+ }
+
@Test
public void testAddToken() throws Exception {
final String tokenId = UUID.randomUUID().toString();
@@ -259,7 +270,7 @@ public class JDBCTokenStateServiceTest {
}
}
- private Connection getConnection() throws SQLException {
+ private static Connection getConnection() throws SQLException {
return DriverManager.getConnection(CONNECTION_URL, USERNAME, PASSWORD);
}
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/DefaultRemoteConfigurationMonitorTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorServiceTest.java
similarity index 91%
rename from
gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/DefaultRemoteConfigurationMonitorTest.java
rename to
gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorServiceTest.java
index 0d48828ac..da7fb442f 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/DefaultRemoteConfigurationMonitorTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorServiceTest.java
@@ -21,7 +21,7 @@ import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistr
import org.easymock.EasyMock;
import org.junit.Test;
-public class DefaultRemoteConfigurationMonitorTest {
+public class ZkRemoteConfigurationMonitorServiceTest {
@Test(expected=IllegalStateException.class)
public void testInitWithoutRequiredConfig() {
GatewayConfig gatewayConfig = EasyMock.createNiceMock(GatewayConfig.class);
@@ -32,6 +32,6 @@ public class DefaultRemoteConfigurationMonitorTest {
RemoteConfigurationRegistryClientService
remoteConfigurationRegistryClientService =
EasyMock.createNiceMock(RemoteConfigurationRegistryClientService.class);
- new DefaultRemoteConfigurationMonitor(gatewayConfig,
remoteConfigurationRegistryClientService);
+ new ZkRemoteConfigurationMonitorService(gatewayConfig,
remoteConfigurationRegistryClientService);
}
}
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java
index a72c1f007..8a5842859 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java
@@ -170,7 +170,7 @@ public class ZooKeeperConfigurationMonitorTest {
clientService.init(gc, Collections.emptyMap());
clientService.start();
- DefaultRemoteConfigurationMonitor cm = new
DefaultRemoteConfigurationMonitor(gc, clientService);
+ ZkRemoteConfigurationMonitorService cm = new
ZkRemoteConfigurationMonitorService(gc, clientService);
// Create a provider configuration in the test ZK, prior to starting
the monitor, to make sure that the monitor
// will download existing entries upon starting.
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorServiceTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorServiceTest.java
new file mode 100644
index 000000000..466595661
--- /dev/null
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorServiceTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.monitor.db;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptySet;
+
+import java.time.Instant;
+import java.util.HashSet;
+
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DbRemoteConfigurationMonitorServiceTest {
+ private DbRemoteConfigurationMonitorService monitor;
+ private RemoteConfigDatabase db;
+ private LocalDirectory providersDir;
+ private LocalDirectory descriptorsDir;
+ private Instant NOW = Instant.now();
+
+ @Before
+ public void setUp() throws Exception {
+ db = EasyMock.createMock(RemoteConfigDatabase.class);
+ providersDir = EasyMock.createMock(LocalDirectory.class);
+ descriptorsDir = EasyMock.createMock(LocalDirectory.class);
+ monitor = new DbRemoteConfigurationMonitorService(db, providersDir,
descriptorsDir, 60);
+ monitor.start();
+ }
+
+ @Test
+ public void testNothingToSynchronizeWhenDbIsEmpty() throws Exception {
+ EasyMock.expect(db.selectDescriptors()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
+
+ EasyMock.expect(providersDir.list()).andReturn(emptySet()).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(emptySet()).anyTimes();
+
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @Test
+ public void testCreatesNewProvider() throws Exception {
+ EasyMock.expect(providersDir.list()).andReturn(emptySet()).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(emptySet()).anyTimes();
+
+ EasyMock.expect(db.selectDescriptors()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectProviders()).andReturn(asList(
+ new RemoteConfig("test-prov", "test-prov-content",
NOW))).anyTimes();
+
+ providersDir.writeFile("test-prov", "test-prov-content");
+ EasyMock.expectLastCall().once();
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @Test
+ public void testDeletesProviderFromFileSystem() throws Exception {
+ EasyMock.expect(providersDir.list()).andReturn(new
HashSet<>(asList("local"))).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(emptySet()).anyTimes();
+
+ EasyMock.expect(db.selectDescriptors()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
+
+ EasyMock.expect(providersDir.deleteFile("local")).andReturn(true).once();
+
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @Test
+ public void testCreatesNewDescriptor() throws Exception {
+ EasyMock.expect(providersDir.list()).andReturn(emptySet()).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(emptySet()).anyTimes();
+
+ EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectDescriptors()).andReturn(asList(
+ new RemoteConfig("test-desc", "test-desc-content",
NOW))).anyTimes();
+
+ descriptorsDir.writeFile("test-desc", "test-desc-content");
+ EasyMock.expectLastCall().once();
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @Test
+ public void testDeletesDescriptorFromFileSystem() throws Exception {
+ EasyMock.expect(providersDir.list()).andReturn(emptySet()).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(new
HashSet<>(asList("local"))).anyTimes();
+
+ EasyMock.expect(db.selectDescriptors()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
+
+ EasyMock.expect(descriptorsDir.deleteFile("local")).andReturn(true).once();
+
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @Test
+ public void testMixedDeleteUpdateAtTheSameTime() throws Exception {
+ // Local FS
+ EasyMock.expect(providersDir.list()).andReturn(new
HashSet<>(asList("prov1"))).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(new
HashSet<>(asList("desc1", "desc2", "desc3-to-be-deleted"))).anyTimes();
+ // Local Contents
+
EasyMock.expect(descriptorsDir.fileContent("desc1")).andReturn("desc1-same-content").anyTimes();
+
EasyMock.expect(descriptorsDir.fileContent("desc2")).andReturn("desc2-local-content").anyTimes();
+
EasyMock.expect(providersDir.fileContent("prov1")).andReturn(null).anyTimes();
+
+ // Remote DB
+ EasyMock.expect(db.selectProviders()).andReturn(asList(
+ new RemoteConfig("prov1", "prov1-new-file", NOW)
+ )).anyTimes();
+
+ EasyMock.expect(db.selectDescriptors()).andReturn(asList(
+ new RemoteConfig("desc1", "desc1-same-content", NOW),
+ new RemoteConfig("desc2", "desc2-remote-content", NOW)
+ )).anyTimes();
+
+ // Expectations
+ descriptorsDir.writeFile("desc2", "desc2-remote-content");
+ EasyMock.expectLastCall().once();
+
EasyMock.expect(descriptorsDir.deleteFile("desc3-to-be-deleted")).andReturn(true).once();
+
+ providersDir.writeFile("prov1", "prov1-new-file");
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @Test
+ public void testTwoSyncOnlyOneUpdate() throws Exception {
+ // Local FS
+ EasyMock.expect(providersDir.list()).andReturn(emptySet()).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(new
HashSet<>(asList("desc1"))).anyTimes();
+ // Local Contents
+
EasyMock.expect(descriptorsDir.fileContent("desc1")).andReturn("local-content").anyTimes();
+
+ // Remote DB
+ EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectDescriptors()).andReturn(asList(
+ new RemoteConfig("desc1", "remote-content", NOW.minusSeconds(15))
+ )).anyTimes();
+
+ // Expectations
+ descriptorsDir.writeFile("desc1", "remote-content");
+ EasyMock.expectLastCall().once();
+
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @Test
+ public void testTwoSyncTwoUpdate() throws Exception {
+ // Local FS
+ EasyMock.expect(providersDir.list()).andReturn(emptySet()).anyTimes();
+ EasyMock.expect(descriptorsDir.list()).andReturn(new
HashSet<>(asList("desc1"))).anyTimes();
+ // Local Contents
+
EasyMock.expect(descriptorsDir.fileContent("desc1")).andReturn("local-content").anyTimes();
+
+ // Remote DB
+ EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectDescriptors()).andReturn(asList(
+ new RemoteConfig("desc1", "remote-content", NOW.plusSeconds(10))
+ )).once();
+ EasyMock.expect(db.selectDescriptors()).andReturn(asList(
+ new RemoteConfig("desc1", "remote-content", NOW.plusSeconds(20))
+ )).once();
+
+ // Expectations
+ descriptorsDir.writeFile("desc1", "remote-content");
+ EasyMock.expectLastCall().times(2);
+
+ EasyMock.replay(providersDir, descriptorsDir, db);
+ monitor.sync();
+ monitor.sync();
+ EasyMock.verify(providersDir, descriptorsDir, db);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (monitor != null) {
+ monitor.stop();
+ }
+ }
+
+}
\ No newline at end of file
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/LocalDirectoryTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/LocalDirectoryTest.java
new file mode 100644
index 000000000..bd052f3f2
--- /dev/null
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/LocalDirectoryTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.monitor.db;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class LocalDirectoryTest {
+ @Rule
+ public final TemporaryFolder testFolder = new TemporaryFolder();
+ private LocalDirectory localDirectory;
+
+ @Before
+ public void setUp() throws Exception {
+ localDirectory = new LocalDirectory(testFolder.getRoot());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (localDirectory != null) {
+ localDirectory.list().forEach(localDirectory::deleteFile);
+ }
+ }
+
+ @Test
+ public void testEmpty() {
+ assertEquals(0, localDirectory.list().size());
+ }
+
+ @Test
+ public void testDeleteNonExisting() {
+ assertFalse(localDirectory.deleteFile("non_existing.txt"));
+ }
+
+ @Test
+ public void testDeleteExisting() throws Exception {
+ localDirectory.writeFile("new.txt", "content");
+ assertTrue(localDirectory.deleteFile("new.txt"));
+ }
+
+ @Test
+ public void testCreateNewWithContent() throws Exception {
+ localDirectory.writeFile("new.txt", "content");
+ assertEquals("content", localDirectory.fileContent("new.txt"));
+ }
+
+ @Test
+ public void testOverwriteExisting() throws Exception {
+ localDirectory.writeFile("existing.txt", "old content");
+ localDirectory.writeFile("existing.txt", "new content");
+ assertEquals("new content", localDirectory.fileContent("existing.txt"));
+ }
+
+ @Test
+ public void testContentOfNonExistingIsNull() throws Exception {
+ assertEquals(null, localDirectory.fileContent("non_existing"));
+ }
+
+ @Test
+ public void testList() throws Exception {
+ localDirectory.writeFile("1.txt", "any");
+ localDirectory.writeFile("2.txt", "any");
+ assertEquals(2, localDirectory.list().size());
+ assertTrue(localDirectory.list().contains("1.txt"));
+ assertTrue(localDirectory.list().contains("2.txt"));
+ }
+
+ @Test
+ public void testTwoWithDifferentContent() throws Exception {
+ localDirectory.writeFile("1.txt", "content 1");
+ localDirectory.writeFile("2.txt", "content 2");
+ assertEquals("content 1", localDirectory.fileContent("1.txt"));
+ assertEquals("content 2", localDirectory.fileContent("2.txt"));
+ }
+
+ @Test
+ public void testCreatesDirectoryIfDoesNotExist() throws Exception {
+ LocalDirectory subDir = new LocalDirectory(new File(testFolder.getRoot(),
"sub1/sub2"));
+ assertTrue(subDir.list().isEmpty());
+ subDir.writeFile("s.txt", "any");
+ assertTrue(subDir.list().contains("s.txt"));
+ }
+}
\ No newline at end of file
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabaseTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabaseTest.java
new file mode 100644
index 000000000..baa9aecce
--- /dev/null
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabaseTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.monitor.db;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+
+import org.hsqldb.jdbc.JDBCDataSource;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class RemoteConfigDatabaseTest {
+ public static final String DB_NAME = "remote_config_test";
+ public static final String USER = "sa";
+ public static final String PASSWORD = "";
+ private static JDBCDataSource dataSource;
+ private RemoteConfigDatabase db;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ dataSource = new JDBCDataSource();
+ dataSource.setDatabaseName(DB_NAME);
+ dataSource.setUser(USER);
+ dataSource.setPassword(PASSWORD);
+ dataSource.setUrl("jdbc:hsqldb:mem:knox;sql.syntax_pgs=true"); //
sql.syntax_pgs => use postgres syntax
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ db = new RemoteConfigDatabase(dataSource);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try (Connection connection = dataSource.getConnection(USER, PASSWORD);
+ Statement statement = connection.createStatement()) {
+ statement.execute("DROP TABLE KNOX_PROVIDERS");
+ statement.execute("DROP TABLE KNOX_DESCRIPTORS");
+ }
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ try (Connection connection = dataSource.getConnection(USER, PASSWORD);
+ Statement statement = connection.createStatement()) {
+ statement.execute("SHUTDOWN");
+ }
+ }
+
+ @Test
+ public void testEmpty() {
+ assertTrue(db.selectDescriptors().isEmpty());
+ assertTrue(db.selectDescriptors().isEmpty());
+ }
+
+ @Test
+ public void testList() {
+ db.putProvider("provider_1", "test provider1 content");
+ db.putProvider("provider_2", "test provider2 content");
+ db.putDescriptor("descriptor_1", "test descriptor content");
+
+ List<RemoteConfig> providers = db.selectProviders();
+ List<RemoteConfig> descriptors = db.selectDescriptors();
+
+ assertEquals(2, providers.size());
+ assertEquals(1, descriptors.size());
+
+ RemoteConfig descriptor = descriptors.get(0);
+ assertEquals("descriptor_1", descriptor.getName());
+ assertEquals("test descriptor content", descriptor.getContent());
+ assertTrue(Duration.between(Instant.now(),
descriptor.getLastModified()).toMillis() < 1000);
+
+ RemoteConfig provider1 = providers.stream().filter(each ->
each.getName().equals("provider_1")).findFirst().get();
+ assertEquals("test provider1 content", provider1.getContent());
+ assertTrue(Duration.between(Instant.now(),
provider1.getLastModified()).toMillis() < 1000);
+
+ RemoteConfig provider2 = providers.stream().filter(each ->
each.getName().equals("provider_2")).findFirst().get();
+ assertEquals("test provider2 content", provider2.getContent());
+ assertTrue(Duration.between(Instant.now(),
provider2.getLastModified()).toMillis() < 1000);
+ }
+
+ @Test
+ public void testInsertOverwrite() {
+ db.putProvider("provider_1", "test provider1 content v1");
+ db.putProvider("provider_1", "test provider1 content v2");
+ List<RemoteConfig> providers = db.selectProviders();
+ assertEquals(1, providers.size());
+ assertEquals("test provider1 content v2",
db.selectProviders().get(0).getContent());
+
+ db.putDescriptor("descriptor_1", "test descriptor1 content v1");
+ db.putDescriptor("descriptor_1", "test descriptor1 content v2");
+ List<RemoteConfig> descriptors = db.selectDescriptors();
+ assertEquals(1, descriptors.size());
+ assertEquals("test descriptor1 content v2",
db.selectDescriptors().get(0).getContent());
+ }
+
+ @Test
+ public void testDelete() {
+ db.putProvider("provider_1", "test provider1 content");
+ db.putProvider("provider_2", "test provider2 content");
+ db.putDescriptor("descriptor_1", "test descriptor content");
+
+ assertEquals(2, db.selectProviders().size());
+ assertEquals(1, db.selectDescriptors().size());
+
+ db.deleteProvider("provider_2");
+
+ assertEquals(1, db.selectProviders().size());
+ assertEquals(1, db.selectDescriptors().size());
+
+ db.deleteDescriptor("descriptor_1");
+
+ assertEquals(1, db.selectProviders().size());
+ assertEquals(0, db.selectDescriptors().size());
+
+ assertEquals("provider_1", db.selectProviders().get(0).getName());
+ }
+}
\ No newline at end of file
diff --git
a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
index 59d47a0cc..f9b87296b 100644
---
a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
+++
b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
@@ -1010,6 +1010,11 @@ public class GatewayTestConfig extends Configuration
implements GatewayConfig {
return null;
}
+ @Override
+ public long getDbRemoteConfigMonitorPollingInterval() {
+ return 30;
+ }
+
@Override
public long getConcurrentSessionVerifierExpiredTokensCleaningPeriod() {
return 0;
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
index a3bc6ac57..423b2e6a6 100644
---
a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
+++
b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
@@ -860,6 +860,8 @@ public interface GatewayConfig {
Set<String> getSessionVerificationUnlimitedUsers();
+ long getDbRemoteConfigMonitorPollingInterval();
+
long getConcurrentSessionVerifierExpiredTokensCleaningPeriod();
/**
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/services/ServiceType.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/services/ServiceType.java
index a41f92429..f0123043c 100644
---
a/gateway-spi/src/main/java/org/apache/knox/gateway/services/ServiceType.java
+++
b/gateway-spi/src/main/java/org/apache/knox/gateway/services/ServiceType.java
@@ -36,7 +36,8 @@ public enum ServiceType {
TOKEN_SERVICE("TokenService"),
TOKEN_STATE_SERVICE("TokenStateService"),
TOPOLOGY_SERVICE("TopologyService"),
- CONCURRENT_SESSION_VERIFIER("ConcurrentSessionVerifier");
+ CONCURRENT_SESSION_VERIFIER("ConcurrentSessionVerifier"),
+ REMOTE_CONFIGURATION_MONITOR("RemoteConfigurationMonitor");
private final String serviceTypeName;
private final String shortName;
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitor.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitor.java
index 5159d5e43..a3ecb7fe5 100644
---
a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitor.java
+++
b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitor.java
@@ -16,14 +16,11 @@
*/
package org.apache.knox.gateway.topology.monitor;
-import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClient;
-
-public interface RemoteConfigurationMonitor {
-
- void start() throws Exception;
-
- void stop() throws Exception;
-
- RemoteConfigurationRegistryClient getClient();
+import org.apache.knox.gateway.services.Service;
+public interface RemoteConfigurationMonitor extends Service {
+ boolean createProvider(String name, String content);
+ boolean createDescriptor(String name, String content);
+ boolean deleteProvider(String name);
+ boolean deleteDescriptor(String name);
}
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorProvider.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorProvider.java
deleted file mode 100644
index ea9003a54..000000000
---
a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorProvider.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-package org.apache.knox.gateway.topology.monitor;
-
-
-import org.apache.knox.gateway.config.GatewayConfig;
-import
org.apache.knox.gateway.services.config.client.RemoteConfigurationRegistryClientService;
-
-public interface RemoteConfigurationMonitorProvider {
-
- /**
- *
- * @param config The gateway configuration.
- * @param clientService The RemoteConfigurationRegistryClientService for
accessing the remote configuration.
- *
- * @return A RemoteConfigurationMonitor for keeping the local config in
sync with the remote config
- */
- RemoteConfigurationMonitor newInstance(GatewayConfig config,
RemoteConfigurationRegistryClientService clientService);
-
-}
diff --git
a/gateway-test/src/test/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorTest.java
b/gateway-test/src/test/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorTest.java
index d8529c0c4..6419df216 100644
---
a/gateway-test/src/test/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorTest.java
+++
b/gateway-test/src/test/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorTest.java
@@ -248,10 +248,7 @@ public class RemoteConfigurationMonitorTest {
clientService.init(gc, Collections.emptyMap());
clientService.start();
- RemoteConfigurationMonitorFactory.setClientService(clientService);
-
- RemoteConfigurationMonitor cm =
RemoteConfigurationMonitorFactory.get(gc);
- assertNotNull("Failed to load RemoteConfigurationMonitor", cm);
+ RemoteConfigurationMonitor cm = new
ZkRemoteConfigurationMonitorService(gc, clientService);
final ACL ANY_AUTHENTICATED_USER_ALL = new ACL(ZooDefs.Perms.ALL, new
Id("auth", ""));
List<ACL> acls = Arrays.asList(ANY_AUTHENTICATED_USER_ALL, new
ACL(ZooDefs.Perms.WRITE, ZooDefs.Ids.ANYONE_ID_UNSAFE));
@@ -285,7 +282,6 @@ public class RemoteConfigurationMonitorTest {
}
}
-
/*
* KNOX-1135
*/
@@ -324,10 +320,7 @@ public class RemoteConfigurationMonitorTest {
clientService.init(gc, Collections.emptyMap());
clientService.start();
- RemoteConfigurationMonitorFactory.setClientService(clientService);
-
- RemoteConfigurationMonitor cm =
RemoteConfigurationMonitorFactory.get(gc);
- assertNotNull("Failed to load RemoteConfigurationMonitor", cm);
+ RemoteConfigurationMonitor cm = new
ZkRemoteConfigurationMonitorService(gc, clientService);
final ACL ANY_AUTHENTICATED_USER_ALL = new ACL(ZooDefs.Perms.ALL, new
Id("auth", ""));
List<ACL> acls = Arrays.asList(ANY_AUTHENTICATED_USER_ALL, new
ACL(ZooDefs.Perms.WRITE, ZooDefs.Ids.ANYONE_ID_UNSAFE));
@@ -398,10 +391,7 @@ public class RemoteConfigurationMonitorTest {
clientService.init(gc, Collections.emptyMap());
clientService.start();
- RemoteConfigurationMonitorFactory.setClientService(clientService);
-
- RemoteConfigurationMonitor cm =
RemoteConfigurationMonitorFactory.get(gc);
- assertNotNull("Failed to load RemoteConfigurationMonitor", cm);
+ RemoteConfigurationMonitor cm = new
ZkRemoteConfigurationMonitorService(gc, clientService);
List<ACL> acls = Collections.singletonList(ANY_AUTHENTICATED_USER_ALL);
client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(acls).forPath(PATH_KNOX);
@@ -472,10 +462,7 @@ public class RemoteConfigurationMonitorTest {
clientService.init(gc, Collections.emptyMap());
clientService.start();
- RemoteConfigurationMonitorFactory.setClientService(clientService);
-
- RemoteConfigurationMonitor cm =
RemoteConfigurationMonitorFactory.get(gc);
- assertNotNull("Failed to load RemoteConfigurationMonitor", cm);
+ RemoteConfigurationMonitor cm = new
ZkRemoteConfigurationMonitorService(gc, clientService);
// Check that the config nodes really don't yet exist (the monitor
will create them if they're not present)
assertNull(client.checkExists().forPath(PATH_KNOX));
