This is an automated email from the ASF dual-hosted git repository.
amagyar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new 616a4e06e KNOX-2844 - Remote config monitor should not automatically
delete local files if they're missing from the DB (#680)
616a4e06e is described below
commit 616a4e06e8c24c6ba65250d4f853851d6a7d81b9
Author: Attila Magyar <[email protected]>
AuthorDate: Tue Nov 29 10:16:19 2022 +0100
KNOX-2844 - Remote config monitor should not automatically delete local
files if they're missing from the DB (#680)
---
.../org/apache/knox/gateway/GatewayMessages.java | 8 +++
.../gateway/config/impl/GatewayConfigImpl.java | 10 +++-
.../topology/impl/DefaultTopologyService.java | 16 +++---
.../RemoteConfigurationMonitorServiceFactory.java | 15 +++++-
.../ZkRemoteConfigurationMonitorService.java | 2 +-
.../db/DbRemoteConfigurationMonitorService.java | 49 ++++++++++++-----
.../gateway/topology/monitor/db/RemoteConfig.java | 10 ++++
.../topology/monitor/db/RemoteConfigDatabase.java | 57 ++++++++++++++++----
.../main/resources/createKnoxProvidersTable.sql | 1 +
.../resources/createKnoxTokenDescriptorsTable.sql | 1 +
.../DbRemoteConfigurationMonitorServiceTest.java | 17 +++---
.../monitor/db/RemoteConfigDatabaseTest.java | 63 ++++++++++++++++++++--
.../org/apache/knox/gateway/GatewayTestConfig.java | 5 ++
.../apache/knox/gateway/config/GatewayConfig.java | 2 +
14 files changed, 213 insertions(+), 43 deletions(-)
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
index 428f40f8e..cba99f933 100644
--- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
+++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java
@@ -767,4 +767,12 @@ public interface GatewayMessages {
@Message(level = MessageLevel.DEBUG,
text = "Creating local {0} with name {1}")
void creatingLocalDescriptorProvider(String type, String name);
+
+ @Message(level = MessageLevel.DEBUG,
+ text = "Cleaning remote configuration db. Deleting logically deleted
rows which are older than {0} seconds.")
+ void cleaningRemoteConfigTables(int cleanUpPeriodSeconds);
+
+ @Message(level = MessageLevel.INFO,
+ text = "Initializing remote configuration db. Sync interval={0}
seconds. Clean up interval={1} seconds.")
+ void initDbRemoteConfigMonitor(long syncIntervalSeconds, int
cleanUpPeriodSeconds);
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
index 71e839dc3..33bbf7066 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java
@@ -243,7 +243,10 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
REMOTE_CONFIG_MONITOR_CLIENT_NAME + ".allowUnauthenticatedReadAccess";
private static final String
REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS = GATEWAY_CONFIG_FILE_PREFIX
+ ".remote.config.monitor.db.poll.interval.seconds";
- private static final long
REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS_DEFAULT = 15;
+ private static final long
REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS_DEFAULT = 30;
+
+ private static final String
REMOTE_CONFIG_MONITOR_DB_POLLING_CLEANUP_INTERVAL_SECONDS =
GATEWAY_CONFIG_FILE_PREFIX +
".remote.config.monitor.db.cleanup.interval.seconds";
+ private static final int
REMOTE_CONFIG_MONITOR_DB_POLLING_CLEANUP_INTERVAL_DEFAULT = 3 * 60 * 60;
/* @since 1.1.0 Default discovery configuration */
static final String DEFAULT_DISCOVERY_ADDRESS = GATEWAY_CONFIG_FILE_PREFIX +
".discovery.default.address";
@@ -1434,6 +1437,11 @@ public class GatewayConfigImpl extends Configuration
implements GatewayConfig {
return getLong(REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS,
REMOTE_CONFIG_MONITOR_DB_POLLING_INTERVAL_SECONDS_DEFAULT);
}
+ @Override
+ public int getDbRemoteConfigMonitorCleanUpInterval() {
+ return getInt(REMOTE_CONFIG_MONITOR_DB_POLLING_CLEANUP_INTERVAL_SECONDS,
REMOTE_CONFIG_MONITOR_DB_POLLING_CLEANUP_INTERVAL_DEFAULT);
+ }
+
@Override
public long getConcurrentSessionVerifierExpiredTokensCleaningPeriod() {
return
getLong(GATEWAY_SESSION_VERIFICATION_EXPIRED_TOKENS_CLEANING_PERIOD,
GATEWAY_SESSION_VERIFICATION_EXPIRED_TOKENS_CLEANING_PERIOD_DEFAULT);
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
index 223fdd4d4..0ab8a7095 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java
@@ -503,15 +503,15 @@ public class DefaultTopologyService extends
FileAlterationListenerAdaptor implem
public boolean deleteDescriptor(String name) {
boolean result;
- if (remoteMonitor != null) {
- // If the remote config monitor is configured, delete the descriptor
from the remote registry
- remoteMonitor.deleteDescriptor(name);
- }
-
// Whether the remote configuration registry is being employed or not,
delete the local file
File descriptor = getExistingFile(descriptorsDirectory, name);
result = (descriptor == null) || descriptor.delete();
+ if (remoteMonitor != null && descriptor != null) {
+ // If the remote config monitor is configured, delete the descriptor
from the remote registry
+ remoteMonitor.deleteDescriptor(descriptor.getName());
+ }
+
return result;
}
@@ -654,8 +654,10 @@ public class DefaultTopologyService extends
FileAlterationListenerAdaptor implem
log.configuredMonitoringProviderConfigChangesInDirectory(sharedProvidersDirectory.getAbsolutePath());
// Initialize the remote configuration monitor, if it has been configured
- RemoteConfigurationMonitorServiceFactory provider = new
RemoteConfigurationMonitorServiceFactory();
- remoteMonitor = (RemoteConfigurationMonitor) provider.create(gwServices,
ServiceType.REMOTE_CONFIGURATION_MONITOR, config, Collections.emptyMap());
+ if (gwServices != null) { // if it was called from knoxcli, we don't
have gwServices
+ RemoteConfigurationMonitorServiceFactory provider = new
RemoteConfigurationMonitorServiceFactory();
+ remoteMonitor = (RemoteConfigurationMonitor)
provider.create(gwServices, ServiceType.REMOTE_CONFIGURATION_MONITOR, config,
Collections.emptyMap());
+ }
} catch (Exception e) {
throw new ServiceLifecycleException(e.getMessage(), e);
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorServiceFactory.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorServiceFactory.java
index 80bb5f9ca..88992095f 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorServiceFactory.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/RemoteConfigurationMonitorServiceFactory.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
import org.apache.knox.gateway.config.GatewayConfig;
import org.apache.knox.gateway.services.GatewayServices;
import org.apache.knox.gateway.services.ServiceLifecycleException;
@@ -36,6 +37,8 @@ import org.apache.knox.gateway.util.JDBCUtils;
public class RemoteConfigurationMonitorServiceFactory extends
AbstractServiceFactory {
+ private static final String DEFAULT_IMPLEMENTATION =
ZkRemoteConfigurationMonitorService.class.getName();
+
@Override
protected RemoteConfigurationMonitor createService(GatewayServices
gatewayServices,
ServiceType serviceType,
@@ -43,6 +46,9 @@ public class RemoteConfigurationMonitorServiceFactory extends
AbstractServiceFac
Map<String, String> options,
String implementation) throws
ServiceLifecycleException {
RemoteConfigurationMonitor service = null;
+ if (StringUtils.isBlank(implementation) &&
gatewayConfig.getRemoteConfigurationMonitorClientName() != null) {
+ implementation = DEFAULT_IMPLEMENTATION; // for backward
compatibility
+ }
if (matchesImplementation(implementation,
ZkRemoteConfigurationMonitorService.class)) {
service = new ZkRemoteConfigurationMonitorService(gatewayConfig,
gatewayServices.getService(ServiceType.REMOTE_REGISTRY_CLIENT_SERVICE));
} else if (matchesImplementation(implementation,
DbRemoteConfigurationMonitorService.class)) {
@@ -57,7 +63,12 @@ public class RemoteConfigurationMonitorServiceFactory
extends AbstractServiceFac
LocalDirectory descriptorDir = new LocalDirectory(new
File(config.getGatewayDescriptorsDir()));
LocalDirectory providerDir = new LocalDirectory(new
File(config.getGatewayProvidersConfigDir()));
return new DbRemoteConfigurationMonitorService(
- db, providerDir, descriptorDir,
config.getDbRemoteConfigMonitorPollingInterval());
+ db,
+ providerDir,
+ descriptorDir,
+ config.getDbRemoteConfigMonitorPollingInterval(),
+ config.getDbRemoteConfigMonitorCleanUpInterval()
+ );
} catch (SQLException | AliasServiceException e) {
throw new ServiceLifecycleException("Cannot create
DbRemoteConfigurationMonitor", e);
}
@@ -70,6 +81,6 @@ public class RemoteConfigurationMonitorServiceFactory extends
AbstractServiceFac
@Override
protected Collection<String> getKnownImplementations() {
- return
Arrays.asList(ZkRemoteConfigurationMonitorService.class.getName(),
DbRemoteConfigurationMonitorService.class.getName());
+ return Arrays.asList(DEFAULT_IMPLEMENTATION,
DbRemoteConfigurationMonitorService.class.getName());
}
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorService.java
index 29a0b6c44..0d8e34c57 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/ZkRemoteConfigurationMonitorService.java
@@ -238,7 +238,7 @@ class ZkRemoteConfigurationMonitorService implements
RemoteConfigurationMonitor
boolean result = false;
List<String> existingProviderConfigs =
client.listChildEntries(entryParent);
for (String entryName : existingProviderConfigs) {
- if (FilenameUtils.getBaseName(entryName).equals(name)) {
+ if (FilenameUtils.getName(entryName).equals(name)) {
String entryPath = entryParent + "/" + entryName;
client.deleteEntry(entryPath);
result = !client.entryExists(entryPath);
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorService.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorService.java
index d48ded957..8718ed138 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorService.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorService.java
@@ -16,7 +16,7 @@
*/
package org.apache.knox.gateway.topology.monitor.db;
-import static java.util.stream.Collectors.toSet;
+import static java.util.stream.Collectors.toList;
import java.io.IOException;
import java.time.Instant;
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.knox.gateway.GatewayMessages;
import org.apache.knox.gateway.config.GatewayConfig;
@@ -39,25 +40,30 @@ public class DbRemoteConfigurationMonitorService implements
RemoteConfigurationM
private final RemoteConfigDatabase db;
private final LocalDirectory providersDir;
private final LocalDirectory descriptorsDir;
- private long intervalSeconds;
+ private final long syncIntervalSeconds;
private final ScheduledExecutorService executor;
+ private final int cleanUpPeriodSeconds;
private Instant lastSyncTime;
- public DbRemoteConfigurationMonitorService(RemoteConfigDatabase db,
LocalDirectory providersDir, LocalDirectory descriptorsDir, long
intervalSeconds) {
+ public DbRemoteConfigurationMonitorService(RemoteConfigDatabase db,
LocalDirectory providersDir, LocalDirectory descriptorsDir, long
syncIntervalSeconds, int cleanUpPeriodSeconds) {
this.db = db;
this.providersDir = providersDir;
this.descriptorsDir = descriptorsDir;
this.executor = Executors.newSingleThreadScheduledExecutor();
- this.intervalSeconds = intervalSeconds;
+ this.syncIntervalSeconds = syncIntervalSeconds;
+ this.cleanUpPeriodSeconds = cleanUpPeriodSeconds;
}
@Override
- public void init(GatewayConfig config, Map<String, String> options) throws
ServiceLifecycleException {}
+ public void init(GatewayConfig config, Map<String, String> options) throws
ServiceLifecycleException {
+ LOG.initDbRemoteConfigMonitor(syncIntervalSeconds, cleanUpPeriodSeconds);
+ }
@Override
public void start() throws ServiceLifecycleException {
- LOG.startingDbRemoteConfigurationMonitor(intervalSeconds);
- executor.scheduleWithFixedDelay(this::sync, intervalSeconds,
intervalSeconds, TimeUnit.SECONDS);
+ LOG.startingDbRemoteConfigurationMonitor(syncIntervalSeconds);
+ executor.scheduleWithFixedDelay(this::sync, syncIntervalSeconds,
syncIntervalSeconds, TimeUnit.SECONDS);
+ executor.scheduleWithFixedDelay(this::cleanUp, cleanUpPeriodSeconds,
cleanUpPeriodSeconds, TimeUnit.SECONDS);
}
@Override
@@ -101,8 +107,15 @@ public class DbRemoteConfigurationMonitorService
implements RemoteConfigurationM
}
private void syncLocalWithRemote(List<RemoteConfig> remoteConfigs,
LocalDirectory localDir) {
- createOrUpdateLocalFiles(remoteConfigs, localDir);
- deleteLocalFiles(remoteConfigs, localDir);
+ List<RemoteConfig> existingConfigs = remoteConfigs.stream()
+ .filter(each -> !each.isDeleted())
+ .collect(toList());
+ createOrUpdateLocalFiles(existingConfigs, localDir);
+ Set<String> deletedConfigs = remoteConfigs.stream()
+ .filter(RemoteConfig::isDeleted)
+ .map(RemoteConfig::getName)
+ .collect(Collectors.toSet());
+ deleteLocalFiles(deletedConfigs, localDir);
}
private void createOrUpdateLocalFiles(List<RemoteConfig> remoteConfigs,
LocalDirectory localDir) {
@@ -137,13 +150,21 @@ public class DbRemoteConfigurationMonitorService
implements RemoteConfigurationM
}
}
- private void deleteLocalFiles(List<RemoteConfig> remoteConfigs,
LocalDirectory localDir) {
- Set<String> remoteConfigNames =
remoteConfigs.stream().map(RemoteConfig::getName).collect(toSet());
+ private void deleteLocalFiles(Set<String> deletedRemoteConfigNames,
LocalDirectory localDir) {
for (String localFileName : localDir.list()) {
- if (!remoteConfigNames.contains(localFileName)) {
- LOG.deletingProviderDescriptor(localFileName, localDir);
- localDir.deleteFile(localFileName);
+ if (deletedRemoteConfigNames.contains(localFileName)) {
+ if (localDir.deleteFile(localFileName)) {
+ LOG.deletingProviderDescriptor(localFileName, localDir);
+ }
}
}
}
+
+ /**
+ * Remove entries which were logically deleted, and they're older than the
given cleanUpPeriodHours
+ */
+ private void cleanUp() {
+ LOG.cleaningRemoteConfigTables(cleanUpPeriodSeconds);
+ db.cleanTables(cleanUpPeriodSeconds);
+ }
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfig.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfig.java
index 87c9f87d5..6ee353931 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfig.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfig.java
@@ -22,11 +22,17 @@ public class RemoteConfig {
private final String name;
private final String content;
private final Instant lastModified;
+ private final boolean deleted;
public RemoteConfig(String name, String content, Instant lastModified) {
+ this(name, content, lastModified, false);
+ }
+
+ public RemoteConfig(String name, String content, Instant lastModified,
boolean deleted) {
this.name = name;
this.content = content;
this.lastModified = lastModified;
+ this.deleted = deleted;
}
public String getName() {
@@ -40,4 +46,8 @@ public class RemoteConfig {
public Instant getLastModified() {
return lastModified;
}
+
+ public boolean isDeleted() {
+ return deleted;
+ }
}
diff --git
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabase.java
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabase.java
index 891c526b0..e298d6a6a 100644
---
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabase.java
+++
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabase.java
@@ -55,10 +55,16 @@ public class RemoteConfigDatabase {
}
}
+ /**
+ * @return all remote providers, including the deleted ones
+ */
public List<RemoteConfig> selectProviders() {
return selectFrom(KNOX_PROVIDERS_TABLE_NAME);
}
+ /**
+ * @return all remote descriptors, including the deleted one
+ */
public List<RemoteConfig> selectDescriptors() {
return selectFrom(KNOX_DESCRIPTORS_TABLE_NAME);
}
@@ -66,10 +72,16 @@ public class RemoteConfigDatabase {
private List<RemoteConfig> selectFrom(String tableName) {
List<RemoteConfig> result = new ArrayList<>();
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement = connection.prepareStatement("SELECT
name, content, last_modified_time FROM " + tableName)) {
+ PreparedStatement statement = connection.prepareStatement(
+ "SELECT name, content, last_modified_time, deleted FROM " +
tableName)) {
try (ResultSet rs = statement.executeQuery()) {
while(rs.next()) {
- result.add(new RemoteConfig(rs.getString(1), rs.getString(2),
rs.getTimestamp(3).toInstant()));
+ result.add(new RemoteConfig(
+ rs.getString(1),
+ rs.getString(2),
+ rs.getTimestamp(3).toInstant(),
+ rs.getBoolean(4)
+ ));
}
}
} catch (SQLException e) {
@@ -97,10 +109,11 @@ public class RemoteConfigDatabase {
if (exists(name, tableName)) {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(
- "UPDATE " + tableName + " SET content = ?,
last_modified_time = ? WHERE name = ?")) {
+ "UPDATE " + tableName + " SET content = ?,
last_modified_time = ?, deleted = ? WHERE name = ?")) {
statement.setString(1, content);
statement.setTimestamp(2, Timestamp.from(Instant.now()));
- statement.setString(3, name);
+ statement.setBoolean(3, false);
+ statement.setString(4, name);
return statement.executeUpdate() == 1;
}
} else {
@@ -129,22 +142,48 @@ public class RemoteConfigDatabase {
}
}
+ /**
+ * Logically delete a provider
+ */
public boolean deleteProvider(String name) {
- return delete(name, KNOX_PROVIDERS_TABLE_NAME);
+ return deleteLogical(name, KNOX_PROVIDERS_TABLE_NAME);
}
+ /**
+ * Logically delete a descriptor
+ */
public boolean deleteDescriptor(String name) {
- return delete(name, KNOX_DESCRIPTORS_TABLE_NAME);
+ return deleteLogical(name, KNOX_DESCRIPTORS_TABLE_NAME);
}
- private boolean delete(String name, String tableName) {
+ private boolean deleteLogical(String name, String tableName) {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(
- "DELETE FROM " + tableName + " WHERE name = ?")) {
- statement.setString(1, name);
+ "UPDATE " + tableName + " SET deleted = ?, last_modified_time
= ? WHERE name = ?")) {
+ statement.setBoolean(1, true);
+ statement.setTimestamp(2, Timestamp.from(Instant.now()));
+ statement.setString(3, name);
return statement.executeUpdate() == 1;
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
+
+ public int cleanTables(int olderThanSeconds) {
+ Instant instant = Instant.now().minusSeconds(olderThanSeconds);
+ return cleanTable(KNOX_PROVIDERS_TABLE_NAME, instant)
+ + cleanTable(KNOX_DESCRIPTORS_TABLE_NAME, instant);
+ }
+
+ private int cleanTable(String tableName, Instant instant) {
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement(
+ "DELETE FROM " + tableName + " WHERE deleted = ? AND
last_modified_time <= ?")) {
+ statement.setBoolean(1, true);
+ statement.setTimestamp(2, Timestamp.from(instant));
+ return statement.executeUpdate();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
diff --git a/gateway-server/src/main/resources/createKnoxProvidersTable.sql
b/gateway-server/src/main/resources/createKnoxProvidersTable.sql
index da4577a5c..a4eda9dd9 100644
--- a/gateway-server/src/main/resources/createKnoxProvidersTable.sql
+++ b/gateway-server/src/main/resources/createKnoxProvidersTable.sql
@@ -17,5 +17,6 @@ CREATE TABLE IF NOT EXISTS KNOX_PROVIDERS ( -- IF NOT EXISTS
syntax is not suppo
name varchar(256) NOT NULL,
content TEXT NOT NULL,
last_modified_time TIMESTAMP NOT NULL,
+ deleted boolean DEFAULT false NOT NULL,
PRIMARY KEY (name)
)
\ No newline at end of file
diff --git
a/gateway-server/src/main/resources/createKnoxTokenDescriptorsTable.sql
b/gateway-server/src/main/resources/createKnoxTokenDescriptorsTable.sql
index 1887d1695..614c7a139 100644
--- a/gateway-server/src/main/resources/createKnoxTokenDescriptorsTable.sql
+++ b/gateway-server/src/main/resources/createKnoxTokenDescriptorsTable.sql
@@ -17,5 +17,6 @@ CREATE TABLE IF NOT EXISTS KNOX_DESCRIPTORS ( -- IF NOT
EXISTS syntax is not sup
name varchar(256) NOT NULL,
content TEXT NOT NULL,
last_modified_time TIMESTAMP NOT NULL,
+ deleted boolean DEFAULT false NOT NULL,
PRIMARY KEY (name)
)
\ No newline at end of file
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorServiceTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorServiceTest.java
index 466595661..35e78c282 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorServiceTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/DbRemoteConfigurationMonitorServiceTest.java
@@ -21,6 +21,7 @@ import static java.util.Collections.emptyList;
import static java.util.Collections.emptySet;
import java.time.Instant;
+import java.util.Arrays;
import java.util.HashSet;
import org.easymock.EasyMock;
@@ -40,7 +41,7 @@ public class DbRemoteConfigurationMonitorServiceTest {
db = EasyMock.createMock(RemoteConfigDatabase.class);
providersDir = EasyMock.createMock(LocalDirectory.class);
descriptorsDir = EasyMock.createMock(LocalDirectory.class);
- monitor = new DbRemoteConfigurationMonitorService(db, providersDir,
descriptorsDir, 60);
+ monitor = new DbRemoteConfigurationMonitorService(db, providersDir,
descriptorsDir, 60, 3600);
monitor.start();
}
@@ -75,13 +76,15 @@ public class DbRemoteConfigurationMonitorServiceTest {
@Test
public void testDeletesProviderFromFileSystem() throws Exception {
- EasyMock.expect(providersDir.list()).andReturn(new
HashSet<>(asList("local"))).anyTimes();
+ EasyMock.expect(providersDir.list()).andReturn(new
HashSet<>(asList("to-be-deleted"))).anyTimes();
EasyMock.expect(descriptorsDir.list()).andReturn(emptySet()).anyTimes();
+ EasyMock.expect(db.selectProviders()).andReturn(Arrays.asList(
+ new RemoteConfig("to-be-deleted", "any", NOW, true)
+ )).anyTimes();
EasyMock.expect(db.selectDescriptors()).andReturn(emptyList()).anyTimes();
- EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
- EasyMock.expect(providersDir.deleteFile("local")).andReturn(true).once();
+
EasyMock.expect(providersDir.deleteFile("to-be-deleted")).andReturn(true).once();
EasyMock.replay(providersDir, descriptorsDir, db);
monitor.sync();
@@ -109,7 +112,8 @@ public class DbRemoteConfigurationMonitorServiceTest {
EasyMock.expect(providersDir.list()).andReturn(emptySet()).anyTimes();
EasyMock.expect(descriptorsDir.list()).andReturn(new
HashSet<>(asList("local"))).anyTimes();
- EasyMock.expect(db.selectDescriptors()).andReturn(emptyList()).anyTimes();
+ EasyMock.expect(db.selectDescriptors()).andReturn(Arrays.asList(
+ new RemoteConfig("local", "any", NOW, true))).anyTimes();
EasyMock.expect(db.selectProviders()).andReturn(emptyList()).anyTimes();
EasyMock.expect(descriptorsDir.deleteFile("local")).andReturn(true).once();
@@ -136,7 +140,8 @@ public class DbRemoteConfigurationMonitorServiceTest {
EasyMock.expect(db.selectDescriptors()).andReturn(asList(
new RemoteConfig("desc1", "desc1-same-content", NOW),
- new RemoteConfig("desc2", "desc2-remote-content", NOW)
+ new RemoteConfig("desc2", "desc2-remote-content", NOW),
+ new RemoteConfig("desc3-to-be-deleted", "any", NOW, true)
)).anyTimes();
// Expectations
diff --git
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabaseTest.java
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabaseTest.java
index baa9aecce..34c558482 100644
---
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabaseTest.java
+++
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/monitor/db/RemoteConfigDatabaseTest.java
@@ -17,7 +17,9 @@
package org.apache.knox.gateway.topology.monitor.db;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.Statement;
@@ -128,14 +130,69 @@ public class RemoteConfigDatabaseTest {
db.deleteProvider("provider_2");
- assertEquals(1, db.selectProviders().size());
+ assertEquals(2, db.selectProviders().size());
+ for (RemoteConfig provider : db.selectProviders()) {
+ if (provider.getName().equals("provider_1")) {
+ assertFalse(provider.isDeleted());
+ } else if (provider.getName().equals("provider_2")) {
+ assertTrue(provider.isDeleted());
+ } else {
+ fail("unexpected name: " + provider.getName());
+ }
+ }
+
+ db.deleteDescriptor("descriptor_1");
+
+ assertEquals(2, db.selectProviders().size());
assertEquals(1, db.selectDescriptors().size());
+ assertTrue(db.selectDescriptors().get(0).isDeleted());
+ }
+
+ @Test
+ public void testPutBackDeletedOne() {
+ db.putProvider("provider_1", "test provider1 content");
+ db.putDescriptor("descriptor_1", "test descriptor content");
+
+ db.deleteProvider("provider_1");
db.deleteDescriptor("descriptor_1");
assertEquals(1, db.selectProviders().size());
- assertEquals(0, db.selectDescriptors().size());
+ assertTrue(db.selectProviders().get(0).isDeleted());
+ assertEquals(1, db.selectDescriptors().size());
+ assertTrue(db.selectDescriptors().get(0).isDeleted());
+
+ db.putProvider("provider_1", "test provider1 new content");
+ db.putDescriptor("descriptor_1", "test descriptor new content");
+
+ assertEquals(1, db.selectProviders().size());
+ assertEquals(1, db.selectDescriptors().size());
+
+ assertEquals(1, db.selectProviders().size());
+ assertFalse(db.selectProviders().get(0).isDeleted());
+ assertEquals(1, db.selectDescriptors().size());
+ assertFalse(db.selectDescriptors().get(0).isDeleted());
+
+ assertEquals("test provider1 new content",
db.selectProviders().get(0).getContent());
+ assertEquals("test descriptor new content",
db.selectDescriptors().get(0).getContent());
+ }
+
+ @Test
+ public void testPhysicalDelete() {
+ db.putProvider("provider_1", "any");
+ db.putDescriptor("descriptor_1", "any");
- assertEquals("provider_1", db.selectProviders().get(0).getName());
+ assertEquals(0, db.cleanTables(0));
+
+ assertEquals(1, db.selectProviders().size());
+ assertEquals(1, db.selectDescriptors().size());
+
+ db.deleteDescriptor("descriptor_1");
+ db.deleteProvider("provider_1");
+
+ assertEquals(2, db.cleanTables(0));
+
+ assertEquals(0, db.selectProviders().size());
+ assertEquals(0, db.selectDescriptors().size());
}
}
\ No newline at end of file
diff --git
a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
index f9b87296b..fb70ded11 100644
---
a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
+++
b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java
@@ -1015,6 +1015,11 @@ public class GatewayTestConfig extends Configuration
implements GatewayConfig {
return 30;
}
+ @Override
+ public int getDbRemoteConfigMonitorCleanUpInterval() {
+ return 1;
+ }
+
@Override
public long getConcurrentSessionVerifierExpiredTokensCleaningPeriod() {
return 0;
diff --git
a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
index 423b2e6a6..ced53f013 100644
---
a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
+++
b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java
@@ -862,6 +862,8 @@ public interface GatewayConfig {
long getDbRemoteConfigMonitorPollingInterval();
+ int getDbRemoteConfigMonitorCleanUpInterval();
+
long getConcurrentSessionVerifierExpiredTokensCleaningPeriod();
/**
