nastra commented on code in PR #14398:
URL: https://github.com/apache/iceberg/pull/14398#discussion_r2697695851
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java:
##########
@@ -3475,6 +3535,497 @@ public void
testLoadTableWithMissingMetadataFile(@TempDir Path tempDir) {
.hasMessageContaining("No in-memory file found for location: " +
metadataFileLocation);
}
+ @Test
+ public void testInvalidTableCacheParameters() {
+ RESTCatalog catalog = new RESTCatalog(config -> new
RESTCatalogAdapter(backendCatalog));
+
+ assertThatThrownBy(
+ () ->
+ catalog.initialize(
+ "test",
Map.of(RESTCatalogProperties.TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, "0")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid expire after write: zero or negative");
+
+ assertThatThrownBy(
+ () ->
+ catalog.initialize(
+ "test",
Map.of(RESTCatalogProperties.TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, "-1")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid expire after write: zero or negative");
+
+ assertThatThrownBy(
+ () ->
+ catalog.initialize(
+ "test",
Map.of(RESTCatalogProperties.TABLE_CACHE_MAX_ENTRIES, "-1")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid max entries: negative");
+ }
+
+ @Test
+ public void testFreshnessAwareLoading() {
+ catalog().createNamespace(TABLE.namespace());
+
+ catalog().createTable(TABLE, SCHEMA);
+
+ Cache<SessionIdTableId, TableSupplierWithETag> tableCache =
+ restCatalog.sessionCatalog().tableCache().tableCache();
+ assertThat(tableCache.estimatedSize()).isZero();
+
+ expectFullTableLoadForLoadTable(TABLE, adapterForRESTServer);
+
+ BaseTable tableAfterFirstLoad = (BaseTable) catalog().loadTable(TABLE);
+
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ expectNotModifiedResponseForLoadTable(TABLE, adapterForRESTServer);
+
+ BaseTable tableAfterSecondLoad = (BaseTable) catalog().loadTable(TABLE);
+
+ assertThat(tableAfterFirstLoad).isNotEqualTo(tableAfterSecondLoad);
+ assertThat(tableAfterFirstLoad.operations().current().location())
+ .isEqualTo(tableAfterSecondLoad.operations().current().location());
+ assertThat(
+ tableCache
+ .asMap()
+ .get(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE))
+ .tableSupplier()
+ .get()
+ .operations()
+ .current()
+ .metadataFileLocation())
+
.isEqualTo(tableAfterFirstLoad.operations().current().metadataFileLocation());
+
+ Mockito.verify(adapterForRESTServer, times(2))
+ .execute(reqMatcher(HTTPMethod.GET, RESOURCE_PATHS.table(TABLE)),
any(), any(), any());
+ }
+
+ @Test
+ public void testFreshnessAwareLoadingMetadataTables() {
+ catalog().createNamespace(TABLE.namespace());
+
+ catalog().createTable(TABLE, SCHEMA);
+
+ Cache<SessionIdTableId, TableSupplierWithETag> tableCache =
+ restCatalog.sessionCatalog().tableCache().tableCache();
+ assertThat(tableCache.estimatedSize()).isZero();
+
+ BaseTable table = (BaseTable) catalog().loadTable(TABLE);
+
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ TableIdentifier metadataTableIdentifier =
+ TableIdentifier.of(TABLE.namespace().toString(), TABLE.name(),
"partitions");
+
+ BaseMetadataTable metadataTable =
+ (BaseMetadataTable) catalog().loadTable(metadataTableIdentifier);
+
+ assertThat(tableCache.stats().hitCount()).isEqualTo(1);
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ assertThat(table).isNotEqualTo(metadataTable.table());
+ assertThat(table.operations().current().metadataFileLocation())
+
.isEqualTo(metadataTable.table().operations().current().metadataFileLocation());
+
+ ResourcePaths paths =
+ ResourcePaths.forCatalogProperties(
+ ImmutableMap.of(RESTCatalogProperties.NAMESPACE_SEPARATOR, "%2E"));
+
+ Mockito.verify(adapterForRESTServer, times(2))
+ .execute(reqMatcher(HTTPMethod.GET, paths.table(TABLE)), any(), any(),
any());
+
+ Mockito.verify(adapterForRESTServer)
+ .execute(
+ reqMatcher(HTTPMethod.GET, paths.table(metadataTableIdentifier)),
any(), any(), any());
+ }
+
+ @Test
+ public void testRenameTableInvalidatesTable() {
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ (catalog) ->
+ catalog.renameTable(TABLE, TableIdentifier.of(TABLE.namespace(),
"other_table")),
+ 0);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDropTableInvalidatesTable(boolean purge) {
+ runTableInvalidationTest(
+ restCatalog, adapterForRESTServer, (catalog) ->
catalog.dropTable(TABLE, purge), 0);
+ }
+
+ @Test
+ public void testTableExistViaHeadRequestInvalidatesTable() {
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ ((catalog) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // The main catalog still has the table in cache
+
assertThat(catalog.sessionCatalog().tableCache().tableCache().asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ catalog.tableExists(TABLE);
+ }),
+ 0);
+ }
+
+ @Test
+ public void testTableExistViaGetRequestInvalidatesTable() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+
+ // Configure REST server to answer tableExists query via GET
+ Mockito.doAnswer(
+ invocation ->
+ ConfigResponse.builder()
+ .withEndpoints(
+ ImmutableList.of(
+ Endpoint.V1_LOAD_TABLE,
+ Endpoint.V1_CREATE_NAMESPACE,
+ Endpoint.V1_CREATE_TABLE))
+ .build())
+ .when(adapter)
+ .execute(
+ reqMatcher(HTTPMethod.GET, ResourcePaths.config()),
+ eq(ConfigResponse.class),
+ any(),
+ any());
+
+ RESTCatalog catalog = new RESTCatalog(DEFAULT_SESSION_CONTEXT, config ->
adapter);
+ catalog.initialize(
+ "catalog",
+ ImmutableMap.of(
+ CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
+
+ runTableInvalidationTest(
+ catalog,
+ adapter,
+ (cat) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // The main catalog still has the table in cache
+ assertThat(cat.sessionCatalog().tableCache().tableCache().asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ cat.tableExists(TABLE);
+ },
+ 1);
+ }
+
+ @Test
+ public void testLoadTableInvalidatesCache() {
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ (catalog) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // The main catalog still has the table in cache
+
assertThat(catalog.sessionCatalog().tableCache().tableCache().asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ assertThatThrownBy(() -> catalog.loadTable(TABLE))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessage("Table does not exist: %s", TABLE);
+ },
+ 1);
+ }
+
+ @Test
+ public void testLoadTableWithMetadataTableNameInvalidatesCache() {
+ TableIdentifier metadataTableIdentifier =
+ TableIdentifier.of(TABLE.namespace().toString(), TABLE.name(),
"partitions");
+
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ (catalog) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // The main catalog still has the table in cache
+
assertThat(catalog.sessionCatalog().tableCache().tableCache().asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ assertThatThrownBy(() -> catalog.loadTable(metadataTableIdentifier))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessage("Table does not exist: %s", TABLE);
+ },
+ 1);
+
+ ResourcePaths paths =
+ ResourcePaths.forCatalogProperties(
+ ImmutableMap.of(RESTCatalogProperties.NAMESPACE_SEPARATOR, "%2E"));
+
+ Mockito.verify(adapterForRESTServer)
+ .execute(
+ reqMatcher(HTTPMethod.GET, paths.table(metadataTableIdentifier)),
any(), any(), any());
+ }
+
+ private void runTableInvalidationTest(
+ RESTCatalog catalog,
+ RESTCatalogAdapter adapterToVerify,
+ Consumer<RESTCatalog> action,
+ int loadTableCountFromAction) {
+ catalog.createNamespace(TABLE.namespace());
+
+ catalog.createTable(TABLE, SCHEMA);
+
+ BaseTable originalTable = (BaseTable) catalog.loadTable(TABLE);
+
+ Cache<SessionIdTableId, TableSupplierWithETag> tableCache =
+ catalog.sessionCatalog().tableCache().tableCache();
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ action.accept(catalog);
+
+ // Check that 'action' invalidates cache
+ assertThat(tableCache.estimatedSize()).isZero();
+
+ assertThatThrownBy(() -> catalog.loadTable(TABLE))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessageContaining("Table does not exist: %s", TABLE);
+
+ catalog.createTable(TABLE, SCHEMA);
+
+ expectFullTableLoadForLoadTable(TABLE, adapterToVerify);
+
+ BaseTable newTableWithSameName = (BaseTable) catalog.loadTable(TABLE);
+
+
assertThat(tableCache.stats().hitCount()).isEqualTo(loadTableCountFromAction);
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ assertThat(newTableWithSameName).isNotEqualTo(originalTable);
+
assertThat(newTableWithSameName.operations().current().metadataFileLocation())
+
.isNotEqualTo(originalTable.operations().current().metadataFileLocation());
+
+ Mockito.verify(adapterToVerify, times(3 + loadTableCountFromAction))
+ .execute(reqMatcher(HTTPMethod.GET, RESOURCE_PATHS.table(TABLE)),
any(), any(), any());
+ }
+
+ @Test
+ public void testTableCacheWithMultiSessions() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+
+ RESTSessionCatalog sessionCatalog = new RESTSessionCatalog(config ->
adapter, null);
+ sessionCatalog.initialize("test_session_catalog", Map.of());
+
+ SessionCatalog.SessionContext otherSessionContext =
+ new SessionCatalog.SessionContext(
+ "session_id_2", "user", ImmutableMap.of("credential",
"user:12345"), ImmutableMap.of());
+
+ sessionCatalog.createNamespace(DEFAULT_SESSION_CONTEXT, TABLE.namespace());
+
+ sessionCatalog.buildTable(DEFAULT_SESSION_CONTEXT, TABLE, SCHEMA).create();
+
+ expectFullTableLoadForLoadTable(TABLE, adapter);
+
+ sessionCatalog.loadTable(DEFAULT_SESSION_CONTEXT, TABLE);
+
+ Cache<SessionIdTableId, TableSupplierWithETag> tableCache =
+ sessionCatalog.tableCache().tableCache();
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(),
TABLE));
+
+ expectFullTableLoadForLoadTable(TABLE, adapter);
+
+ sessionCatalog.loadTable(otherSessionContext, TABLE);
+
+ assertThat(tableCache.asMap())
+ .containsOnlyKeys(
+ SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(), TABLE),
+ SessionIdTableId.of(otherSessionContext.sessionId(), TABLE));
+ }
+
+ @Test
+ public void test304NotModifiedResponseWithEmptyTableCache() {
+ Mockito.doAnswer(invocation -> null)
+ .when(adapterForRESTServer)
+ .execute(
+ reqMatcher(HTTPMethod.GET, RESOURCE_PATHS.table(TABLE)),
+ eq(LoadTableResponse.class),
+ any(),
+ any());
+
+ catalog().createNamespace(TABLE.namespace());
+
+ catalog().createTable(TABLE, SCHEMA);
+
+ catalog().invalidateTable(TABLE);
+
+ // Table is not in the cache and null LoadTableResponse is received
+ assertThatThrownBy(() -> catalog().loadTable(TABLE))
+ .isInstanceOf(RESTException.class)
+ .hasMessage(
+ "Invalid (NOT_MODIFIED) response for request: method=%s, path=%s",
+ HTTPMethod.GET, RESOURCE_PATHS.table(TABLE));
+ }
+
+ @Test
+ public void testTableCacheNotUpdatedWithoutETag() {
+ RESTCatalogAdapter adapter =
+ Mockito.spy(
+ new RESTCatalogAdapter(backendCatalog) {
+ @Override
+ public <T extends RESTResponse> T execute(
+ HTTPRequest request,
+ Class<T> responseType,
+ Consumer<ErrorResponse> errorHandler,
+ Consumer<Map<String, String>> responseHeaders) {
+ // Wrap the original responseHeaders to not accept ETag.
+ Consumer<Map<String, String>> noETagConsumer =
+ headers -> {
+ if (!headers.containsKey(HttpHeaders.ETAG)) {
+ responseHeaders.accept(headers);
+ }
+ };
+ return super.execute(request, responseType, errorHandler,
noETagConsumer);
+ }
+ });
+
+ RESTCatalog catalog = new RESTCatalog(DEFAULT_SESSION_CONTEXT, config ->
adapter);
+ catalog.initialize(
+ "catalog",
+ ImmutableMap.of(
+ CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
+
+ catalog.createNamespace(TABLE.namespace());
+
+ catalog.createTable(TABLE, SCHEMA);
+
+ catalog.loadTable(TABLE);
+
+
assertThat(catalog.sessionCatalog().tableCache().tableCache().estimatedSize()).isZero();
+ }
+
+ @Test
+ public void testTableCacheIsDisabled() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+
+ RESTCatalog catalog = new RESTCatalog(DEFAULT_SESSION_CONTEXT, config ->
adapter);
+ catalog.initialize(
+ "catalog",
+ ImmutableMap.of(
+ CatalogProperties.FILE_IO_IMPL,
+ "org.apache.iceberg.inmemory.InMemoryFileIO",
+ RESTCatalogProperties.TABLE_CACHE_MAX_ENTRIES,
+ "0"));
+
+ catalog.createNamespace(TABLE.namespace());
+
+ catalog.createTable(TABLE, SCHEMA);
+
+
assertThat(catalog.sessionCatalog().tableCache().tableCache().estimatedSize()).isZero();
+
+ expectFullTableLoadForLoadTable(TABLE, adapter);
+
+ catalog.loadTable(TABLE);
+
+ catalog.sessionCatalog().tableCache().tableCache().cleanUp();
+
+
assertThat(catalog.sessionCatalog().tableCache().tableCache().estimatedSize()).isZero();
+ }
+
+ @Test
+ public void testFullTableLoadAfterExpiryFromCache() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+
+ FakeTicker ticker = new FakeTicker();
+
+ TestableRESTCatalog catalog =
+ new TestableRESTCatalog(DEFAULT_SESSION_CONTEXT, config -> adapter,
ticker);
+ catalog.initialize("catalog", Map.of());
+
+ catalog.createNamespace(TABLE.namespace());
+
+ catalog.createTable(TABLE, SCHEMA);
+
+ catalog.loadTable(TABLE);
+
+ Cache<SessionIdTableId, TableSupplierWithETag> tableCache =
+ catalog.sessionCatalog().tableCache().tableCache();
+ SessionIdTableId tableCacheKey =
+ SessionIdTableId.of(DEFAULT_SESSION_CONTEXT.sessionId(), TABLE);
+
+ assertThat(tableCache.asMap()).containsOnlyKeys(tableCacheKey);
+
assertThat(tableCache.policy().expireAfterWrite().get().ageOf(tableCacheKey))
+ .isPresent()
+ .get()
+ .isEqualTo(Duration.ZERO);
+
+ ticker.advance(HALF_OF_TABLE_EXPIRATION);
+
+ assertThat(tableCache.asMap()).containsOnlyKeys(tableCacheKey);
+
assertThat(tableCache.policy().expireAfterWrite().get().ageOf(tableCacheKey))
+ .isPresent()
+ .get()
+ .isEqualTo(HALF_OF_TABLE_EXPIRATION);
+
+ ticker.advance(HALF_OF_TABLE_EXPIRATION.plus(Duration.ofSeconds(10)));
+
+ assertThat(tableCache.asMap()).doesNotContainKey(tableCacheKey);
+
+ expectFullTableLoadForLoadTable(TABLE, adapter);
+
+ catalog.loadTable(TABLE);
+
+ assertThat(tableCache.stats().hitCount()).isEqualTo(0);
+ assertThat(tableCache.asMap()).containsOnlyKeys(tableCacheKey);
+
assertThat(tableCache.policy().expireAfterWrite().get().ageOf(tableCacheKey))
+ .isPresent()
+ .get()
+ .isEqualTo(Duration.ZERO);
+ }
+
+ @Test
+ public void testTableCacheAgeDoesNotRefreshesAfterAccess() {
+ FakeTicker ticker = new FakeTicker();
+
+ TestableRESTCatalog catalog =
+ new TestableRESTCatalog(
+ DEFAULT_SESSION_CONTEXT, config -> new
RESTCatalogAdapter(backendCatalog), ticker);
+ catalog.initialize("catalog", Map.of());
+
+ catalog.createNamespace(TABLE.namespace());
+
+ catalog.createTable(TABLE, SCHEMA);
+
+ catalog.loadTable(TABLE);
+
Review Comment:
I would remove all of thew empty new lines as I don't see any value having
them. It's fine to have them in between assertions to have a visual separator,
but creating the namespace/table and loading the table feels like the same
logical step, so this can be visually grouped together
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
