pvary commented on code in PR #14398:
URL: https://github.com/apache/iceberg/pull/14398#discussion_r2498228791
##########
core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java:
##########
@@ -3066,6 +3238,528 @@ public void testCommitStateUnknownNotReconciled() {
.satisfies(ex -> assertThat(((CommitStateUnknownException)
ex).getSuppressed()).isEmpty());
}
+ @Test
+ public void testInvalidTableCacheParameters() {
+ RESTCatalog catalog = new RESTCatalog(config -> new
RESTCatalogAdapter(backendCatalog));
+
+ assertThatThrownBy(
+ () ->
+ catalog.initialize(
+ "test",
Map.of(RESTCatalogProperties.TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, "0")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid expire after write: zero or negative");
+
+ assertThatThrownBy(
+ () ->
+ catalog.initialize(
+ "test",
Map.of(RESTCatalogProperties.TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, "-1")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid expire after write: zero or negative");
+
+ assertThatThrownBy(
+ () ->
+ catalog.initialize(
+ "test",
Map.of(RESTCatalogProperties.TABLE_CACHE_MAX_ENTRIES, "-1")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid max entries: negative");
+ }
+
+ @Test
+ public void testFreshnessAwareLoading() {
+ catalog().createNamespace(TABLE.namespace());
+
+ catalog().createTable(TABLE, SCHEMA);
+
+ Cache<SessionIDTableID, TableWithETag> tableCache =
restCatalog.sessionCatalog().tableCache();
+ assertThat(tableCache.estimatedSize()).isZero();
+
+ expectFullTableLoadForLoadTable(TABLE, adapterForRESTServer);
+
+ BaseTable tableAfterFirstLoad = (BaseTable) catalog().loadTable(TABLE);
+
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIDTableID.of(defaultSessionContext.sessionId(),
TABLE));
+
+ expectNotModifiedResponseForLoadTable(TABLE, adapterForRESTServer);
+
+ BaseTable tableAfterSecondLoad = (BaseTable) catalog().loadTable(TABLE);
+
+ assertThat(tableAfterFirstLoad).isNotEqualTo(tableAfterSecondLoad);
+ assertThat(tableAfterFirstLoad.operations().current().location())
+ .isEqualTo(tableAfterSecondLoad.operations().current().location());
+ assertThat(
+ tableCache
+ .asMap()
+ .get(SessionIDTableID.of(defaultSessionContext.sessionId(),
TABLE))
+ .table()
+ .operations()
+ .current()
+ .metadataFileLocation())
+
.isEqualTo(tableAfterFirstLoad.operations().current().metadataFileLocation());
+
+ Mockito.verify(adapterForRESTServer, times(2))
+ .execute(reqMatcher(HTTPMethod.GET, RESOURCE_PATHS.table(TABLE)),
any(), any(), any());
+ }
+
+ @Test
+ public void testFreshnessAwareLoadingMetadataTables() {
+ catalog().createNamespace(TABLE.namespace());
+
+ catalog().createTable(TABLE, SCHEMA);
+
+ Cache<SessionIDTableID, TableWithETag> tableCache =
restCatalog.sessionCatalog().tableCache();
+ assertThat(tableCache.estimatedSize()).isZero();
+
+ BaseTable table = (BaseTable) catalog().loadTable(TABLE);
+
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIDTableID.of(defaultSessionContext.sessionId(),
TABLE));
+
+ TableIdentifier metadataTableIdentifier =
+ TableIdentifier.of(TABLE.namespace().toString(), TABLE.name(),
"partitions");
+
+ BaseMetadataTable metadataTable =
+ (BaseMetadataTable) catalog().loadTable(metadataTableIdentifier);
+
+ assertThat(tableCache.stats().hitCount()).isEqualTo(1);
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIDTableID.of(defaultSessionContext.sessionId(),
TABLE));
+
+ assertThat(table).isNotEqualTo(metadataTable.table());
+ assertThat(table.operations().current().metadataFileLocation())
+
.isEqualTo(metadataTable.table().operations().current().metadataFileLocation());
+
+ Mockito.verify(adapterForRESTServer, times(2))
+ .execute(reqMatcher(HTTPMethod.GET, RESOURCE_PATHS.table(TABLE)),
any(), any(), any());
+
+ Mockito.verify(adapterForRESTServer)
+ .execute(
+ reqMatcher(HTTPMethod.GET,
RESOURCE_PATHS.table(metadataTableIdentifier)),
+ any(),
+ any(),
+ any());
+ }
+
+ @Test
+ public void testRenameTableInvalidatesTable() {
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ (catalog) ->
+ catalog.renameTable(TABLE, TableIdentifier.of(TABLE.namespace(),
"other_table")),
+ 0);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDropTableInvalidatesTable(boolean purge) {
+ runTableInvalidationTest(
+ restCatalog, adapterForRESTServer, (catalog) ->
catalog.dropTable(TABLE, purge), 0);
+ }
+
+ @Test
+ public void testTableExistViaHeadRequestInvalidatesTable() {
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ ((catalog) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // This catalog still has the table in cache
+ catalog.tableExists(TABLE);
+ }),
+ 0);
+ }
+
+ @Test
+ public void testTableExistViaGetRequestInvalidatesTable() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+
+ // Configure REST server to answer tableExists query via GET
+ Mockito.doAnswer(
+ invocation ->
+ ConfigResponse.builder()
+ .withEndpoints(
+ ImmutableList.of(
+ Endpoint.V1_LOAD_TABLE,
+ Endpoint.V1_CREATE_NAMESPACE,
+ Endpoint.V1_CREATE_TABLE))
+ .build())
+ .when(adapter)
+ .execute(
+ reqMatcher(HTTPMethod.GET, ResourcePaths.config()),
+ eq(ConfigResponse.class),
+ any(),
+ any());
+
+ RESTCatalog catalog = new RESTCatalog(defaultSessionContext, config ->
adapter);
+ catalog.initialize(
+ "catalog",
+ ImmutableMap.of(
+ CatalogProperties.FILE_IO_IMPL,
"org.apache.iceberg.inmemory.InMemoryFileIO"));
+
+ runTableInvalidationTest(
+ catalog,
+ adapter,
+ (cat) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // This catalog still has the table in cache
+ cat.tableExists(TABLE);
+ },
+ 1);
+ }
+
+ @Test
+ public void testLoadTableInvalidatesCache() {
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ (catalog) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // This catalog still has the table in cache
+ assertThatThrownBy(() -> catalog.loadTable(TABLE))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessage("Table does not exist: %s", TABLE);
+ },
+ 1);
+ }
+
+ @Test
+ public void testLoadTableWithMetadataTableNameInvalidatesCache() {
+ TableIdentifier metadataTableIdentifier =
+ TableIdentifier.of(TABLE.namespace().toString(), TABLE.name(),
"partitions");
+
+ runTableInvalidationTest(
+ restCatalog,
+ adapterForRESTServer,
+ (catalog) -> {
+ // Use a different catalog to drop the table
+ catalog(new RESTCatalogAdapter(backendCatalog)).dropTable(TABLE,
true);
+
+ // This catalog still has the table in cache
+ assertThatThrownBy(() -> catalog.loadTable(metadataTableIdentifier))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessage("Table does not exist: %s", TABLE);
+ },
+ 1);
+
+ Mockito.verify(adapterForRESTServer)
+ .execute(
+ reqMatcher(HTTPMethod.GET,
RESOURCE_PATHS.table(metadataTableIdentifier)),
+ any(),
+ any(),
+ any());
+ }
+
+ private void runTableInvalidationTest(
+ RESTCatalog catalog,
+ RESTCatalogAdapter adapterToVerify,
+ Consumer<RESTCatalog> action,
+ int loadTableCountFromAction) {
+ catalog.createNamespace(TABLE.namespace());
+
+ catalog.createTable(TABLE, SCHEMA);
+
+ BaseTable originalTable = (BaseTable) catalog.loadTable(TABLE);
+
+ Cache<SessionIDTableID, TableWithETag> tableCache =
catalog.sessionCatalog().tableCache();
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIDTableID.of(defaultSessionContext.sessionId(),
TABLE));
+
+ action.accept(catalog);
+
+ // Check that 'action' invalidates cache
+ assertThat(tableCache.estimatedSize()).isZero();
+
+ assertThatThrownBy(() -> catalog.loadTable(TABLE))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessageContaining("Table does not exist: %s", TABLE);
+
+ catalog.createTable(TABLE, SCHEMA);
+
+ expectFullTableLoadForLoadTable(TABLE, adapterToVerify);
+
+ BaseTable newTableWithSameName = (BaseTable) catalog.loadTable(TABLE);
+
+
assertThat(tableCache.stats().hitCount()).isEqualTo(loadTableCountFromAction);
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIDTableID.of(defaultSessionContext.sessionId(),
TABLE));
+
+ assertThat(newTableWithSameName).isNotEqualTo(originalTable);
+
assertThat(newTableWithSameName.operations().current().metadataFileLocation())
+
.isNotEqualTo(originalTable.operations().current().metadataFileLocation());
+
+ Mockito.verify(adapterToVerify, times(3 + loadTableCountFromAction))
+ .execute(reqMatcher(HTTPMethod.GET, RESOURCE_PATHS.table(TABLE)),
any(), any(), any());
+ }
+
+ @Test
+ public void testTableCacheWithMultiSessions() {
+ RESTCatalogAdapter adapter = Mockito.spy(new
RESTCatalogAdapter(backendCatalog));
+
+ RESTSessionCatalog sessionCatalog = new RESTSessionCatalog(config ->
adapter, null);
+ sessionCatalog.initialize("test_session_catalog", Map.of());
+
+ SessionCatalog.SessionContext otherSessionContext =
+ new SessionCatalog.SessionContext(
+ "session_id_2", "user", ImmutableMap.of("credential",
"user:12345"), ImmutableMap.of());
+
+ sessionCatalog.createNamespace(defaultSessionContext, TABLE.namespace());
+
+ sessionCatalog.buildTable(defaultSessionContext, TABLE, SCHEMA).create();
+
+ expectFullTableLoadForLoadTable(TABLE, adapter);
+
+ sessionCatalog.loadTable(defaultSessionContext, TABLE);
+
+ Cache<SessionIDTableID, TableWithETag> tableCache =
sessionCatalog.tableCache();
+ assertThat(tableCache.stats().hitCount()).isZero();
+ assertThat(tableCache.asMap())
+
.containsOnlyKeys(SessionIDTableID.of(defaultSessionContext.sessionId(),
TABLE));
+
+ expectFullTableLoadForLoadTable(TABLE, adapter);
+
+ sessionCatalog.loadTable(otherSessionContext, TABLE);
+
+ assertThat(tableCache.asMap())
+ .containsOnlyKeys(
+ SessionIDTableID.of(defaultSessionContext.sessionId(), TABLE),
+ SessionIDTableID.of(otherSessionContext.sessionId(), TABLE));
+ }
+
+ @Test
+ public void testTableCacheWithTableCommit() {
Review Comment:
Is this a requirement, or a side effect of being lazy, and not implementing
cache refresh everywhere?
If this is just a side effect, then we might not want to add test as later
generation might mistake this for a requirement
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
