collado-mike commented on code in PR #590: URL: https://github.com/apache/polaris/pull/590#discussion_r1904455549
########## integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java: ########## @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public class CatalogApi extends RestApi { + CatalogApi(PolarisApiClient client, AuthToken token, URI uri) { + super(client, token, uri); + } + + public AuthToken obtainToken(PrincipalWithCredentials principal) { + try (Response response = + request("v1/oauth/tokens") + .post( + Entity.form( + new MultivaluedHashMap<>( + Map.of( + "grant_type", + "client_credentials", + "scope", + "PRINCIPAL_ROLE:ALL", Review Comment: Can we make scope an optional argument (default to `PRINCIPAL_ROLE:ALL`) so tests can verify sub-scoped tokens? ########## dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java: ########## @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.polaris.service.dropwizard.PolarisApplication; +import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.Server; +import org.apache.polaris.service.it.ext.PolarisServerManager; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class DropwizardServerManager implements PolarisServerManager { + // referenced in polaris-server-integrationtest.yml + public static final String TEST_REALM = "POLARIS"; + public static final String SERVER_CONFIG_PATH = "polaris-server-integrationtest.yml"; + + @Override + public Server serverForContext(ExtensionContext context) { + try { + return new Holder(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static class Holder implements Server { + private final DropwizardAppExtension<PolarisApplicationConfig> ext; + private final Path logDir; + + public Holder() throws IOException { + logDir = Files.createTempDirectory("polaris-dw-"); + Path logFile = logDir.resolve("application.log").toAbsolutePath(); + + Map<String, String> config = new HashMap<>(); + // Bind to random port to support parallelism + config.put("server.applicationConnectors[0].port", "0"); + config.put("server.adminConnectors[0].port", "0"); + config.put("logging.appenders[1].type", "file"); + config.put("logging.appenders[1].currentLogFilename", logFile.toString()); + + ConfigOverride[] overrides = + config.entrySet().stream() + .map((e) -> ConfigOverride.config(e.getKey(), e.getValue())) + .toList() + .toArray(new ConfigOverride[0]); Review Comment: nit: I tend to prefer `Stream.toArray(ConfigOverride[]::new)` for this purpose ########## integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java: ########## @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogGrant; +import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CatalogRoles; +import org.apache.polaris.core.admin.model.Catalogs; +import org.apache.polaris.core.admin.model.CreatePrincipalRequest; +import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.GrantResources; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalRoles; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.Principals; +import org.apache.polaris.core.admin.model.UpdateCatalogRequest; + +public class ManagementApi extends RestApi { + ManagementApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + super(client, endpoints, authToken, uri); + } + + public PrincipalWithCredentials createPrincipalWithRole(String principalName, String roleName) { + PrincipalWithCredentials credentials = createPrincipal(principalName); + createPrincipalRole(roleName); + assignPrincipalRole(principalName, roleName); + return credentials; + } + + public PrincipalWithCredentials createPrincipal(String name) { + try (Response createPResponse = + request("v1/principals").post(Entity.json(new Principal(name)))) { + assertThat(createPResponse).returns(CREATED.getStatusCode(), Response::getStatus); + return createPResponse.readEntity(PrincipalWithCredentials.class); + } + } + + public PrincipalWithCredentials createPrincipal(CreatePrincipalRequest request) { + try (Response createPResponse = request("v1/principals").post(Entity.json(request))) { + assertThat(createPResponse).returns(CREATED.getStatusCode(), Response::getStatus); + return createPResponse.readEntity(PrincipalWithCredentials.class); + } + } + + public void createPrincipalRole(String name) { + createPrincipalRole(new PrincipalRole(name)); + } + + public void createPrincipalRole(PrincipalRole role) { + try (Response createPrResponse = request("v1/principal-roles").post(Entity.json(role))) { + assertThat(createPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void assignPrincipalRole(String principalName, String roleName) { + try (Response assignPrResponse = + request("v1/principals/{prince}/principal-roles", Map.of("prince", principalName)) + .put(Entity.json(new GrantPrincipalRoleRequest(new PrincipalRole(roleName))))) { + assertThat(assignPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void createCatalogRole(String catalogName, String catalogRoleName) { + try (Response response = + request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) + .post(Entity.json(new CatalogRole(catalogRoleName)))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .put(Entity.json(grant))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void grantCatalogRoleToPrincipalRole( + String principalRoleName, String catalogName, CatalogRole catalogRole) { + try (Response response = + request( + "v1/principal-roles/{role}/catalog-roles/{cat}", + Map.of("role", principalRoleName, "cat", catalogName)) + .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public GrantResources listGrants(String catalogName, String catalogRoleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .get()) { + assertThat(response).returns(OK.getStatusCode(), Response::getStatus); + return response.readEntity(GrantResources.class); + } + } + + public void createCatalog(String principalRoleName, Catalog catalog) { + createCatalog(catalog); + + // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS + String catalogRoleName = "custom-admin"; + createCatalogRole(catalog.getName(), catalogRoleName); Review Comment: Why create a custom admin role? The service_admin already has the default catalog admin role that was created when the Catalog was created. ########## dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java: ########## @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import org.apache.polaris.service.dropwizard.PolarisApplication; +import org.apache.polaris.service.dropwizard.auth.TokenUtils; +import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.it.env.AuthToken; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.Server; +import org.apache.polaris.service.it.ext.PolarisServerManager; + +public class DropwizardServerManager implements PolarisServerManager { + public static final String TEST_REALM = "POLARIS"; + + @Override + public String realm() { + return TEST_REALM; + } + + @Override + public Server startServer(Map<String, String> featureConfig) { + return new Holder(featureConfig); + } + + private static class Holder implements Server { + private final DropwizardAppExtension<PolarisApplicationConfig> ext; + private final AuthToken token; + + public Holder(Map<String, String> featureConfig) { + Map<String, String> config = new HashMap<>(); + featureConfig.forEach((k, v) -> config.put("featureConfiguration." + k, v)); + // Bind to random port to support parallelism + config.put("server.applicationConnectors[0].port", "0"); + config.put("server.adminConnectors[0].port", "0"); + + ConfigOverride[] overrides = + config.entrySet().stream() + .map((e) -> ConfigOverride.config(e.getKey(), e.getValue())) + .toList() + .toArray(new ConfigOverride[0]); + ext = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), + overrides); + + try { + ext.before(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + token = new AuthToken(getToken(), "root"); + } + + private String getToken() { + return TokenUtils.getTokenFromSecrets( + ext.client(), + baseUri().toString(), + adminCredentials().clientId(), + adminCredentials().clientSecret(), + TEST_REALM); + } + + @Override + public URI baseUri() { + return URI.create(String.format("http://localhost:%d";, ext.getLocalPort())); Review Comment: I like the proposal for remote tests to implement their own `PolarisServerManager`. Otherwise, the concern for one use case ends up cluttering the code for the other use case. ########## integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java: ########## @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import java.net.URI; + +public interface Server extends AutoCloseable { Review Comment: Can we add javadocs to public classes an interfaces? It's really helpful to see, at a glance, what classes purpose some of these classes are intended to serve. ########## integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java: ########## @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static org.apache.polaris.service.it.ext.PolarisServerManagerLoader.polarisServerManager; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import jakarta.ws.rs.client.Client; +import java.util.Random; +import org.apache.iceberg.rest.RESTSerializers; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public final class PolarisClient implements AutoCloseable { + private final PolarisApiEndpoints endpoints; + private final Client client; + // Use an alphanumeric ID for widest compatibility in HTTP and SQL. + // Use MAX_RADIX for shorter output. + private final String clientId = + Long.toString(Math.abs(new Random().nextLong()), Character.MAX_RADIX); + + private PolarisClient(PolarisApiEndpoints endpoints) { + this.endpoints = endpoints; + + this.client = polarisServerManager().createClient(); + } + + public static PolarisClient polarisClient(PolarisApiEndpoints endpoints) { + return new PolarisClient(endpoints); + } + + public static ObjectMapper buildObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(mapper); + return mapper; + } + + public String newEntityName(String hint) { + return polarisServerManager().transformEntityName(hint + "_" + clientId); + } + + public ManagementApi managementApi(String authToken) { + return new ManagementApi(client, endpoints, authToken, endpoints.managementApiEndpoint()); + } + + public ManagementApi managementApi(ClientCredentials credentials) { + return managementApi(obtainToken(credentials)); + } + + public ManagementApi managementApi(PrincipalWithCredentials principal) { + return managementApi(obtainToken(principal)); + } + + public CatalogApi catalogApi(PrincipalWithCredentials principal) { + return new CatalogApi( + client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint()); + } + + public CatalogApi catalogApi(ClientCredentials credentials) { + return new CatalogApi( + client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint()); + } + + public String obtainToken(PrincipalWithCredentials principal) { + return obtainToken( + new ClientCredentials( + principal.getCredentials().getClientId(), + principal.getCredentials().getClientSecret(), + "dummy-principal")); Review Comment: use the principal name? ########## dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java: ########## @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.polaris.service.dropwizard.PolarisApplication; +import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.Server; +import org.apache.polaris.service.it.ext.PolarisServerManager; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class DropwizardServerManager implements PolarisServerManager { + // referenced in polaris-server-integrationtest.yml + public static final String TEST_REALM = "POLARIS"; + public static final String SERVER_CONFIG_PATH = "polaris-server-integrationtest.yml"; + + @Override + public Server serverForContext(ExtensionContext context) { + try { + return new Holder(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static class Holder implements Server { + private final DropwizardAppExtension<PolarisApplicationConfig> ext; + private final Path logDir; + + public Holder() throws IOException { + logDir = Files.createTempDirectory("polaris-dw-"); + Path logFile = logDir.resolve("application.log").toAbsolutePath(); + + Map<String, String> config = new HashMap<>(); + // Bind to random port to support parallelism + config.put("server.applicationConnectors[0].port", "0"); + config.put("server.adminConnectors[0].port", "0"); + config.put("logging.appenders[1].type", "file"); + config.put("logging.appenders[1].currentLogFilename", logFile.toString()); + + ConfigOverride[] overrides = + config.entrySet().stream() + .map((e) -> ConfigOverride.config(e.getKey(), e.getValue())) + .toList() + .toArray(new ConfigOverride[0]); + ext = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath(SERVER_CONFIG_PATH), + overrides); + + try { + ext.before(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertThat(logFile) + .exists() + .content() + .hasSizeGreaterThan(0) + .doesNotContain("ERROR", "FATAL") + .contains("PolarisApplication: Server started successfully"); + } + + @Override + public String realmId() { + return TEST_REALM; Review Comment: Most of the tests today create a distinct realm per test so that there's no overlap for roles or catalogs, etc. It also means that a test that fails to clean up doesn't leave clutter that effects other tests. ########## integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java: ########## @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import java.net.URI; + +public interface Server extends AutoCloseable { + URI baseUri(); + + AuthToken adminToken(); + + ClientCredentials adminCredentials(); Review Comment: The intention of the `SnowmanExtension` was simply to have a non-root user easily available so that all the tests didn't just run everything as root, thus missing issues that would be faced by non-root users. If `snowman` is just as easy to access without the extension, that's fine with me. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
