[pinot] branch master updated: enable readonly access to controller UI for users without table restrictions (#10329)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3772b55dc4 enable readonly access to controller UI for users without
table restrictions (#10329)
3772b55dc4 is described below
commit 3772b55dc4c35673762a182b2ee650469560aa97
Author: Alexander Pucher
AuthorDate: Fri Feb 24 19:15:37 2023 -0800
enable readonly access to controller UI for users without table
restrictions (#10329)
---
.../api/resources/PinotControllerAuthResource.java | 3 ++-
.../main/java/org/apache/pinot/tools/AuthQuickstart.java | 16 ++--
2 files changed, 12 insertions(+), 7 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
index 77b77479ca..52e518cfed 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
@@ -28,6 +28,7 @@ import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
import javax.inject.Inject;
+import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
@@ -72,7 +73,7 @@ public class PinotControllerAuthResource {
@ApiResponse(code = 500, message = "Verification error")
})
public boolean verify(@ApiParam(value = "Table name without type")
@QueryParam("tableName") String tableName,
- @ApiParam(value = "API access type") @QueryParam("accessType")
AccessType accessType,
+ @ApiParam(value = "API access type") @DefaultValue("READ")
@QueryParam("accessType") AccessType accessType,
@ApiParam(value = "Endpoint URL") @QueryParam("endpointUrl") String
endpointUrl) {
AccessControl accessControl = _accessControlFactory.create();
return accessControl.hasAccess(tableName, accessType, _httpHeaders,
endpointUrl);
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
index 264219..e4eaa89715 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
@@ -47,19 +47,23 @@ public class AuthQuickstart extends Quickstart {
properties.put("pinot.controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
properties.put("controller.admin.access.control.factory.class",
"org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory");
-properties.put("controller.admin.access.control.principals", "admin,
user");
+properties.put("controller.admin.access.control.principals", "admin, user,
service, tableonly");
properties.put("controller.admin.access.control.principals.admin.password",
"verysecret");
+
properties.put("controller.admin.access.control.principals.service.password",
"verysecrettoo");
properties.put("controller.admin.access.control.principals.user.password",
"secret");
-properties.put("controller.admin.access.control.principals.user.tables",
"baseballStats");
-
properties.put("controller.admin.access.control.principals.user.permissions",
"read");
+
properties.put("controller.admin.access.control.principals.user.permissions",
"READ");
+
properties.put("controller.admin.access.control.principals.tableonly.password",
"secrettoo");
+
properties.put("controller.admin.access.control.principals.tableonly.permissions",
"READ");
+
properties.put("controller.admin.access.control.principals.tableonly.tables",
"baseballStats");
// broker
properties.put("pinot.broker.access.control.class",
"org.apache.pinot.broker.broker.BasicAuthAccessControlFactory");
-properties.put("pinot.broker.access.control.principals", "admin, user");
+properties.put("pinot.broker.access.control.principals", "admin, user,
service, tableonly");
properties.put("pinot.broker.access.control.principals.admin.password",
"verysecret");
+properties.put("pinot.broker.access.control.principals.service.password",
"verysecrettoo");
[pinot] branch auth-enable-readonly-access-controller-ui updated (95b73d30fc -> fcf1114d1b)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch auth-enable-readonly-access-controller-ui in repository https://gitbox.apache.org/repos/asf/pinot.git from 95b73d30fc enable readonly access to controller UI for users without table restrictions add fcf1114d1b use @DefaulValue annotation instead No new revisions were added by this update. Summary of changes: .../controller/api/resources/PinotControllerAuthResource.java | 8 ++-- 1 file changed, 2 insertions(+), 6 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: enable readonly access to controller UI for users without table restrictions
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch auth-enable-readonly-access-controller-ui
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 95b73d30fcd3cc15aa7f9a1a190eb962b243c999
Author: Alexander Pucher
AuthorDate: Thu Feb 23 19:59:47 2023 -0800
enable readonly access to controller UI for users without table restrictions
---
.../api/resources/PinotControllerAuthResource.java | 5 +
.../main/java/org/apache/pinot/tools/AuthQuickstart.java | 16 ++--
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
index 77b77479ca..6288b17be6 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerAuthResource.java
@@ -74,6 +74,11 @@ public class PinotControllerAuthResource {
public boolean verify(@ApiParam(value = "Table name without type")
@QueryParam("tableName") String tableName,
@ApiParam(value = "API access type") @QueryParam("accessType")
AccessType accessType,
@ApiParam(value = "Endpoint URL") @QueryParam("endpointUrl") String
endpointUrl) {
+
+if (accessType == null) {
+ accessType = AccessType.READ;
+}
+
AccessControl accessControl = _accessControlFactory.create();
return accessControl.hasAccess(tableName, accessType, _httpHeaders,
endpointUrl);
}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
index 264219..e4eaa89715 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
@@ -47,19 +47,23 @@ public class AuthQuickstart extends Quickstart {
properties.put("pinot.controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
properties.put("controller.admin.access.control.factory.class",
"org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory");
-properties.put("controller.admin.access.control.principals", "admin,
user");
+properties.put("controller.admin.access.control.principals", "admin, user,
service, tableonly");
properties.put("controller.admin.access.control.principals.admin.password",
"verysecret");
+
properties.put("controller.admin.access.control.principals.service.password",
"verysecrettoo");
properties.put("controller.admin.access.control.principals.user.password",
"secret");
-properties.put("controller.admin.access.control.principals.user.tables",
"baseballStats");
-
properties.put("controller.admin.access.control.principals.user.permissions",
"read");
+
properties.put("controller.admin.access.control.principals.user.permissions",
"READ");
+
properties.put("controller.admin.access.control.principals.tableonly.password",
"secrettoo");
+
properties.put("controller.admin.access.control.principals.tableonly.permissions",
"READ");
+
properties.put("controller.admin.access.control.principals.tableonly.tables",
"baseballStats");
// broker
properties.put("pinot.broker.access.control.class",
"org.apache.pinot.broker.broker.BasicAuthAccessControlFactory");
-properties.put("pinot.broker.access.control.principals", "admin, user");
+properties.put("pinot.broker.access.control.principals", "admin, user,
service, tableonly");
properties.put("pinot.broker.access.control.principals.admin.password",
"verysecret");
+properties.put("pinot.broker.access.control.principals.service.password",
"verysecrettoo");
properties.put("pinot.broker.access.control.principals.user.password",
"secret");
-properties.put("pinot.broker.access.control.principals.user.tables",
"baseballStats");
-properties.put("pinot.broker.access.control.principals.user.permissions",
"read");
+
properties.put("pinot.broker.access.control.principals.tableonly.password",
"secrettoo");
+properties.put("pinot.broker.access.control.principals.tableonly.tables",
"baseballStats");
// server
properties.put("pinot.server.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch auth-enable-readonly-access-controller-ui created (now 95b73d30fc)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch auth-enable-readonly-access-controller-ui in repository https://gitbox.apache.org/repos/asf/pinot.git at 95b73d30fc enable readonly access to controller UI for users without table restrictions This branch includes the following new commits: new 95b73d30fc enable readonly access to controller UI for users without table restrictions The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: Add back auth token for UploadSegmentCommand (#9960)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0ca5fcb472 Add back auth token for UploadSegmentCommand (#9960)
0ca5fcb472 is described below
commit 0ca5fcb4727184545fc3913b4a7e7d568fa0985e
Author: Tim Santos
AuthorDate: Sat Dec 10 23:28:44 2022 -0800
Add back auth token for UploadSegmentCommand (#9960)
---
.../org/apache/pinot/tools/admin/command/UploadSegmentCommand.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/UploadSegmentCommand.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/UploadSegmentCommand.java
index c721fc3e01..1e610b6f18 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/UploadSegmentCommand.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/UploadSegmentCommand.java
@@ -186,7 +186,8 @@ public class UploadSegmentCommand extends
AbstractBaseAdminCommand implements Co
}
LOGGER.info("Uploading segment tar file: {}", segmentTarFile);
-List headerList = makeAuthHeaders(_authProvider);
+List headerList =
+makeAuthHeaders(makeAuthProvider(_authProvider, _authTokenUrl,
_authToken, _user, _password));
FileInputStream fileInputStream = new FileInputStream(segmentTarFile);
fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI,
segmentTarFile.getName(),
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated (38ac70a9c1 -> f3eb5dabad)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
from 38ac70a9c1 [Upsert] persist validDocsIndex snapshot for Pinot upsert
optimization (#9062)
add f3eb5dabad broker filter (#9391)
No new revisions were added by this update.
Summary of changes:
.../pinot/broker/api/HttpRequesterIdentity.java| 12 +++
.../broker/api/resources/PinotClientRequest.java | 3 +
.../pinot/broker/broker/AuthenticationFilter.java | 93 ++
.../broker/broker/BrokerAdminApiApplication.java | 4 +-
.../broker/broker/helix/BaseBrokerStarter.java | 2 +-
.../api/access/AuthenticationFilter.java | 1 +
.../api/resources/PinotQueryResource.java | 2 +-
.../api/resources/PinotSchemaRestletResource.java | 2 +-
.../api/resources/PinotTableRestletResource.java | 2 +-
.../api/resources/TableConfigsRestletResource.java | 2 +-
.../pinot/core/auth}/ManualAuthorization.java | 2 +-
11 files changed, 118 insertions(+), 7 deletions(-)
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/broker/AuthenticationFilter.java
rename {pinot-controller/src/main/java/org/apache/pinot/controller/api/access
=>
pinot-core/src/main/java/org/apache/pinot/core/auth}/ManualAuthorization.java
(96%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: Disable logging for interrupted exceptions in kinesis (#9405)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c9587df919 Disable logging for interrupted exceptions in kinesis
(#9405)
c9587df919 is described below
commit c9587df919516c43e4b0040e53dcd3c83843031a
Author: Kartik Khare
AuthorDate: Fri Sep 16 02:13:50 2022 +0530
Disable logging for interrupted exceptions in kinesis (#9405)
---
.../java/org/apache/pinot/plugin/stream/kinesis/KinesisConsumer.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/src/main/java/org/apache/pinot/plugin/stream/kinesis/KinesisConsumer.java
b/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/src/main/java/org/apache/pinot/plugin/stream/kinesis/KinesisConsumer.java
index ccc9b7ee66..3fad3a39a6 100644
---
a/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/src/main/java/org/apache/pinot/plugin/stream/kinesis/KinesisConsumer.java
+++
b/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/src/main/java/org/apache/pinot/plugin/stream/kinesis/KinesisConsumer.java
@@ -175,7 +175,9 @@ public class KinesisConsumer extends
KinesisConnectionHandler implements Partiti
debugOrLogWarning("Encountered unknown unrecoverable AWS exception", e);
throw new RuntimeException(e);
} catch (AbortedException e) {
- debugOrLogWarning("Task aborted due to exception", e);
+ if (!(e.getCause() instanceof InterruptedException)) {
+debugOrLogWarning("Task aborted due to exception", e);
+ }
return handleException(kinesisStartCheckpoint, recordList);
} catch (Throwable e) {
// non transient errors
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: add @ManualAuthorization annotation for non-standard endpoints (#9252)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f6e26c2350 add @ManualAuthorization annotation for non-standard
endpoints (#9252)
f6e26c2350 is described below
commit f6e26c2350d7b997fa6d4bd690bc9c9058875309
Author: Alexander Pucher
AuthorDate: Wed Aug 24 21:56:13 2022 -0700
add @ManualAuthorization annotation for non-standard endpoints (#9252)
This PR adds a new annotation @ManualAuthorization for REST endpoints,
which allows developers to skip the default authorization and deserialize
payloads before manually invoking authorization, e.g. via
AccessControlUtils.validatePermissions(). This annotation comes with obvious
risks and should be used sparingly, as it enables requests to bypass most of
the AuthFilter.
---
.../pinot/controller/api/access/AccessControl.java | 16 +++---
.../controller/api/access/AccessControlUtils.java | 56 ++--
.../api/access/AuthenticationFilter.java | 8 ++-
.../controller/api/access/ManualAuthorization.java | 36 +
.../access/ZkBasicAuthAccessControlFactory.java| 8 ---
.../PinotAccessControlUserRestletResource.java | 33 ++--
.../api/resources/PinotControllerAuthResource.java | 6 ---
.../api/resources/PinotQueryResource.java | 16 +++---
.../api/resources/PinotSchemaRestletResource.java | 18 +--
.../PinotSegmentUploadDownloadRestletResource.java | 2 +-
.../api/resources/PinotTableRestletResource.java | 25 -
.../api/resources/TableConfigsRestletResource.java | 31 ++-
.../pinot/controller/api/AccessControlTest.java| 60 --
.../pinot/controller/helix/ControllerTest.java | 5 +-
.../pinot/spi/utils/builder/TableNameBuilder.java | 3 ++
15 files changed, 156 insertions(+), 167 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControl.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControl.java
index 3e19a9a45c..1b7c61993a 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControl.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControl.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.controller.api.access;
+import javax.annotation.Nullable;
import javax.ws.rs.core.HttpHeaders;
import org.apache.pinot.spi.annotations.InterfaceAudience;
import org.apache.pinot.spi.annotations.InterfaceStability;
@@ -40,7 +41,9 @@ public interface AccessControl {
* @return Whether the client has data access to the table
*/
@Deprecated
- boolean hasDataAccess(HttpHeaders httpHeaders, String tableName);
+ default boolean hasDataAccess(HttpHeaders httpHeaders, String tableName) {
+return hasAccess(tableName, AccessType.READ, httpHeaders, null);
+ }
/**
* Return whether the client has permission to the given table
@@ -51,7 +54,8 @@ public interface AccessControl {
* @param endpointUrl the request url for which this access control is called
* @return whether the client has permission
*/
- default boolean hasAccess(String tableName, AccessType accessType,
HttpHeaders httpHeaders, String endpointUrl) {
+ default boolean hasAccess(@Nullable String tableName, AccessType accessType,
HttpHeaders httpHeaders,
+ @Nullable String endpointUrl) {
return true;
}
@@ -63,12 +67,8 @@ public interface AccessControl {
* @param endpointUrl the request url for which this access control is called
* @return whether the client has permission
*/
- default boolean hasAccess(AccessType accessType, HttpHeaders httpHeaders,
String endpointUrl) {
-return true;
- }
-
- default boolean hasAccess(HttpHeaders httpHeaders) {
-return true;
+ default boolean hasAccess(AccessType accessType, HttpHeaders httpHeaders,
@Nullable String endpointUrl) {
+return hasAccess(null, accessType, httpHeaders, endpointUrl);
}
/**
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
index a19cad5054..59c0b3a9d4 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
@@ -50,56 +50,32 @@ public final class AccessControlUtils {
*/
public static void validatePermission(@Nullable String tableName, AccessType
accessType,
@Nullable HttpHeaders httpHeaders, @Nullable String endpointUrl,
AccessControl accessControl) {
-String message = null;
+String userMessage = getUserMessage(tableName, accessType, endpointUrl);
+String
[pinot] branch manual-authorization-annotation updated (890aa3cb8c -> 8c1aa68df1)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git from 890aa3cb8c switch null tableName in access control utils add 8bf94e7252 refactor query and option compilation (#9239) add 8c1aa68df1 Merge branch 'master' into manual-authorization-annotation No new revisions were added by this update. Summary of changes: .../broker/api/resources/PinotClientRequest.java | 4 +- .../requesthandler/BaseBrokerRequestHandler.java | 63 ++-- .../BrokerRequestHandlerDelegate.java | 37 +++ .../MultiStageBrokerRequestHandler.java| 13 +-- .../requesthandler/BrokerRequestOptionsTest.java | 108 ++--- .../pinot/common/utils/request/RequestUtils.java | 65 + .../apache/pinot/sql/parsers/CalciteSqlParser.java | 5 +- .../pinot/sql/parsers/SqlNodeAndOptions.java | 4 +- .../pinot/sql/parsers/CalciteSqlCompilerTest.java | 2 +- .../api/resources/PinotQueryResource.java | 13 ++- 10 files changed, 157 insertions(+), 157 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (7feb69b506 -> 890aa3cb8c)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch manual-authorization-annotation
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard 7feb69b506 switch null tableName in access control utils
discard f0e1b3f370 use READ permissions for validation endpoints
discard f93020b8d5 retain hasDataAccess()
discard f63fadc50a remove AccessControlTest (see BasicAuthTest and
UrlAuthRealtimeIntegrationTest)
discard 22d06093e7 endpoint url fixes
discard 470959a6d4 controller auth interface cleanup
discard a991d1a142 annotation fix
discard 67431e2964 linter fix
discard 714efedce6 linter fix
discard 9f050acaaf add @ManualAuthorization annotation for non-standard
endpoints
add 0adf5efe67 Emit metrics for ratio of actual consumption rate to rate
limit in realtime tables (#9201)
add dafa5bff8d [UI]: hide query console tab based on cluster config (#9261)
add e14fba0c2b Fix a typo in TableUpsertMetadataManager (#9269)
add 526653067a add metrics entry (#9270)
add 022fd37d24 Fix message count (#9271)
add 4f95945c49 refine query cancel resp msg (#9242)
add 7bdcc84f88 add @ManualAuthorization annotation for non-standard
endpoints
add 047309ee3d linter fix
add 0dd2f03adf linter fix
add ffa8eecb39 annotation fix
add 78076b3221 controller auth interface cleanup
add fd7ed798f3 endpoint url fixes
add 449ac69d28 remove AccessControlTest (see BasicAuthTest and
UrlAuthRealtimeIntegrationTest)
add e0fe9e7da8 retain hasDataAccess()
add 2df0ba3528 use READ permissions for validation endpoints
add 890aa3cb8c switch null tableName in access control utils
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (7feb69b506)
\
N -- N -- N refs/heads/manual-authorization-annotation
(890aa3cb8c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../configs/controller.yml | 3 +
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 3 +
.../requesthandler/BaseBrokerRequestHandler.java | 21 ++--
.../pinot/common/exception/QueryException.java | 4 +
.../apache/pinot/common/metrics/ServerGauge.java | 3 +-
pinot-controller/src/main/resources/app/App.tsx| 5 +
.../src/main/resources/app/app_state.ts| 1 +
.../src/main/resources/app/components/Layout.tsx | 17 ++-
.../realtime/LLRealtimeSegmentDataManager.java | 3 +-
.../realtime/RealtimeConsumptionRateManager.java | 59 +-
.../core/operator/combine/BaseCombineOperator.java | 3 +
.../combine/SelectionOrderByCombineOperator.java | 7 +-
.../apache/pinot/core/plan/CombinePlanNode.java| 3 +
.../query/executor/ServerQueryExecutorV1Impl.java | 21 +++-
.../pinot/core/query/scheduler/QueryScheduler.java | 83 -
.../core/transport/ChannelHandlerFactory.java | 7 +-
.../core/transport/InstanceRequestHandler.java | 130 ++---
.../apache/pinot/core/transport/QueryServer.java | 28 ++---
.../RealtimeConsumptionRateManagerTest.java| 68 +++
.../operator/combine/CombineSlowOperatorsTest.java | 129 +++-
.../pinot/core/plan/CombinePlanNodeTest.java | 54 -
.../InstanceRequestHandlerTest.java} | 28 +++--
.../pinot/core/transport/QueryRoutingTest.java | 9 +-
.../local/upsert/TableUpsertMetadataManager.java | 2 +-
.../pinot/server/api/resources/QueryResource.java | 4 +-
.../pinot/server/starter/ServerInstance.java | 33 --
...Exception.java => QueryCancelledException.java} | 8 +-
.../apache/pinot/spi/utils/CommonConstants.java| 1 +
28 files changed, 559 insertions(+), 178 deletions(-)
rename
pinot-core/src/test/java/org/apache/pinot/core/{query/scheduler/QuerySchedulerTest.java
=> transport/InstanceRequestHandlerTest.java} (69%)
copy
pinot-spi/src/main/java/org/apache/pinot/spi/exception/{BadQueryRequestException.java
=> QueryCancelledException.java} (79%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (f63fadc50a -> 7feb69b506)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git from f63fadc50a remove AccessControlTest (see BasicAuthTest and UrlAuthRealtimeIntegrationTest) add f93020b8d5 retain hasDataAccess() add f0e1b3f370 use READ permissions for validation endpoints add 7feb69b506 switch null tableName in access control utils No new revisions were added by this update. Summary of changes: .../apache/pinot/controller/api/access/AccessControl.java | 4 +++- .../pinot/controller/api/access/AccessControlUtils.java | 10 -- .../controller/api/resources/PinotQueryResource.java | 7 +++ .../api/resources/PinotSchemaRestletResource.java | 4 ++-- .../PinotSegmentUploadDownloadRestletResource.java| 15 ++- .../api/resources/PinotTableRestletResource.java | 4 ++-- .../api/resources/TableConfigsRestletResource.java| 6 +++--- 7 files changed, 35 insertions(+), 15 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (22d06093e7 -> f63fadc50a)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git from 22d06093e7 endpoint url fixes add f63fadc50a remove AccessControlTest (see BasicAuthTest and UrlAuthRealtimeIntegrationTest) No new revisions were added by this update. Summary of changes: .../pinot/controller/api/AccessControlTest.java| 77 -- .../pinot/controller/helix/ControllerTest.java | 5 +- 2 files changed, 2 insertions(+), 80 deletions(-) delete mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/api/AccessControlTest.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (c43fdad6cc -> 22d06093e7)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch manual-authorization-annotation
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard c43fdad6cc endpoint url fixes
discard 959809ecea controller auth interface cleanup
discard c17bf27ac6 annotation fix
discard f90e6d8727 linter fix
discard ca1dacc3d6 linter fix
discard 908169c38c add @ManualAuthorization annotation for non-standard
endpoints
add 9bba76f45a Fix a UT that only shows up on host with more cores (#9257)
add e01f8c20e0 Cleanup: Fix README to indicate `SQL based query execution
engine.` (#9267)
add fad3de8023 Do not use aggregation result for distinct query in
IntermediateResultsBlock (#9262)
add fd5c942fc8 Add metrics to track controller segment download and upload
requests in progress (#9258)
add 9f050acaaf add @ManualAuthorization annotation for non-standard
endpoints
add 714efedce6 linter fix
add 67431e2964 linter fix
add a991d1a142 annotation fix
add 470959a6d4 controller auth interface cleanup
add 22d06093e7 endpoint url fixes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c43fdad6cc)
\
N -- N -- N refs/heads/manual-authorization-annotation
(22d06093e7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
README.md | 2 +-
.../pinot/common/metrics/ControllerGauge.java | 9 +-
.../InflightRequestMetricsInterceptor.java | 85 +++
.../resources/LLCSegmentCompletionHandlers.java| 3 +
.../PinotSegmentUploadDownloadRestletResource.java | 11 +++
.../api/resources/TrackInflightRequestMetrics.java | 18 ++--
.../TrackedByGauge.java} | 15 ++--
.../InflightRequestMetricsInterceptorTest.java | 99 ++
.../operator/blocks/IntermediateResultsBlock.java | 77 +
.../operator/combine/DistinctCombineOperator.java | 25 ++
.../query/DictionaryBasedDistinctOperator.java | 13 +--
.../core/operator/query/DistinctOperator.java | 11 +--
.../combine/SelectionCombineOperatorTest.java | 20 -
.../apache/pinot/queries/DistinctQueriesTest.java | 8 +-
...erSegmentAggregationSingleValueQueriesTest.java | 69 ---
...InnerSegmentDistinctSingleValueQueriesTest.java | 72
16 files changed, 391 insertions(+), 146 deletions(-)
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/InflightRequestMetricsInterceptor.java
copy
pinot-spi/src/main/java/org/apache/pinot/spi/annotations/metrics/MetricsFactory.java
=>
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TrackInflightRequestMetrics.java
(71%)
copy
pinot-controller/src/main/java/org/apache/pinot/controller/api/{access/Authenticate.java
=> resources/TrackedByGauge.java} (68%)
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/InflightRequestMetricsInterceptorTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/InnerSegmentDistinctSingleValueQueriesTest.java
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (a569240c35 -> c43fdad6cc)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git discard a569240c35 controller auth interface cleanup discard 7357972c10 annotation fix discard 797bc5f213 linter fix discard 8d6585f6c7 linter fix discard b4f623d5e3 add @ManualAuthorization annotation for non-standard endpoints add 718f41f411 Don't throw exception if partition count can't be fetched (#9249) add 80e69c5ab9 Show most recent scheduling errors (#9161) add 13b363c109 Fix bug with logging request headers (#9247) add 908169c38c add @ManualAuthorization annotation for non-standard endpoints add ca1dacc3d6 linter fix add f90e6d8727 linter fix add c17bf27ac6 annotation fix add 959809ecea controller auth interface cleanup add c43fdad6cc endpoint url fixes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (a569240c35) \ N -- N -- N refs/heads/manual-authorization-annotation (c43fdad6cc) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../pinot/broker/api/HttpRequesterIdentity.java| 31 + .../apache/pinot/broker/api/RequesterIdentity.java | 5 +++ .../requesthandler/BaseBrokerRequestHandler.java | 21 + .../api/access/AuthenticationFilter.java | 3 +- .../api/resources/PinotQueryResource.java | 20 + .../src/main/resources/app/pages/TaskDetail.tsx| 4 -- .../main/resources/app/pages/TaskQueueTable.tsx| 13 -- .../src/main/resources/app/requests/index.ts | 3 ++ .../main/resources/app/utils/PinotMethodUtils.ts | 10 - .../realtime/RealtimeConsumptionRateManager.java | 49 + .../RealtimeConsumptionRateManagerTest.java| 51 -- .../apache/pinot/spi/utils/CommonConstants.java| 1 + 12 files changed, 161 insertions(+), 50 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (f67ac4522d -> a569240c35)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch manual-authorization-annotation
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard f67ac4522d controller auth interface cleanup
discard d38c84cd02 annotation fix
discard 13736c79f9 linter fix
discard 5ff57efb8c linter fix
discard ee391e918b add @ManualAuthorization annotation for non-standard
endpoints
add f473bf656e [Refactor] Extract common segment assignment code (#9243)
add 6e6e0a4e86 Fix NPE in gap-fill when order-by is not specified (#9250)
add e6c2f5bd3b Refactor upsert quickstarts (#9246)
add 0a272b2956 Remove currentDataTableVersion from helm value (#9255)
add 7af3edbd52 Fix flakiness and cleanup SegmentLineageCleanupTest (#9256)
add a5a83aa81b Add more failure info in
RealtimeNonReplicaGroupSegmentAssignmentTest.testRelocateCompletedSegments
(#9254)
add b4f623d5e3 add @ManualAuthorization annotation for non-standard
endpoints
add 8d6585f6c7 linter fix
add 797bc5f213 linter fix
add 7357972c10 annotation fix
add a569240c35 controller auth interface cleanup
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f67ac4522d)
\
N -- N -- N refs/heads/manual-authorization-annotation
(a569240c35)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
kubernetes/helm/pinot/values.yaml | 1 -
.../assignment/segment/BaseSegmentAssignment.java | 253 +++
.../segment/OfflineSegmentAssignment.java | 279 -
.../segment/RealtimeSegmentAssignment.java | 257 ---
...altimeNonReplicaGroupSegmentAssignmentTest.java | 39 +--
.../core/retention/SegmentLineageCleanupTest.java | 189 +-
.../org/apache/pinot/core/util/GapfillUtils.java | 21 +-
.../pinot/tools/PartialUpsertQuickStart.java | 75 +-
.../org/apache/pinot/tools/QuickStartBase.java | 83 +-
.../apache/pinot/tools/UpsertJsonQuickStart.java | 63 +
.../org/apache/pinot/tools/UpsertQuickStart.java | 59 +
.../pinot/tools/streams/RsvpSourceGenerator.java | 6 +-
...eHandling_meetupRsvp_realtime_table_config.json | 42
.../complexTypeHandling_meetupRsvp_schema.json | 63 -
...psertJsonMeetupRsvp_realtime_table_config.json} | 4 +-
.../upsertJsonMeetupRsvp_schema.json} | 2 +-
.../upsertMeetupRsvp_realtime_table_config.json} | 4 +-
.../upsertMeetupRsvp_schema.json} | 2 +-
...rtPartialMeetupRsvp_realtime_table_config.json} | 4 +-
.../upsertPartialMeetupRsvp_schema.json} | 2 +-
20 files changed, 558 insertions(+), 890 deletions(-)
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/BaseSegmentAssignment.java
delete mode 100644
pinot-tools/src/main/resources/examples/stream/meetupRsvp/complexTypeHandling_meetupRsvp_realtime_table_config.json
delete mode 100644
pinot-tools/src/main/resources/examples/stream/meetupRsvp/complexTypeHandling_meetupRsvp_schema.json
rename
pinot-tools/src/main/resources/examples/stream/{meetupRsvp/upsert_json_meetupRsvp_realtime_table_config.json
=> upsertJsonMeetupRsvp/upsertJsonMeetupRsvp_realtime_table_config.json} (95%)
rename
pinot-tools/src/main/resources/examples/stream/{meetupRsvp/upsert_json_meetupRsvp_schema.json
=> upsertJsonMeetupRsvp/upsertJsonMeetupRsvp_schema.json} (95%)
rename
pinot-tools/src/main/resources/examples/stream/{meetupRsvp/upsert_meetupRsvp_realtime_table_config.json
=> upsertMeetupRsvp/upsertMeetupRsvp_realtime_table_config.json} (94%)
rename
pinot-tools/src/main/resources/examples/stream/{meetupRsvp/upsert_meetupRsvp_schema.json
=> upsertMeetupRsvp/upsertMeetupRsvp_schema.json} (97%)
rename
pinot-tools/src/main/resources/examples/stream/{meetupRsvp/upsert_partial_meetupRsvp_realtime_table_config.json
=> upsertPartialMeetupRsvp/upsertPartialMeetupRsvp_realtime_table_config.json}
(93%)
rename
pinot-tools/src/main/resources/examples/stream/{meetupRsvp/upsert_partial_meetupRsvp_schema.json
=> upsertPartialMeetupRsvp/upsertPartialMeetupRsvp_schema.json} (96%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (d38c84cd02 -> f67ac4522d)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git from d38c84cd02 annotation fix add f67ac4522d controller auth interface cleanup No new revisions were added by this update. Summary of changes: .../pinot/controller/api/access/AccessControl.java | 12 ++--- .../controller/api/access/AccessControlUtils.java | 56 +- .../access/ZkBasicAuthAccessControlFactory.java| 8 .../PinotAccessControlUserRestletResource.java | 33 ++--- .../api/resources/PinotControllerAuthResource.java | 6 --- .../api/resources/PinotQueryResource.java | 5 +- .../pinot/controller/api/AccessControlTest.java| 23 +++-- .../pinot/spi/utils/builder/TableNameBuilder.java | 3 ++ 8 files changed, 49 insertions(+), 97 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (13736c79f9 -> d38c84cd02)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git from 13736c79f9 linter fix add d38c84cd02 annotation fix No new revisions were added by this update. Summary of changes: .../pinot/controller/api/resources/PinotTableRestletResource.java | 2 ++ .../pinot/controller/api/resources/TableConfigsRestletResource.java | 2 ++ 2 files changed, 4 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (5ff57efb8c -> 13736c79f9)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git from 5ff57efb8c linter fix add 13736c79f9 linter fix No new revisions were added by this update. Summary of changes: .../api/resources/PinotSegmentUploadDownloadRestletResource.java | 1 - 1 file changed, 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch manual-authorization-annotation updated (ee391e918b -> 5ff57efb8c)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git from ee391e918b add @ManualAuthorization annotation for non-standard endpoints add 5ff57efb8c linter fix No new revisions were added by this update. Summary of changes: .../pinot/controller/api/access/ManualAuthorization.java | 1 - .../api/resources/PinotSchemaRestletResource.java | 3 ++- .../PinotSegmentUploadDownloadRestletResource.java | 14 +- .../api/resources/TableConfigsRestletResource.java | 1 + 4 files changed, 4 insertions(+), 15 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: add @ManualAuthorization annotation for non-standard endpoints
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch manual-authorization-annotation
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit ee391e918b188e6e7e6dade5f11157b4c04cf1d4
Author: Alexander Pucher
AuthorDate: Fri Aug 19 13:21:45 2022 -0700
add @ManualAuthorization annotation for non-standard endpoints
---
.../api/access/AuthenticationFilter.java | 5 +++
.../controller/api/access/ManualAuthorization.java | 37 ++
.../api/resources/PinotQueryResource.java | 2 ++
.../api/resources/PinotSchemaRestletResource.java | 15 +++--
.../api/resources/PinotTableRestletResource.java | 23 --
.../api/resources/TableConfigsRestletResource.java | 28 +++-
6 files changed, 105 insertions(+), 5 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
index 8ebd1a2883..2ff482acc9 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
@@ -82,6 +82,11 @@ public class AuthenticationFilter implements
ContainerRequestFilter {
return;
}
+// check if the method's authorization is disabled (i.e. performed
manually within method)
+if (endpointMethod.isAnnotationPresent(ManualAuthorization.class)) {
+ return;
+}
+
// Note that table name is extracted from "path parameters" or "query
parameters" if it's defined as one of the
// followings:
// - "tableName",
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/ManualAuthorization.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/ManualAuthorization.java
new file mode 100644
index 00..b0624418c8
--- /dev/null
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/ManualAuthorization.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.controller.api.access;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * Annotation to be used on top of REST endpoints. Methods annotated with this
annotation don't perform default
+ * authorization via AuthenticationFilter. This is useful when performing
authorization manually via calls to
+ * {@code AuthenticationFiler.validatePermissions()}
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface ManualAuthorization {
+
+}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
index a382122c33..ebee631174 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotQueryResource.java
@@ -54,6 +54,7 @@ import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.api.access.AccessControl;
import org.apache.pinot.controller.api.access.AccessControlFactory;
+import org.apache.pinot.controller.api.access.ManualAuthorization;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.core.query.executor.sql.SqlQueryExecutor;
import org.apache.pinot.spi.utils.CommonConstants;
@@ -87,6 +88,7 @@ public class PinotQueryResource {
@POST
@Path("sql")
+ @ManualAuthorization // performed by broker
public String handlePostSql(String requestJsonStr, @Context HttpHeaders
httpHeaders) {
try {
JsonNode requestJson = JsonUtils.stringToJsonNode(requestJsonStr);
diff --git
a/pinot-contro
[pinot] branch manual-authorization-annotation created (now ee391e918b)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch manual-authorization-annotation in repository https://gitbox.apache.org/repos/asf/pinot.git at ee391e918b add @ManualAuthorization annotation for non-standard endpoints This branch includes the following new commits: new ee391e918b add @ManualAuthorization annotation for non-standard endpoints The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: use default authorization behavior for zk user config
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch use-default-authorization-for-zk-user-config
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 76184bbfe29b845de0975451af423c0319d07126
Author: Alexander Pucher
AuthorDate: Fri Aug 19 13:13:25 2022 -0700
use default authorization behavior for zk user config
---
.../PinotAccessControlUserRestletResource.java | 31 --
1 file changed, 5 insertions(+), 26 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotAccessControlUserRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotAccessControlUserRestletResource.java
index b1d3b9a437..d4d5de2da5 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotAccessControlUserRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotAccessControlUserRestletResource.java
@@ -44,8 +44,6 @@ import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.utils.BcryptUtils;
-import org.apache.pinot.controller.api.access.AccessControlFactory;
-import org.apache.pinot.controller.api.access.AccessControlUtils;
import org.apache.pinot.controller.api.access.AccessType;
import org.apache.pinot.controller.api.access.Authenticate;
import
org.apache.pinot.controller.api.exception.ControllerApplicationException;
@@ -54,7 +52,6 @@ import
org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.spi.config.user.ComponentType;
import org.apache.pinot.spi.config.user.UserConfig;
import org.apache.pinot.spi.utils.JsonUtils;
-import org.glassfish.grizzly.http.server.Request;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,17 +90,12 @@ public class PinotAccessControlUserRestletResource {
@Inject
PinotHelixResourceManager _pinotHelixResourceManager;
-@Inject
-AccessControlFactory _accessControlFactory;
-
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/users")
@ApiOperation(value = "List all uses in cluster", notes = "List all users
in cluster")
-public String listUers(@Context HttpHeaders httpHeaders, @Context Request
request) {
+public String listUsers() {
try {
-String endpointUrl = request.getRequestURL().toString();
-AccessControlUtils.validatePermission(httpHeaders, endpointUrl,
_accessControlFactory.create());
ZkHelixPropertyStore propertyStore =
_pinotHelixResourceManager.getPropertyStore();
Map allUserInfo =
ZKMetadataProvider.getAllUserInfo(propertyStore);
return JsonUtils.newObjectNode().set("users",
JsonUtils.objectToJsonNode(allUserInfo)).toString();
@@ -117,10 +109,8 @@ public class PinotAccessControlUserRestletResource {
@Path("/users/{username}")
@ApiOperation(value = "Get an user in cluster", notes = "Get an user in
cluster")
public String getUser(@PathParam("username") String username,
@QueryParam("component") String componentTypeStr,
-@Context HttpHeaders httpHeaders, @Context Request request) {
+@Context HttpHeaders httpHeaders) {
try {
-String endpointUrl = request.getRequestURL().toString();
-AccessControlUtils.validatePermission(httpHeaders, endpointUrl,
_accessControlFactory.create());
ZkHelixPropertyStore propertyStore =
_pinotHelixResourceManager.getPropertyStore();
ComponentType componentType =
Constants.validateComponentType(componentTypeStr);
String usernameWithType = username + "_" + componentType.name();
@@ -136,7 +126,7 @@ public class PinotAccessControlUserRestletResource {
@Produces(MediaType.APPLICATION_JSON)
@Path("/users")
@ApiOperation(value = "Add a user", notes = "Add a user")
-public SuccessResponse addUser(String userConfigStr, @Context HttpHeaders
httpHeaders, @Context Request request) {
+public SuccessResponse addUser(String userConfigStr) {
// TODO introduce a table config ctor with json string.
UserConfig userConfig;
@@ -144,8 +134,6 @@ public class PinotAccessControlUserRestletResource {
try {
userConfig = JsonUtils.stringToObject(userConfigStr,
UserConfig.class);
username = userConfig.getUserName();
-String endpointUrl = request.getRequestURL().toString();
-AccessControlUtils.validatePermission(httpHeaders, endpointUrl,
_accessControlFactory.create());
if (username.contains(".") || username.contains(" ")) {
[pinot] branch use-default-authorization-for-zk-user-config created (now 76184bbfe2)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch use-default-authorization-for-zk-user-config in repository https://gitbox.apache.org/repos/asf/pinot.git at 76184bbfe2 use default authorization behavior for zk user config This branch includes the following new commits: new 76184bbfe2 use default authorization behavior for zk user config The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: add-table refactor of auth (#9228)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new bf66f4 add-table refactor of auth (#9228)
bf66f4 is described below
commit bf66f42a4e1890b9a57cd38153b51f2f38a4
Author: Alexander Pucher
AuthorDate: Thu Aug 18 10:12:53 2022 -0700
add-table refactor of auth (#9228)
---
.../controller/api/access/AccessControlUtils.java | 85 ++
.../api/access/AuthenticationFilter.java | 45 ++--
.../PinotAccessControlUserRestletResource.java | 17 ++---
.../api/resources/PinotSchemaRestletResource.java | 5 +-
.../api/resources/PinotTableRestletResource.java | 5 +-
.../api/resources/TableConfigsRestletResource.java | 7 +-
.../api/access/AuthenticationFilterTest.java | 23 ++
.../pinot/tools/admin/command/AddTableCommand.java | 56 +++---
8 files changed, 109 insertions(+), 134 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
index c12ba307bd..a19cad5054 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
@@ -19,9 +19,10 @@
package org.apache.pinot.controller.api.access;
-import java.util.Optional;
+import javax.annotation.Nullable;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
+import org.apache.commons.lang3.StringUtils;
import
org.apache.pinot.controller.api.exception.ControllerApplicationException;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
@@ -31,7 +32,11 @@ import org.slf4j.LoggerFactory;
/**
* Utility class to simplify access control validation. This class is simple
wrapper around AccessControl class.
*/
-public class AccessControlUtils {
+public final class AccessControlUtils {
+ private AccessControlUtils() {
+// left blank
+ }
+
private static final Logger LOGGER =
LoggerFactory.getLogger(AccessControlUtils.class);
/**
@@ -43,9 +48,28 @@ public class AccessControlUtils {
* @param endpointUrl the request url for which this access control is called
* @param accessControl AccessControl object which does the actual validation
*/
- public void validatePermission(String tableName, AccessType accessType,
HttpHeaders httpHeaders, String endpointUrl,
- AccessControl accessControl) {
-validatePermission(Optional.of(tableName), accessType, httpHeaders,
endpointUrl, accessControl);
+ public static void validatePermission(@Nullable String tableName, AccessType
accessType,
+ @Nullable HttpHeaders httpHeaders, @Nullable String endpointUrl,
AccessControl accessControl) {
+String message = null;
+try {
+ if (StringUtils.isBlank(tableName)) {
+message = String.format("%s '%s'", accessType, endpointUrl);
+if (!accessControl.hasAccess(accessType, httpHeaders, endpointUrl)) {
+ accessDenied(message);
+}
+ } else {
+message = String.format("%s '%s' for table '%s'", accessType,
endpointUrl, tableName);
+String rawTableName = TableNameBuilder.extractRawTableName(tableName);
+if (!accessControl.hasAccess(rawTableName, accessType, httpHeaders,
endpointUrl)) {
+ accessDenied(message);
+}
+ }
+} catch (ControllerApplicationException e) {
+ throw e;
+} catch (Exception e) {
+ throw new ControllerApplicationException(LOGGER, "Caught exception while
validating permission for " + message,
+ Response.Status.INTERNAL_SERVER_ERROR, e);
+}
}
/**
@@ -56,55 +80,26 @@ public class AccessControlUtils {
* @param endpointUrl the request url for which this access control is called
* @param accessControl AccessControl object which does the actual validation
*/
- public void validatePermission(AccessType accessType, HttpHeaders
httpHeaders, String endpointUrl,
- AccessControl accessControl) {
-validatePermission(Optional.empty(), accessType, httpHeaders, endpointUrl,
accessControl);
- }
-
- /**
- * Validate permission for the given access type against the given table
- *
- * @param tableNameOpt name of the table to be accessed; if `none`, it's a
non-table level endpoint.
- * @param accessType type of the access
- * @param httpHeaders HTTP headers containing requester identity required by
access control object
- * @param endpointUrl the request url for which this access control is called
- * @param accessControl AccessControl object which does the actual validation
- */
- public void validatePermission(Opt
[pinot] branch add-table-auth-cleanup updated (379766a080 -> 42161ca9e3)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch add-table-auth-cleanup in repository https://gitbox.apache.org/repos/asf/pinot.git from 379766a080 cleanup add 42161ca9e3 review fixes No new revisions were added by this update. Summary of changes: .../controller/api/access/AccessControlUtils.java | 22 +++--- 1 file changed, 11 insertions(+), 11 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch add-table-auth-cleanup updated (e3bd48115d -> 379766a080)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch add-table-auth-cleanup
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard e3bd48115d cleanup
discard 3613589df5 linter fix
discard a70cac6703 add-table refactor for clean auth
add 5c703695d2 Add close method to upsert interfaces (#9212)
add 807f84392c fix empty data block not returning schema (#9222)
add fe90978939 Bump zstd-jni from 1.4.9-5 to 1.5.2-3 (#9232)
add c1bdd10e6b Adding kafka offset support for period and timestamp (#9193)
add d41ec0bfd4 Refactor stream quickstart (#9227)
add 3a655d2d0e support to show running queries and cancel query by id
(#9171)
add 74dc8784fb add-table refactor for clean auth
add 4d65ecfbf9 linter fix
add 379766a080 cleanup
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e3bd48115d)
\
N -- N -- N refs/heads/add-table-auth-cleanup (379766a080)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../broker/api/resources/PinotClientRequest.java | 63
.../broker/broker/BrokerAdminApiApplication.java | 14 ++
.../requesthandler/BaseBrokerRequestHandler.java | 139 -
.../requesthandler/BrokerRequestHandler.java | 19 +++
.../BrokerRequestHandlerDelegate.java | 19 +++
.../BaseBrokerRequestHandlerTest.java | 96
...{MultiGetRequest.java => MultiHttpRequest.java} | 58 +---
.../pinot/common/utils/config/InstanceUtils.java | 26
...tRequestTest.java => MultiHttpRequestTest.java} | 8 +-
.../pinot/controller/BaseControllerStarter.java| 3 +-
.../helix/core/PinotHelixResourceManager.java | 24 +--
.../controller/util/CompletionServiceHelper.java | 4 +-
.../manager/realtime/RealtimeTableDataManager.java | 7 +
.../core/query/request/ServerQueryRequest.java | 8 +
.../pinot/core/query/scheduler/QueryScheduler.java | 86 ++-
.../core/transport/InstanceRequestHandler.java | 2 +-
.../pinot/core/transport/ServerInstance.java | 9 ++
.../realtime/LLRealtimeSegmentDataManagerTest.java | 108 +-
.../core/query/scheduler/QuerySchedulerTest.java | 76 ++
.../core/realtime/stream/OffsetCriteriaTest.java | 26
.../pinot/core/transport/QueryRoutingTest.java | 2 +-
.../tests/MultiStageEngineIntegrationTest.java | 8 +-
.../kafka20/KafkaStreamMetadataProvider.java | 32
.../kafka20/server/KafkaDataServerStartable.java | 31
.../kafka20/KafkaPartitionLevelConsumerTest.java | 5 +
.../pinot/query/service/QueryDispatcher.java | 8 +-
.../pinot/query/runtime/QueryRunnerTest.java | 3 +
...oncurrentMapPartitionUpsertMetadataManager.java | 6 +
.../ConcurrentMapTableUpsertMetadataManager.java | 8 +
.../upsert/PartitionUpsertMetadataManager.java | 3 +-
.../local/upsert/TableUpsertMetadataManager.java | 3 +-
.../pinot/server/api/resources/QueryResource.java | 100 +
.../pinot/server/starter/ServerInstance.java | 4 +
.../apache/pinot/spi/stream/OffsetCriteria.java| 42 --
.../apache/pinot/spi/utils/CommonConstants.java| 1 +
.../java/org/apache/pinot/spi/utils/TimeUtils.java | 34 +
.../org/apache/pinot/spi/utils/TimeUtilsTest.java | 59
.../org/apache/pinot/tools/GenericQuickstart.java | 29
.../apache/pinot/tools/GitHubEventsQuickstart.java | 28 ++--
.../org/apache/pinot/tools/HybridQuickstart.java | 165 +++--
.../pinot/tools/PartialUpsertQuickStart.java | 18 +--
.../org/apache/pinot/tools/QuickStartBase.java | 164
.../apache/pinot/tools/QuickstartTableRequest.java | 21 +++
.../RealtimeComplexTypeHandlingQuickStart.java | 83 +--
.../pinot/tools/RealtimeJsonIndexQuickStart.java | 92 ++--
.../org/apache/pinot/tools/RealtimeQuickStart.java | 102 +
.../pinot/tools/RealtimeQuickStartWithMinion.java | 126 ++--
.../apache/pinot/tools/UpsertJsonQuickStart.java | 19 +--
.../org/apache/pinot/tools/UpsertQuickStart.java | 19 +--
.../pinot/tools/streams/AirlineDataStream.java | 13 +-
.../pinot/tools/streams/MeetupRsvpStream.java | 18 ++-
.../pinot/tools/streams/RsvpSo
[pinot] branch add-table-auth-cleanup updated (3613589df5 -> e3bd48115d)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch add-table-auth-cleanup in repository https://gitbox.apache.org/repos/asf/pinot.git from 3613589df5 linter fix add e3bd48115d cleanup No new revisions were added by this update. Summary of changes: .../controller/api/access/AccessControlUtils.java | 30 -- 1 file changed, 16 insertions(+), 14 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch add-table-auth-cleanup updated (81812ea006 -> 3613589df5)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch add-table-auth-cleanup in repository https://gitbox.apache.org/repos/asf/pinot.git omit 81812ea006 linter fix omit 1bf9931a53 add-table refactor for clean auth add 2aa2165367 fix UT failure on explain plan (#9226) add 661a7d6a5d Adding pinot base docker image to reduce build time (#9229) add a70cac6703 add-table refactor for clean auth add 3613589df5 linter fix This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (81812ea006) \ N -- N -- N refs/heads/add-table-auth-cleanup (3613589df5) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: docker/images/pinot-base/README.md | 59 ++ .../images/pinot-base/pinot-base-build/Dockerfile | 48 ++ .../pinot-base/pinot-base-runtime/Dockerfile | 39 ++ docker/images/pinot/Dockerfile | 41 ++- .../pinot/queries/ExplainPlanQueriesTest.java | 5 +- 5 files changed, 153 insertions(+), 39 deletions(-) create mode 100644 docker/images/pinot-base/README.md create mode 100644 docker/images/pinot-base/pinot-base-build/Dockerfile create mode 100644 docker/images/pinot-base/pinot-base-runtime/Dockerfile - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch add-table-auth-cleanup updated (1bf9931a53 -> 81812ea006)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch add-table-auth-cleanup in repository https://gitbox.apache.org/repos/asf/pinot.git from 1bf9931a53 add-table refactor for clean auth add 81812ea006 linter fix No new revisions were added by this update. Summary of changes: .../org/apache/pinot/tools/admin/command/AddTableCommand.java | 8 1 file changed, 4 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch add-table-auth-cleanup created (now 1bf9931a53)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch add-table-auth-cleanup in repository https://gitbox.apache.org/repos/asf/pinot.git at 1bf9931a53 add-table refactor for clean auth This branch includes the following new commits: new 1bf9931a53 add-table refactor for clean auth The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: add-table refactor for clean auth
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch add-table-auth-cleanup
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 1bf9931a534913b6a137040d032a4e034335ce50
Author: Alexander Pucher
AuthorDate: Tue Aug 16 21:31:53 2022 -0700
add-table refactor for clean auth
---
.../controller/api/access/AccessControlUtils.java | 79 ++
.../api/access/AuthenticationFilter.java | 45 ++--
.../PinotAccessControlUserRestletResource.java | 17 ++---
.../api/resources/PinotSchemaRestletResource.java | 5 +-
.../api/resources/PinotTableRestletResource.java | 5 +-
.../api/resources/TableConfigsRestletResource.java | 7 +-
.../api/access/AuthenticationFilterTest.java | 23 +++
.../pinot/tools/admin/command/AddTableCommand.java | 52 +++---
8 files changed, 103 insertions(+), 130 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
index c12ba307bd..88d71736ba 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AccessControlUtils.java
@@ -19,9 +19,9 @@
package org.apache.pinot.controller.api.access;
-import java.util.Optional;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
+import org.apache.commons.lang3.StringUtils;
import
org.apache.pinot.controller.api.exception.ControllerApplicationException;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
@@ -31,7 +31,11 @@ import org.slf4j.LoggerFactory;
/**
* Utility class to simplify access control validation. This class is simple
wrapper around AccessControl class.
*/
-public class AccessControlUtils {
+public final class AccessControlUtils {
+ private AccessControlUtils() {
+// left blank
+ }
+
private static final Logger LOGGER =
LoggerFactory.getLogger(AccessControlUtils.class);
/**
@@ -43,9 +47,28 @@ public class AccessControlUtils {
* @param endpointUrl the request url for which this access control is called
* @param accessControl AccessControl object which does the actual validation
*/
- public void validatePermission(String tableName, AccessType accessType,
HttpHeaders httpHeaders, String endpointUrl,
- AccessControl accessControl) {
-validatePermission(Optional.of(tableName), accessType, httpHeaders,
endpointUrl, accessControl);
+ public static void validatePermission(String tableName, AccessType
accessType, HttpHeaders httpHeaders,
+ String endpointUrl, AccessControl accessControl) {
+String accessTypeToEndpointMsg =
+String.format("access type '%s' to the endpoint '%s' for table '%s'",
accessType, endpointUrl, tableName);
+try {
+ if (StringUtils.isBlank(tableName)) {
+if (!accessControl.hasAccess(accessType, httpHeaders, endpointUrl)) {
+ accessDenied(accessTypeToEndpointMsg);
+}
+ } else {
+String rawTableName = TableNameBuilder.extractRawTableName(tableName);
+if (!accessControl.hasAccess(rawTableName, accessType, httpHeaders,
endpointUrl)) {
+ accessDenied(accessTypeToEndpointMsg);
+}
+ }
+} catch (Exception e) {
+ if (!(e instanceof ControllerApplicationException)) {
+throw new ControllerApplicationException(LOGGER,
+"Caught exception while validating permission for " +
accessTypeToEndpointMsg,
+Response.Status.INTERNAL_SERVER_ERROR, e);
+ }
+}
}
/**
@@ -56,55 +79,25 @@ public class AccessControlUtils {
* @param endpointUrl the request url for which this access control is called
* @param accessControl AccessControl object which does the actual validation
*/
- public void validatePermission(AccessType accessType, HttpHeaders
httpHeaders, String endpointUrl,
+ public static void validatePermission(AccessType accessType, HttpHeaders
httpHeaders, String endpointUrl,
AccessControl accessControl) {
-validatePermission(Optional.empty(), accessType, httpHeaders, endpointUrl,
accessControl);
+validatePermission(null, accessType, httpHeaders, endpointUrl,
accessControl);
}
/**
* Validate permission for the given access type against the given table
*
- * @param tableNameOpt name of the table to be accessed; if `none`, it's a
non-table level endpoint.
- * @param accessType type of the access
* @param httpHeaders HTTP headers containing requester identity required by
access control object
* @param endpointUrl the request url for which this access control is called
- * @param accessControl AccessControl object which does the actual validation
*/
- public void validatePermission(O
[pinot] branch master updated (cd55015c0a -> 24e2ad53bf)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git from cd55015c0a Update ZK config in helm chart (#8817) add 24e2ad53bf Fix auth provider for minion (#8831) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/pinot/minion/BaseMinionStarter.java | 4 +++- .../src/main/java/org/apache/pinot/minion/MinionContext.java | 11 ++- .../apache/pinot/minion/taskfactory/TaskFactoryRegistry.java | 4 +++- .../main/java/org/apache/pinot/spi/utils/CommonConstants.java | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated (7b9e16b65d -> 05271b0b09)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git from 7b9e16b65d Not overwrite the whole SystemResourceInfo config map (#8820) add 05271b0b09 add auth provider to minion client (#8826) No new revisions were added by this update. Summary of changes: .../apache/pinot/common/minion/MinionClient.java | 24 ++ .../pinot/common/minion/MinionClientTest.java | 6 +++--- .../core/query/executor/sql/SqlQueryExecutor.java | 5 +++-- .../org/apache/pinot/tools/BootstrapTableTool.java | 3 ++- 4 files changed, 19 insertions(+), 19 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch auth-minion-client updated (05f0621822 -> 698b59ea45)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch auth-minion-client in repository https://gitbox.apache.org/repos/asf/pinot.git from 05f0621822 add back type ref for 1.8 add 698b59ea45 another TypeRef No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/pinot/common/minion/MinionClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch auth-minion-client updated (5c93b93ccc -> 05f0621822)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch auth-minion-client in repository https://gitbox.apache.org/repos/asf/pinot.git from 5c93b93ccc add auth provider to minion client add 05f0621822 add back type ref for 1.8 No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/pinot/common/minion/MinionClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch auth-minion-client updated: add auth provider to minion client
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch auth-minion-client
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/auth-minion-client by this
push:
new 5c93b93ccc add auth provider to minion client
5c93b93ccc is described below
commit 5c93b9346bd1ccc0df2ac2e065b6762e65f1
Author: Alexander Pucher
AuthorDate: Thu Jun 2 23:04:36 2022 -0700
add auth provider to minion client
---
.../apache/pinot/common/minion/MinionClient.java | 28 ++
.../pinot/common/minion/MinionClientTest.java | 6 ++---
.../core/query/executor/sql/SqlQueryExecutor.java | 5 ++--
.../org/apache/pinot/tools/BootstrapTableTool.java | 3 ++-
4 files changed, 21 insertions(+), 21 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/minion/MinionClient.java
b/pinot-common/src/main/java/org/apache/pinot/common/minion/MinionClient.java
index 1909f0c8e1..2fcd32d919 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/minion/MinionClient.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/minion/MinionClient.java
@@ -30,8 +30,10 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.pinot.common.auth.AuthProviderUtils;
import org.apache.pinot.spi.annotations.InterfaceAudience;
import org.apache.pinot.spi.annotations.InterfaceStability;
+import org.apache.pinot.spi.auth.AuthProvider;
import org.apache.pinot.spi.config.task.AdhocTaskConfig;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -50,17 +52,11 @@ public class MinionClient {
private static final String HTTP = "http";
private final String _controllerUrl;
+ private final AuthProvider _authProvider;
- public MinionClient(String controllerHost, String controllerPort) {
-this(HTTP, controllerHost, controllerPort);
- }
-
- public MinionClient(String scheme, String controllerHost, String
controllerPort) {
-this(String.format("%s://%s:%s", scheme, controllerHost, controllerPort));
- }
-
- public MinionClient(String controllerUrl) {
+ public MinionClient(String controllerUrl, AuthProvider authProvider) {
_controllerUrl = controllerUrl;
+_authProvider = authProvider;
}
public String getControllerUrl() {
@@ -70,7 +66,7 @@ public class MinionClient {
public Map scheduleMinionTasks(@Nullable String taskType,
@Nullable String tableNameWithType)
throws IOException {
HttpPost httpPost = createHttpPostRequest(
-
MinionRequestURLBuilder.baseUrl(getControllerUrl()).forTaskSchedule(taskType,
tableNameWithType));
+
MinionRequestURLBuilder.baseUrl(_controllerUrl).forTaskSchedule(taskType,
tableNameWithType));
HttpResponse response = HTTP_CLIENT.execute(httpPost);
int statusCode = response.getStatusLine().getStatusCode();
final String responseString =
IOUtils.toString(response.getEntity().getContent());
@@ -85,7 +81,7 @@ public class MinionClient {
public Map getTasksStates(String taskType)
throws IOException {
HttpGet httpGet =
-
createHttpGetRequest(MinionRequestURLBuilder.baseUrl(getControllerUrl()).forTasksStates(taskType));
+
createHttpGetRequest(MinionRequestURLBuilder.baseUrl(_controllerUrl).forTasksStates(taskType));
HttpResponse response = HTTP_CLIENT.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
final String responseString =
IOUtils.toString(response.getEntity().getContent());
@@ -93,13 +89,13 @@ public class MinionClient {
throw new HttpException(String
.format("Unable to get tasks states map. Error code %d, Error
message: %s", statusCode, responseString));
}
-return JsonUtils.stringToObject(responseString, new
TypeReference>() {
+return JsonUtils.stringToObject(responseString, new TypeReference<>() {
});
}
public String getTaskState(String taskName)
throws IOException {
-HttpGet httpGet =
createHttpGetRequest(MinionRequestURLBuilder.baseUrl(getControllerUrl()).forTaskState(taskName));
+HttpGet httpGet =
createHttpGetRequest(MinionRequestURLBuilder.baseUrl(_controllerUrl).forTaskState(taskName));
HttpResponse response = HTTP_CLIENT.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
String responseString =
IOUtils.toString(response.getEntity().getContent());
@@ -113,7 +109,7 @@ public class MinionClient {
public Map executeTask(AdhocTaskConfig adhocTaskConfig,
@Nullable Map headers)
throws IOException {
-HttpPost httpPost =
createHttpPostRequest(MinionRequestURLBuilder.baseUrl(getControllerUrl()).forTaskExecute());
+HttpPost httpPost =
createHttpPostRequest(MinionRequestURLBuilder
[pinot] branch auth-minion-client created (now 7a311428bc)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch auth-minion-client in repository https://gitbox.apache.org/repos/asf/pinot.git at 7a311428bc Change DateTimeFormatSpec delimiter (#8779) No new revisions were added by this update. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: Add pluggable client auth provider (#8670)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2c3813bb97 Add pluggable client auth provider (#8670)
2c3813bb97 is described below
commit 2c3813bb9722d0ec6053bafbbdd07d7d2bd32cc3
Author: Alexander Pucher
AuthorDate: Tue May 17 12:07:24 2022 -0700
Add pluggable client auth provider (#8670)
add support for pluggable client auth providers, which enables pinot
components (controller, server, minion) to use dynamically changing tokens,
such as kubernetes service account JWTs
---
.../org/apache/pinot/common/auth/AuthConfig.java | 23 ++-
.../pinot/common/auth/AuthProviderUtils.java | 171 +
.../apache/pinot/common/auth/NullAuthProvider.java | 29 +++-
.../pinot/common/auth/StaticTokenAuthProvider.java | 68
.../apache/pinot/common/auth/UrlAuthProvider.java | 87 +++
.../common/utils/FileUploadDownloadClient.java | 45 +++---
.../common/utils/fetcher/BaseSegmentFetcher.java | 7 +-
.../common/utils/fetcher/HttpSegmentFetcher.java | 4 +-
.../utils/fetcher/SegmentFetcherFactory.java | 15 +-
.../apache/pinot/common/utils/http/HttpClient.java | 70 +++--
.../resources/PinotIngestionRestletResource.java | 12 +-
.../pinot/controller/util/FileIngestionHelper.java | 10 +-
.../core/data/manager/BaseTableDataManager.java| 18 ++-
.../manager/realtime/SegmentCommitterFactory.java | 2 +-
.../realtime/Server2ControllerSegmentUploader.java | 11 +-
.../ServerSegmentCompletionProtocolHandler.java| 18 ++-
.../BaseTableDataManagerAcquireSegmentTest.java| 3 +
.../data/manager/BaseTableDataManagerTest.java | 3 +
.../tests/BasicAuthBatchIntegrationTest.java | 4 +-
.../integration/tests/TlsIntegrationTest.java | 7 +
...st.java => UrlAuthRealtimeIntegrationTest.java} | 55 +--
.../src/test/resources/url-auth-token-prefixed.txt | 1 +
.../src/test/resources/url-auth-token.txt | 1 +
.../BaseMultipleSegmentsConversionExecutor.java| 19 +--
.../tasks/BaseSingleSegmentConversionExecutor.java | 7 +-
.../minion/tasks/SegmentConversionUtils.java | 17 +-
.../segmentuploader/SegmentUploaderDefault.java| 10 +-
.../local/data/manager/TableDataManagerConfig.java | 14 +-
.../pinot/segment/local/utils/IngestionUtils.java | 17 +-
.../segment/local/utils/SegmentPushUtils.java | 14 +-
.../starter/helix/HelixInstanceDataManager.java| 5 +-
.../helix/HelixInstanceDataManagerConfig.java | 8 -
.../auth/{AuthContext.java => AuthProvider.java} | 21 +--
.../config/instance/InstanceDataManagerConfig.java | 2 -
.../batch/spec/SegmentGenerationJobSpec.java | 6 +
.../segment/uploader/SegmentUploader.java | 10 +-
.../apache/pinot/spi/utils/CommonConstants.java| 6 +-
.../org/apache/pinot/tools/AuthQuickstart.java | 6 +-
.../org/apache/pinot/tools/BootstrapTableTool.java | 13 +-
.../org/apache/pinot/tools/EmptyQuickstart.java| 5 +-
.../java/org/apache/pinot/tools/Quickstart.java| 5 +-
.../admin/command/AbstractBaseAdminCommand.java| 38 +++--
.../tools/admin/command/AddSchemaCommand.java | 14 +-
.../pinot/tools/admin/command/AddTableCommand.java | 20 ++-
.../tools/admin/command/AddTenantCommand.java | 17 +-
.../tools/admin/command/BootstrapTableCommand.java | 16 +-
.../tools/admin/command/ChangeTableState.java | 20 ++-
.../tools/admin/command/ImportDataCommand.java | 12 +-
.../command/LaunchDataIngestionJobCommand.java | 11 +-
.../admin/command/OperateClusterConfigCommand.java | 19 ++-
.../tools/admin/command/PostQueryCommand.java | 17 +-
.../tools/admin/command/QuickstartRunner.java | 15 +-
.../tools/admin/command/UploadSegmentCommand.java | 20 ++-
53 files changed, 784 insertions(+), 284 deletions(-)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/auth/AuthContext.java
b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthConfig.java
similarity index 60%
copy from pinot-spi/src/main/java/org/apache/pinot/spi/auth/AuthContext.java
copy to pinot-common/src/main/java/org/apache/pinot/common/auth/AuthConfig.java
index 5a9798c355..60e117bd98 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/auth/AuthContext.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthConfig.java
@@ -16,19 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.spi.auth;
+package org.apache.pinot.common.auth;
+
+import java.util.Map;
+import org.apache.pinot.spi.env.PinotConfiguration;
+
/**
- * Container for all auth related info
+ * Standardized auth config container for AuthProvider
+ * @see AuthProviderUtils#extractAuthConfig(PinotConfigu
[pinot] branch pinot-flexible-auth-provider updated (e700acc841 -> 1b0a2da559)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git discard e700acc841 more test fixes discard 9689b64877 pinot dynamic auth provider add d06338acb8 avoid putting null into segment download uri (#8693) add 3983728e64 make exclusive bounds exclusive (#8703) add 078d8969bf Reduce wait time for pulsar tests (#8702) add d3c97d54d2 Add Pulsar plugin back to distribution (#8657) add f90137b1d5 rationalise Pair types (#8709) add e4a8e6ed60 throw when integer overflow occurs (#8710) add e788a0afc6 Fix when there're two index filter operator h3 inclusion index throw exception (#8707) add 159afb7820 Cleanup segment upload logic and allow validation on real-time table (#8695) add 7548b67592 CSV Realtime Decoder (#8658) add 70f505f028 Bugfixing, don't remove timestmap index during server restart. (#8705) add 09bae15030 Scalable Gapfill Implementation for Avg/Count/Sum (#8647) add d62438c36c [Minor] Make warning log more explicit for segment upload (#8715) add b8af790c2f Enable uploading segments to realtime tables (#8584) add 6390a40eb9 pinot dynamic auth provider add 1b0a2da559 more test fixes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (e700acc841) \ N -- N -- N refs/heads/pinot-flexible-auth-provider (1b0a2da559) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../java/org/apache/pinot/common/utils/Pair.java | 49 -- .../apache/pinot/common/utils/SegmentUtils.java| 32 +- .../org/apache/pinot/common/utils/URIUtils.java| 14 +- .../PinotSegmentUploadDownloadRestletResource.java | 268 +++ .../api/upload/SegmentValidationUtils.java | 94 +++ .../controller/api/upload/SegmentValidator.java| 122 --- .../pinot/controller/api/upload/ZKOperator.java| 116 ++- .../helix/core/PinotHelixResourceManager.java | 60 +- .../segment/RealtimeSegmentAssignment.java | 50 +- .../realtime/PinotLLCRealtimeSegmentManager.java | 16 +- .../core/realtime/PinotRealtimeSegmentManager.java | 10 +- .../helix/core/util/ZKMetadataUtils.java | 12 +- .../controller/api/upload/ZKOperatorTest.java | 40 +- ...altimeNonReplicaGroupSegmentAssignmentTest.java | 114 ++- ...NonReplicaGroupTieredSegmentAssignmentTest.java | 5 +- .../RealtimeReplicaGroupSegmentAssignmentTest.java | 132 +++- .../helix/core/realtime/SegmentCompletionTest.java | 10 +- .../core/data/manager/BaseTableDataManager.java| 15 +- .../manager/offline/OfflineTableDataManager.java | 12 - .../manager/offline/TableDataManagerProvider.java | 2 +- .../manager/realtime/RealtimeTableDataManager.java | 5 +- .../core/operator/filter/FilterOperatorUtils.java | 3 +- .../filter/H3InclusionIndexFilterOperator.java | 52 +- .../predicate/RangePredicateEvaluatorFactory.java | 16 +- .../statement/JsonStatementOptimizer.java | 20 +- .../core/query/reduce/BaseGapfillProcessor.java| 229 ++ .../core/query/reduce/BrokerReduceService.java | 7 +- .../core/query/reduce/CountGapfillProcessor.java | 131 .../pinot/core/query/reduce/GapfillProcessor.java | 138 +--- .../core/query/reduce/GapfillProcessorFactory.java | 65 ++ .../core/query/reduce/SumAvgGapfillProcessor.java | 212 ++ .../server/realtime/ControllerLeaderLocator.java | 6 +- .../ServerSegmentCompletionProtocolHandler.java| 8 +- .../queries/GapfillQueriesScalabilityTest.java | 822 + .../apache/pinot/queries/GapfillQueriesTest.java | 1 + .../apache/pinot/queries/H3IndexQueriesTest.java | 28 + .../queries/JsonIngestionFromAvroQueriesTest.java | 10 +- .../org/apache/pinot/queries/RangeQueriesTest.java | 72 +- .../realtime/ControllerLeaderLocatorTest.java | 30 +- pinot-distribution/pinot-assembly.xml | 2 - .../tests/LLCRealtimeClusterIntegrationTest.java | 96 ++- .../tests/RealtimeClusterIntegrationTest.java | 16 +- .../ControllerLeaderLocatorIntegrationTest.java| 8 +- .../plugin/inputformat/csv/CSVMessageDecoder.java | 146 .../inputformat/csv/CSVMessageDecoderTest.java | 157 .../plugin/stream/pulsar/PulsarConsumerTest.java | 4 +- ..
[pinot] branch pinot-flexible-auth-provider updated (9689b64877 -> e700acc841)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git from 9689b64877 pinot dynamic auth provider add e700acc841 more test fixes No new revisions were added by this update. Summary of changes: .../java/org/apache/pinot/core/data/manager/BaseTableDataManager.java | 3 +++ .../core/data/manager/BaseTableDataManagerAcquireSegmentTest.java | 3 +++ 2 files changed, 6 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch pinot-flexible-auth-provider updated (0fdc0b0b93 -> 9689b64877)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git discard 0fdc0b0b93 test mock fix discard 46152dc5cf rebase regression discard 3a4c9e8bd6 avoid property loss discard 5f55cafe63 review fixes discard 64ede3f1dc review cleanup discard ed87385b03 test cleanup discard 4b224d060f integration test discard 192938d62c auth provider round 2 discard 14bbae30d1 pinot auth provider draft add cfb0074a2a Upgrade h3 to 3.7.2 for M1 Mac support (#8696) add f65b401dc7 Set max number of parallel segment downloads per table in pinot-server (#8694) add 9689b64877 pinot dynamic auth provider This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (0fdc0b0b93) \ N -- N -- N refs/heads/pinot-flexible-auth-provider (9689b64877) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../core/data/manager/BaseTableDataManager.java| 25 +- .../manager/offline/TableDataManagerProvider.java | 5 - .../BaseTableDataManagerAcquireSegmentTest.java| 2 +- .../data/manager/BaseTableDataManagerTest.java | 2 +- .../offline/DimensionTableDataManagerTest.java | 2 +- .../local/data/manager/TableDataManager.java | 2 +- .../helix/HelixInstanceDataManagerConfig.java | 12 +++ .../apache/pinot/server/api/BaseResourceTest.java | 2 +- .../config/instance/InstanceDataManagerConfig.java | 2 ++ pom.xml| 2 +- 10 files changed, 48 insertions(+), 8 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch pinot-flexible-auth-provider updated (46152dc5cf -> 0fdc0b0b93)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git from 46152dc5cf rebase regression add 0fdc0b0b93 test mock fix No new revisions were added by this update. Summary of changes: .../org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java | 3 +++ 1 file changed, 3 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch pinot-flexible-auth-provider updated (3a4c9e8bd6 -> 46152dc5cf)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git from 3a4c9e8bd6 avoid property loss add 46152dc5cf rebase regression No new revisions were added by this update. Summary of changes: pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java | 7 +-- 1 file changed, 1 insertion(+), 6 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch pinot-flexible-auth-provider updated (7bd836b7cf -> 3a4c9e8bd6)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git omit 7bd836b7cf avoid property loss omit fb9f179e59 review fixes omit 509e8e0c05 review cleanup omit cdd45b81ea test cleanup omit a03d5015a4 integration test omit 03e0a5df9a auth provider round 2 omit 50d3545803 pinot auth provider draft add 24a86acec0 Fixing GRPC ITCase timeout (#8686) add f9e55ce7a6 Handle the race condition for common dest segments in startReplaceSegments(). (#8667) add df29b81c27 Change minion's default port, enable minion health and readiness probe via helm chart (#8669) add 088da3f8c2 deduplicate predicates optimistically (#8687) add c3d17b9ed7 Fix PulsarConsumerTest (#8691) add 14bbae30d1 pinot auth provider draft add 192938d62c auth provider round 2 add 4b224d060f integration test add ed87385b03 test cleanup add 64ede3f1dc review cleanup add 5f55cafe63 review fixes add 3a4c9e8bd6 avoid property loss This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (7bd836b7cf) \ N -- N -- N refs/heads/pinot-flexible-auth-provider (3a4c9e8bd6) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: kubernetes/helm/pinot/values.yaml | 4 +- .../helix/core/PinotHelixResourceManager.java | 59 + .../helix/core/PinotHelixResourceManagerTest.java | 62 +++-- .../plan/AggregationGroupByOrderByPlanNode.java| 2 +- .../pinot/core/plan/AggregationPlanNode.java | 2 +- .../org/apache/pinot/core/plan/FilterPlanNode.java | 20 +++--- .../apache/pinot/core/startree/StarTreeUtils.java | 24 --- .../apache/pinot/core/util/ListenerConfigUtil.java | 19 ++ .../pinot/core/startree/v2/BaseStarTreeV2Test.java | 2 +- .../tests/OfflineGRPCServerIntegrationTest.java| 2 +- .../org/apache/pinot/minion/BaseMinionStarter.java | 2 +- .../org/apache/pinot/perf/BenchmarkQueries.java| 6 +- .../plugin/stream/pulsar/PulsarConsumerTest.java | 77 ++ .../apache/pinot/spi/utils/CommonConstants.java| 2 - .../src/main/resources/conf/pinot-minion.conf | 5 +- 15 files changed, 191 insertions(+), 97 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch pinot-flexible-auth-provider updated (fb9f179e59 -> 7bd836b7cf)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git from fb9f179e59 review fixes add 7bd836b7cf avoid property loss No new revisions were added by this update. Summary of changes: .../pinot/segment/local/data/manager/TableDataManagerConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch pinot-flexible-auth-provider updated (a494d80a76 -> fb9f179e59)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git discard a494d80a76 review fixes discard 15acfde711 review cleanup discard 3d5f10c2f8 test cleanup discard 829995430b integration test discard 8ecba358ca auth provider round 2 discard c20007e6dc pinot auth provider draft add d0f9aaf5b3 fix GcsPinotFs listFiles by using bucket directly (#8656) add b4943458a5 [issue-7357] Add support for Avro logical types (#7358) add c0056dc278 Allow broker to automatically rewrite expensive function to its approximate counterpart (#8655) add 8fac50c038 [CI] Reduce quickstart testing to only LTS (#8665) add b9bfb8e752 Support ADLS authentication with AAD + proxy (#8654) add 497d0517ab Introduce Real Time Native Text Index (#8636) add d208f577ed Adding pinot file system command (#8659) add abf62f571d fix pruning in expressions by max/min/bloom (#8672) add 65cecd259e Randomly pick the first URI for RoundRobinURIProvider (#8666) add f4c85e9596 Fix the flaky UpsertTableSegmentUploadIntegrationTest (#8675) add 5135acb95d refine error msg related to segment name generation to be more actionable (#8676) add d616a097e5 Add BigDecimal OnHeap/OffHeap mutable dictionaries (#8678) add a12b760b74 Amend schema to add timestamp index during segment reloading (#8679) add d7f2054b6e Import dry run (#8673) add 23a81d07b5 Add TLS configuration to JDBC driver (#8578) add a7e2c12c99 Add InstanceTypeUtils to get instance type from the id (#8681) add 59bf3827db By default start 1 minion for all quickstart (#8682) add b58650658c Allow updating start/end offset for pushed segments (#8653) add 56282e8b2f track group by evaluation statistics (#8683) add 6d785e0eb3 Upgrade jersey to fix parallel segment upload (#8677) add e12bda1cf6 Fix the race condition of reading time boundary info (#8685) add 50d3545803 pinot auth provider draft add 03e0a5df9a auth provider round 2 add a03d5015a4 integration test add cdd45b81ea test cleanup add 509e8e0c05 review cleanup add fb9f179e59 review fixes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (a494d80a76) \ N -- N -- N refs/heads/pinot-flexible-auth-provider (fb9f179e59) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .github/workflows/pinot_tests.yml | 4 +- .github/workflows/scripts/.pinot_quickstart.sh | 33 +-- LICENSE-binary | 42 ++- .../broker/broker/helix/BaseBrokerStarter.java | 9 +- .../requesthandler/BaseBrokerRequestHandler.java | 289 + .../pinot/broker/routing/BrokerRoutingManager.java | 9 +- .../broker/broker/HelixBrokerStarterTest.java | 2 +- .../requesthandler/DistinctCountRewriteTest.java | 44 .../requesthandler/QueryLimitOverrideTest.java | 50 .../broker/requesthandler/QueryOverrideTest.java | 123 + pinot-clients/pinot-jdbc-client/pom.xml| 4 + .../java/org/apache/pinot/client/PinotDriver.java | 35 ++- .../controller/PinotControllerTransport.java | 31 ++- .../PinotControllerTransportFactory.java} | 12 +- .../org/apache/pinot/client/utils/DriverUtils.java | 19 +- .../common/metadata/segment/SegmentZKMetadata.java | 3 +- .../pinot/common/utils/RoundRobinURIProvider.java | 4 +- .../pinot/common/utils/helix/HelixHelper.java | 4 +- .../common/utils/RoundRobinURIProviderTest.java| 151 ++- .../common/utils/config/TableConfigSerDeTest.java | 6 +- .../pinot/controller/BaseControllerStarter.java| 8 +- .../resources/PinotIngestionRestletResource.java | 20 +- .../pinot/controller/api/upload/ZKOperator.java| 162 ++-- .../helix/core/PinotHelixResourceManager.java | 70 ++--- .../core/minion/MinionInstancesCleanupTask.java| 4 +- .../helix/core/minion/TaskMetricsEmitter.java | 20 +- .../helix/core/util/ZKMetadataUtils.java | 105 +--- .../pinot/controller/api/TableViewsTest.java | 9 +- .../helix/core/retention/RetentionManagerTest.java | 38 ++- .../validation/ValidationManagerTest.java | 3 +-
[pinot] branch pinot-flexible-auth-provider updated (15acfde711 -> a494d80a76)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git from 15acfde711 review cleanup add a494d80a76 review fixes No new revisions were added by this update. Summary of changes: .../pinot/common/auth/AuthProviderUtils.java | 9 --- .../pinot/common/auth/StaticTokenAuthProvider.java | 30 +++--- .../core/data/manager/BaseTableDataManager.java| 14 -- .../manager/offline/TableDataManagerProvider.java | 6 ++--- .../BaseTableDataManagerAcquireSegmentTest.java| 2 +- .../data/manager/BaseTableDataManagerTest.java | 2 +- .../offline/DimensionTableDataManagerTest.java | 2 +- .../realtime/LLRealtimeSegmentDataManagerTest.java | 2 +- .../executor/QueryExecutorExceptionsTest.java | 2 +- .../core/query/executor/QueryExecutorTest.java | 2 +- .../queries/SegmentWithNullValueVectorTest.java| 2 +- .../tests/BasicAuthBatchIntegrationTest.java | 2 +- .../BaseMultipleSegmentsConversionExecutor.java| 4 +-- .../tasks/BaseSingleSegmentConversionExecutor.java | 2 +- .../local/data/manager/TableDataManager.java | 3 +-- .../local/data/manager/TableDataManagerConfig.java | 8 ++ .../segment/local/utils/SegmentPushUtils.java | 6 ++--- .../starter/helix/HelixInstanceDataManager.java| 7 + .../apache/pinot/server/api/BaseResourceTest.java | 2 +- .../org/apache/pinot/tools/AuthQuickstart.java | 2 +- .../admin/command/AbstractBaseAdminCommand.java| 2 +- 21 files changed, 62 insertions(+), 49 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch pinot-flexible-auth-provider updated (3d5f10c2f8 -> 15acfde711)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git from 3d5f10c2f8 test cleanup add 15acfde711 review cleanup No new revisions were added by this update. Summary of changes: .../java/org/apache/pinot/common/auth/AuthConfig.java | 5 + .../org/apache/pinot/common/auth/NullAuthProvider.java | 3 +++ .../pinot/common/auth/StaticTokenAuthProvider.java | 3 +++ .../org/apache/pinot/common/auth/UrlAuthProvider.java | 5 + .../controller/api/access/AuthenticationFilter.java | 3 +-- .../local/data/manager/TableDataManagerConfig.java | 7 --- .../starter/helix/HelixInstanceDataManagerConfig.java | 17 - .../spi/config/instance/InstanceDataManagerConfig.java | 2 -- .../tools/admin/command/AbstractBaseAdminCommand.java | 3 ++- 9 files changed, 19 insertions(+), 29 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 02/02: test cleanup
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch pinot-flexible-auth-provider
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 3d5f10c2f83e26b12a9b6b0d574ad32211b0
Author: Alexander Pucher
AuthorDate: Tue May 10 01:11:24 2022 -0700
test cleanup
---
.../org/apache/pinot/common/auth/AuthProviderUtils.java | 15 ++-
.../pinot/common/utils/FileUploadDownloadClient.java | 2 +-
.../integration/tests/BasicAuthBatchIntegrationTest.java | 2 +-
.../pinot/integration/tests/TlsIntegrationTest.java | 7 +++
.../integration/tests/UrlAuthRealtimeIntegrationTest.java | 15 +++
.../src/test/resources/url-auth-token-prefixed.txt| 1 +
.../src/test/resources/url-auth-token.txt | 1 +
.../tasks/BaseMultipleSegmentsConversionExecutor.java | 4 ++--
.../minion/tasks/BaseSingleSegmentConversionExecutor.java | 2 +-
.../apache/pinot/segment/local/utils/IngestionUtils.java | 2 +-
.../pinot/segment/local/utils/SegmentPushUtils.java | 6 +++---
.../main/java/org/apache/pinot/tools/AuthQuickstart.java | 2 +-
.../java/org/apache/pinot/tools/BootstrapTableTool.java | 2 +-
.../pinot/tools/admin/command/AddSchemaCommand.java | 4
.../apache/pinot/tools/admin/command/AddTableCommand.java | 12 +---
.../pinot/tools/admin/command/AddTenantCommand.java | 7 +--
.../pinot/tools/admin/command/ImportDataCommand.java | 10 --
.../tools/admin/command/OperateClusterConfigCommand.java | 5 -
.../pinot/tools/admin/command/PostQueryCommand.java | 10 --
.../pinot/tools/admin/command/UploadSegmentCommand.java | 10 --
20 files changed, 47 insertions(+), 72 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
index 488c68841f..ecf5323cc8 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
@@ -60,16 +60,16 @@ public final class AuthProviderUtils {
* @return auth provider
*/
public static AuthProvider extractAuthProvider(PinotConfiguration
pinotConfig, String namespace) {
-return makeProvider(extractAuthConfig(pinotConfig, namespace));
+return makeDynamicProvider(extractAuthConfig(pinotConfig, namespace));
}
/**
- * Create auth provider based on the availability of a static token only, if
any.
+ * Create auth provider based on the availability of a static token only, if
any. This typically applies to task specs
*
* @param authToken static auth token
* @return auth provider
*/
- public static AuthProvider makeProvider(String authToken) {
+ public static AuthProvider makeStaticProvider(String authToken) {
if (StringUtils.isBlank(authToken)) {
return new NullAuthProvider();
}
@@ -83,7 +83,7 @@ public final class AuthProviderUtils {
* @param authConfig auth config
* @return auth provider
*/
- public static AuthProvider makeProvider(AuthConfig authConfig) {
+ public static AuthProvider makeDynamicProvider(AuthConfig authConfig) {
if (authConfig == null) {
return new NullAuthProvider();
}
@@ -99,10 +99,15 @@ public final class AuthProviderUtils {
}
}
+// mimic legacy behavior for "auth.token" property
if (authConfig.getProperties().containsKey(StaticTokenAuthProvider.TOKEN))
{
return new StaticTokenAuthProvider(authConfig);
}
+if (!authConfig.getProperties().isEmpty()) {
+ throw new IllegalArgumentException("Some auth properties defined, but no
provider created. Aborting.");
+}
+
return new NullAuthProvider();
}
@@ -136,7 +141,7 @@ public final class AuthProviderUtils {
* @param authProvider auth provider
* @return static token
*/
- public static String toTaskToken(@Nullable AuthProvider authProvider) {
+ public static String toStaticToken(@Nullable AuthProvider authProvider) {
if (authProvider == null) {
return null;
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
index 01d461782e..ac4413ff35 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUploadDownloadClient.java
@@ -937,7 +937,7 @@ public class FileUploadDownloadClient implements
AutoCloseable {
/**
* Deprecated due to lack of auth header support. May break for deployments
with auth enabled
*
- * Download a file using default settings, with an optional auth token
+ * Download a file using default setti
[pinot] branch pinot-flexible-auth-provider updated (8ecba358ca -> 3d5f10c2f8)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch pinot-flexible-auth-provider
in repository https://gitbox.apache.org/repos/asf/pinot.git
from 8ecba358ca auth provider round 2
new 829995430b integration test
new 3d5f10c2f8 test cleanup
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../pinot/common/auth/AuthProviderUtils.java | 15 --
.../common/utils/FileUploadDownloadClient.java | 2 +-
.../tests/BasicAuthBatchIntegrationTest.java | 2 +-
.../integration/tests/TlsIntegrationTest.java | 7 +++
...st.java => UrlAuthRealtimeIntegrationTest.java} | 55 ++
.../src/test/resources/url-auth-token-prefixed.txt | 1 +
.../src/test/resources/url-auth-token.txt | 1 +
.../BaseMultipleSegmentsConversionExecutor.java| 4 +-
.../tasks/BaseSingleSegmentConversionExecutor.java | 2 +-
.../pinot/segment/local/utils/IngestionUtils.java | 2 +-
.../segment/local/utils/SegmentPushUtils.java | 6 +--
.../org/apache/pinot/tools/AuthQuickstart.java | 2 +-
.../org/apache/pinot/tools/BootstrapTableTool.java | 2 +-
.../tools/admin/command/AddSchemaCommand.java | 4 --
.../pinot/tools/admin/command/AddTableCommand.java | 12 +
.../tools/admin/command/AddTenantCommand.java | 7 +--
.../tools/admin/command/ImportDataCommand.java | 10
.../admin/command/OperateClusterConfigCommand.java | 5 --
.../tools/admin/command/PostQueryCommand.java | 10
.../tools/admin/command/UploadSegmentCommand.java | 10
20 files changed, 78 insertions(+), 81 deletions(-)
rename
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/{BasicAuthRealtimeIntegrationTest.java
=> UrlAuthRealtimeIntegrationTest.java} (70%)
create mode 100644
pinot-integration-tests/src/test/resources/url-auth-token-prefixed.txt
create mode 100644
pinot-integration-tests/src/test/resources/url-auth-token.txt
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/02: integration test
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch pinot-flexible-auth-provider
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 829995430b512fc24f69fe3ecf243013b5acdeb1
Author: Alexander Pucher
AuthorDate: Tue May 10 00:23:28 2022 -0700
integration test
---
...st.java => UrlAuthRealtimeIntegrationTest.java} | 40 +-
1 file changed, 31 insertions(+), 9 deletions(-)
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasicAuthRealtimeIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UrlAuthRealtimeIntegrationTest.java
similarity index 78%
rename from
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasicAuthRealtimeIntegrationTest.java
rename to
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UrlAuthRealtimeIntegrationTest.java
index baa9972644..3e165d6b64 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasicAuthRealtimeIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UrlAuthRealtimeIntegrationTest.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import groovy.lang.IntRange;
import java.io.File;
import java.io.IOException;
+import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -48,11 +49,11 @@ import org.testng.annotations.Test;
import static
org.apache.pinot.integration.tests.BasicAuthTestUtils.AUTH_HEADER;
-/**
- * NOTE: fully covered by TlsIntegrationTest. If that one fails for realtime
segments try this one to isolate any TLS
- * related issues.
- */
-public class BasicAuthRealtimeIntegrationTest extends
BaseClusterIntegrationTest {
+public class UrlAuthRealtimeIntegrationTest extends BaseClusterIntegrationTest
{
+ final static URL AUTH_URL =
UrlAuthRealtimeIntegrationTest.class.getResource("/url-auth-token.txt");
+ final static URL AUTH_URL_PREFIXED =
UrlAuthRealtimeIntegrationTest.class.getResource("/url-auth-token-prefixed.txt");
+ final static String AUTH_PREFIX = "Basic";
+
@BeforeClass
public void setUp()
throws Exception {
@@ -95,22 +96,43 @@ public class BasicAuthRealtimeIntegrationTest extends
BaseClusterIntegrationTest
@Override
public Map getDefaultControllerConfiguration() {
-return
BasicAuthTestUtils.addControllerConfiguration(super.getDefaultControllerConfiguration());
+Map conf =
BasicAuthTestUtils.addControllerConfiguration(super.getDefaultControllerConfiguration());
+conf.put("controller.segment.fetcher.auth.url", AUTH_URL);
+conf.put("controller.segment.fetcher.auth.prefix", AUTH_PREFIX);
+
+return conf;
}
@Override
protected PinotConfiguration getDefaultBrokerConfiguration() {
-return
BasicAuthTestUtils.addBrokerConfiguration(super.getDefaultBrokerConfiguration().toMap());
+PinotConfiguration conf =
BasicAuthTestUtils.addBrokerConfiguration(super.getDefaultBrokerConfiguration().toMap());
+// no customization yet
+
+return conf;
}
@Override
protected PinotConfiguration getDefaultServerConfiguration() {
-return
BasicAuthTestUtils.addServerConfiguration(super.getDefaultServerConfiguration().toMap());
+PinotConfiguration conf =
BasicAuthTestUtils.addServerConfiguration(super.getDefaultServerConfiguration().toMap());
+conf.setProperty("pinot.server.segment.fetcher.auth.url", AUTH_URL);
+conf.setProperty("pinot.server.segment.fetcher.auth.prefix", AUTH_PREFIX);
+conf.setProperty("pinot.server.segment.uploader.auth.url", AUTH_URL);
+conf.setProperty("pinot.server.segment.uploader.auth.prefix", AUTH_PREFIX);
+conf.setProperty("pinot.server.instance.auth.url", AUTH_URL);
+conf.setProperty("pinot.server.instance.auth.prefix", AUTH_PREFIX);
+
+return conf;
}
@Override
protected PinotConfiguration getDefaultMinionConfiguration() {
-return
BasicAuthTestUtils.addMinionConfiguration(super.getDefaultMinionConfiguration().toMap());
+PinotConfiguration conf =
BasicAuthTestUtils.addMinionConfiguration(super.getDefaultMinionConfiguration().toMap());
+conf.setProperty("segment.fetcher.auth.url", AUTH_URL_PREFIXED);
+conf.setProperty("segment.fetcher.auth.prefix", AUTH_PREFIX);
+conf.setProperty("task.auth.url", AUTH_URL_PREFIXED);
+conf.setProperty("task.auth.prefix", AUTH_PREFIX);
+
+return conf;
}
@Override
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 02/02: auth provider round 2
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch pinot-flexible-auth-provider
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 8ecba358ca24b4536cc4a42e5372cbda3d89df9f
Author: Alexander Pucher
AuthorDate: Tue May 10 00:05:38 2022 -0700
auth provider round 2
---
.../{NullAuthProvider.java => AuthConfig.java} | 17 ++--
.../pinot/common/auth/AuthProviderUtils.java | 110 +++--
.../apache/pinot/common/auth/NullAuthProvider.java | 15 ++-
.../pinot/common/auth/StaticTokenAuthProvider.java | 35 +--
.../apache/pinot/common/auth/UrlAuthProvider.java | 39 +---
.../common/utils/FileUploadDownloadClient.java | 4 +-
.../common/utils/fetcher/BaseSegmentFetcher.java | 4 +-
.../utils/fetcher/SegmentFetcherFactory.java | 19 ++--
.../apache/pinot/common/utils/http/HttpClient.java | 10 +-
.../resources/PinotIngestionRestletResource.java | 13 +--
.../core/data/manager/BaseTableDataManager.java| 6 +-
.../manager/offline/TableDataManagerProvider.java | 6 +-
.../realtime/Server2ControllerSegmentUploader.java | 2 +-
.../ServerSegmentCompletionProtocolHandler.java| 7 +-
.../BaseTableDataManagerAcquireSegmentTest.java| 2 +-
.../data/manager/BaseTableDataManagerTest.java | 2 +-
.../offline/DimensionTableDataManagerTest.java | 2 +-
.../realtime/LLRealtimeSegmentDataManagerTest.java | 2 +-
.../executor/QueryExecutorExceptionsTest.java | 2 +-
.../core/query/executor/QueryExecutorTest.java | 2 +-
.../queries/SegmentWithNullValueVectorTest.java| 2 +-
.../tests/BasicAuthBatchIntegrationTest.java | 3 +-
.../BaseMultipleSegmentsConversionExecutor.java| 7 +-
.../tasks/BaseSingleSegmentConversionExecutor.java | 4 +-
.../local/data/manager/TableDataManager.java | 3 +-
.../local/data/manager/TableDataManagerConfig.java | 20 ++--
.../pinot/segment/local/utils/IngestionUtils.java | 5 +-
.../segment/local/utils/SegmentPushUtils.java | 14 +--
.../starter/helix/HelixInstanceDataManager.java| 10 +-
.../helix/HelixInstanceDataManagerConfig.java | 29 +++---
.../apache/pinot/server/api/BaseResourceTest.java | 2 +-
.../org/apache/pinot/spi/auth/AuthProvider.java| 4 +-
.../config/instance/InstanceDataManagerConfig.java | 6 +-
.../apache/pinot/spi/utils/CommonConstants.java| 8 +-
.../org/apache/pinot/tools/AuthQuickstart.java | 6 +-
.../org/apache/pinot/tools/BootstrapTableTool.java | 13 ++-
.../org/apache/pinot/tools/EmptyQuickstart.java| 5 +-
.../java/org/apache/pinot/tools/Quickstart.java| 5 +-
.../admin/command/AbstractBaseAdminCommand.java| 32 --
.../tools/admin/command/AddSchemaCommand.java | 12 ++-
.../pinot/tools/admin/command/AddTableCommand.java | 15 ++-
.../tools/admin/command/AddTenantCommand.java | 11 ++-
.../tools/admin/command/BootstrapTableCommand.java | 14 ++-
.../tools/admin/command/ChangeTableState.java | 12 ++-
.../tools/admin/command/ImportDataCommand.java | 11 ++-
.../command/LaunchDataIngestionJobCommand.java | 10 +-
.../admin/command/OperateClusterConfigCommand.java | 11 ++-
.../tools/admin/command/PostQueryCommand.java | 12 ++-
.../tools/admin/command/QuickstartRunner.java | 28 ++
.../tools/admin/command/UploadSegmentCommand.java | 16 ++-
50 files changed, 396 insertions(+), 233 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/auth/NullAuthProvider.java
b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthConfig.java
similarity index 74%
copy from
pinot-common/src/main/java/org/apache/pinot/common/auth/NullAuthProvider.java
copy to pinot-common/src/main/java/org/apache/pinot/common/auth/AuthConfig.java
index 80756f3aae..1da479b0dc 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/auth/NullAuthProvider.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthConfig.java
@@ -18,14 +18,19 @@
*/
package org.apache.pinot.common.auth;
-import java.util.Collections;
import java.util.Map;
-import org.apache.pinot.spi.auth.AuthProvider;
-public class NullAuthProvider implements AuthProvider {
- @Override
- public Map getHttpHeaders() {
-return Collections.emptyMap();
+public class AuthConfig {
+ public static final String PROVIDER_CLASS = "provider.class";
+
+ protected Map _properties;
+
+ public AuthConfig(Map properties) {
+_properties = properties;
+ }
+
+ public Map getProperties() {
+return _properties;
}
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
index c8bac0593f..488c68841f 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
+++
b/pinot-common/src/main/java/org/apache
[pinot] branch pinot-flexible-auth-provider created (now 8ecba358ca)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch pinot-flexible-auth-provider in repository https://gitbox.apache.org/repos/asf/pinot.git at 8ecba358ca auth provider round 2 This branch includes the following new commits: new c20007e6dc pinot auth provider draft new 8ecba358ca auth provider round 2 The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/02: pinot auth provider draft
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch pinot-flexible-auth-provider
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit c20007e6dc95581cc3acb17fb0969cc721ca4238
Author: Alexander Pucher
AuthorDate: Mon May 9 18:03:08 2022 -0700
pinot auth provider draft
---
.../pinot/common/auth/AuthProviderUtils.java | 117 +
.../apache/pinot/common/auth/NullAuthProvider.java | 19 ++--
.../pinot/common/auth/StaticTokenAuthProvider.java | 34 --
.../apache/pinot/common/auth/UrlAuthProvider.java | 65
.../common/utils/FileUploadDownloadClient.java | 43
.../common/utils/fetcher/BaseSegmentFetcher.java | 7 +-
.../common/utils/fetcher/HttpSegmentFetcher.java | 4 +-
.../utils/fetcher/SegmentFetcherFactory.java | 6 ++
.../apache/pinot/common/utils/http/HttpClient.java | 70 +---
.../api/access/AuthenticationFilter.java | 3 +-
.../resources/PinotIngestionRestletResource.java | 11 +-
.../pinot/controller/util/FileIngestionHelper.java | 10 +-
.../core/data/manager/BaseTableDataManager.java| 7 +-
.../manager/realtime/SegmentCommitterFactory.java | 2 +-
.../realtime/Server2ControllerSegmentUploader.java | 11 +-
.../ServerSegmentCompletionProtocolHandler.java| 19 ++--
.../tests/BasicAuthBatchIntegrationTest.java | 3 +-
.../BaseMultipleSegmentsConversionExecutor.java| 20 ++--
.../tasks/BaseSingleSegmentConversionExecutor.java | 7 +-
.../minion/tasks/SegmentConversionUtils.java | 17 +--
.../segmentuploader/SegmentUploaderDefault.java| 10 +-
.../local/data/manager/TableDataManagerConfig.java | 6 ++
.../pinot/segment/local/utils/IngestionUtils.java | 16 +--
.../segment/local/utils/SegmentPushUtils.java | 12 ++-
.../helix/HelixInstanceDataManagerConfig.java | 7 ++
.../auth/{AuthContext.java => AuthProvider.java} | 21 ++--
.../config/instance/InstanceDataManagerConfig.java | 2 +
.../batch/spec/SegmentGenerationJobSpec.java | 6 ++
.../segment/uploader/SegmentUploader.java | 10 +-
.../apache/pinot/spi/utils/CommonConstants.java| 2 +
.../org/apache/pinot/tools/BootstrapTableTool.java | 10 +-
.../admin/command/AbstractBaseAdminCommand.java| 7 +-
.../tools/admin/command/AddSchemaCommand.java | 6 +-
.../pinot/tools/admin/command/AddTableCommand.java | 13 ++-
.../tools/admin/command/AddTenantCommand.java | 5 +-
.../tools/admin/command/BootstrapTableCommand.java | 6 +-
.../tools/admin/command/ChangeTableState.java | 10 +-
.../tools/admin/command/ImportDataCommand.java | 11 +-
.../command/LaunchDataIngestionJobCommand.java | 5 +-
.../admin/command/OperateClusterConfigCommand.java | 5 +-
.../tools/admin/command/PostQueryCommand.java | 11 +-
.../tools/admin/command/QuickstartRunner.java | 19 +++-
.../tools/admin/command/UploadSegmentCommand.java | 14 ++-
43 files changed, 493 insertions(+), 196 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
new file mode 100644
index 00..c8bac0593f
--- /dev/null
+++
b/pinot-common/src/main/java/org/apache/pinot/common/auth/AuthProviderUtils.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.auth;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
+import org.apache.pinot.spi.auth.AuthProvider;
+
+
+/**
+ * Utility class to wrap inference of optimal auth provider from component
configs.
+ */
+public final class AuthProviderUtils {
+ private AuthProviderUtils() {
+// left blank
+ }
+
+ /**
+ * Infer optimal auth provider based on the availability of static token, if
any.
+ *
+ * @param authToken static a
[pinot] branch revert-awesome-pinot created (now 04a24a1)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch revert-awesome-pinot in repository https://gitbox.apache.org/repos/asf/pinot.git. at 04a24a1 Revert "first commit" This branch includes the following new commits: new 04a24a1 Revert "first commit" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: Revert "first commit"
This is an automated email from the ASF dual-hosted git repository. apucher pushed a commit to branch revert-awesome-pinot in repository https://gitbox.apache.org/repos/asf/pinot.git commit 04a24a137aee66b95abd1b8498ee45b3190ebdaf Author: Alexander Pucher AuthorDate: Mon Feb 28 15:11:35 2022 -0800 Revert "first commit" This reverts commit be1a67c036cf018d6d58c9bd21b309ae37e5edd8. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index cab2e03..bb49a72 100644 --- a/README.md +++ b/README.md @@ -124,4 +124,3 @@ Check out [Pinot documentation](https://docs.pinot.apache.org/) for a complete d ## License Apache Pinot is under [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) -# awesome-pinot - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch main created (now be1a67c)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/pinot.git. at be1a67c first commit This branch includes the following new commits: new be1a67c first commit The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: first commit
This is an automated email from the ASF dual-hosted git repository. apucher pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/pinot.git commit be1a67c036cf018d6d58c9bd21b309ae37e5edd8 Author: Alexander Pucher AuthorDate: Mon Feb 28 15:07:17 2022 -0800 first commit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bb49a72..cab2e03 100644 --- a/README.md +++ b/README.md @@ -124,3 +124,4 @@ Check out [Pinot documentation](https://docs.pinot.apache.org/) for a complete d ## License Apache Pinot is under [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) +# awesome-pinot - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: fix controller config validation failure for customized TLS listeners (#8106)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d08c7f8 fix controller config validation failure for customized TLS
listeners (#8106)
d08c7f8 is described below
commit d08c7f8841360f46dc9ed68e07bf1850665c09d7
Author: Alexander Pucher
AuthorDate: Tue Feb 1 18:59:49 2022 -0800
fix controller config validation failure for customized TLS listeners
(#8106)
This is a fix of a config validation error when using the new TLS listerner
specs intoduced in #8082
---
.../integration/tests/TlsIntegrationTest.java | 33 +-
.../apache/pinot/tools/utils/PinotConfigUtils.java | 20 +++--
2 files changed, 44 insertions(+), 9 deletions(-)
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
index 53b09a7..2070aa0 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
@@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.io.FileUtils;
import org.apache.http.Header;
@@ -45,6 +46,7 @@ import
org.apache.pinot.client.JsonAsyncHttpPinotClientTransportFactory;
import org.apache.pinot.client.Request;
import org.apache.pinot.client.ResultSetGroup;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
+import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.core.common.MinionConstants;
import
org.apache.pinot.integration.tests.access.CertBasedTlsChannelAccessControlFactory;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -54,6 +56,7 @@ import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.apache.pinot.tools.utils.PinotConfigUtils;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
@@ -145,7 +148,7 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
prop.put("controller.broker.protocol", "https");
-// announce external only
+// announce internal only
prop.put("controller.vip.protocol", "https");
prop.put("controller.vip.port", DEFAULT_CONTROLLER_PORT);
@@ -274,6 +277,34 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
}
@Test
+ public void testControllerConfigValidation()
+ throws Exception {
+PinotConfigUtils.validateControllerConfig(new
ControllerConf(getDefaultControllerConfiguration()));
+ }
+
+ @Test
+ public void testControllerConfigValidationImplicitProtocol()
+ throws Exception {
+Map prop = new
HashMap<>(getDefaultControllerConfiguration());
+prop.put("controller.access.protocols", "https,http");
+prop.put("controller.access.protocols.https.port",
DEFAULT_CONTROLLER_PORT);
+prop.put("controller.access.protocols.http.port",
EXTERNAL_CONTROLLER_PORT);
+
+PinotConfigUtils.validateControllerConfig(new ControllerConf(prop));
+ }
+
+ @Test(expectedExceptions = ConfigurationException.class)
+ public void testControllerConfigValidationNoProtocol()
+ throws Exception {
+Map prop = new
HashMap<>(getDefaultControllerConfiguration());
+prop.put("controller.access.protocols", "invalid,http");
+prop.put("controller.access.protocols.invalid.port",
DEFAULT_CONTROLLER_PORT);
+prop.put("controller.access.protocols.http.port",
EXTERNAL_CONTROLLER_PORT);
+
+PinotConfigUtils.validateControllerConfig(new ControllerConf(prop));
+ }
+
+ @Test
public void testControllerExternalTrustedServer()
throws Exception {
try (CloseableHttpClient client = makeClient(JKS, _tlsStoreJKS,
_tlsStoreJKS)) {
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/utils/PinotConfigUtils.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/PinotConfigUtils.java
index 41244bb..239a19b 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/utils/PinotConfigUtils.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/PinotConfigUtils.java
@@ -217,18 +217,18 @@ public class PinotConfigUtils {
private static List validateControllerAccessProtoco
[pinot] branch listener-tls-customization-validation-fixes updated (4c742fb -> 4c743b8)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch listener-tls-customization-validation-fixes in repository https://gitbox.apache.org/repos/asf/pinot.git. from 4c742fb fix config config validation failure for custom TLS listeners add 4c743b8 reduce confusion by reverting param name change No new revisions were added by this update. Summary of changes: .../java/org/apache/pinot/controller/ControllerConf.java | 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: fix config config validation failure for custom TLS listeners
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch listener-tls-customization-validation-fixes
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 4c742fb0b2dab6799c72b381e3cc5d75e8276ab0
Author: Alexander Pucher
AuthorDate: Tue Feb 1 14:41:32 2022 -0800
fix config config validation failure for custom TLS listeners
---
.../apache/pinot/controller/ControllerConf.java| 12
.../integration/tests/TlsIntegrationTest.java | 33 +-
.../apache/pinot/tools/utils/PinotConfigUtils.java | 20 +++--
3 files changed, 50 insertions(+), 15 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
index fbf827d..bea3806 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
@@ -379,16 +379,16 @@ public class ControllerConf extends PinotConfiguration {
getControllerPort() == null ? Arrays.asList("http") : Arrays.asList());
}
- public String getControllerAccessProtocolProperty(String protocol, String
property) {
-return getProperty(CONTROLLER_ACCESS_PROTOCOLS + "." + protocol + "." +
property);
+ public String getControllerAccessProtocolProperty(String name, String
property) {
+return getProperty(CONTROLLER_ACCESS_PROTOCOLS + "." + name + "." +
property);
}
- public String getControllerAccessProtocolProperty(String protocol, String
property, String defaultValue) {
-return getProperty(CONTROLLER_ACCESS_PROTOCOLS + "." + protocol + "." +
property, defaultValue);
+ public String getControllerAccessProtocolProperty(String name, String
property, String defaultValue) {
+return getProperty(CONTROLLER_ACCESS_PROTOCOLS + "." + name + "." +
property, defaultValue);
}
- public boolean getControllerAccessProtocolProperty(String protocol, String
property, boolean defaultValue) {
-return getProperty(CONTROLLER_ACCESS_PROTOCOLS + "." + protocol + "." +
property, defaultValue);
+ public boolean getControllerAccessProtocolProperty(String name, String
property, boolean defaultValue) {
+return getProperty(CONTROLLER_ACCESS_PROTOCOLS + "." + name + "." +
property, defaultValue);
}
public String getDataDir() {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
index 53b09a7..2070aa0 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
@@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.io.FileUtils;
import org.apache.http.Header;
@@ -45,6 +46,7 @@ import
org.apache.pinot.client.JsonAsyncHttpPinotClientTransportFactory;
import org.apache.pinot.client.Request;
import org.apache.pinot.client.ResultSetGroup;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
+import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.core.common.MinionConstants;
import
org.apache.pinot.integration.tests.access.CertBasedTlsChannelAccessControlFactory;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -54,6 +56,7 @@ import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.apache.pinot.tools.utils.PinotConfigUtils;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
@@ -145,7 +148,7 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
prop.put("controller.broker.protocol", "https");
-// announce external only
+// announce internal only
prop.put("controller.vip.protocol", "https");
prop.put("controller.vip.port", DEFAULT_CONTROLLER_PORT);
@@ -274,6 +277,34 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
}
@Test
+ public void testControllerConfigValidation()
+ throws Exception {
+PinotConfigUtils.validateControllerConfig(new
ControllerConf(getDefaultControllerConfiguration()));
+ }
+
+ @Test
+ public void testControllerConfigValidationImplici
[pinot] branch listener-tls-customization-validation-fixes created (now 4c742fb)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch listener-tls-customization-validation-fixes in repository https://gitbox.apache.org/repos/asf/pinot.git. at 4c742fb fix config config validation failure for custom TLS listeners This branch includes the following new commits: new 4c742fb fix config config validation failure for custom TLS listeners The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated (2a36d4c -> 0009877)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git. from 2a36d4c Add error message if unable to create index / tmp resource dir (#8076) add 0009877 Listener TLS customization (#8082) No new revisions were added by this update. Summary of changes: .../org/apache/pinot/core/transport/TlsConfig.java | 23 +- .../apache/pinot/core/util/ListenerConfigUtil.java | 67 ++- .../java/org/apache/pinot/core/util/TlsUtils.java | 93 ++-- pinot-integration-tests/pom.xml| 2 +- .../tests/BasicAuthRealtimeIntegrationTest.java| 4 + .../tests/BasicAuthTlsRealtimeIntegrationTest.java | 302 .../integration/tests/TlsIntegrationTest.java | 507 + .../CertBasedTlsChannelAccessControlFactory.java | 3 +- .../src/test/resources/empty.jks | Bin 0 -> 32 bytes .../src/test/resources/empty.p12 | Bin 0 -> 88 bytes .../src/test/resources/tlstest.jks | Bin 2277 -> 2283 bytes .../src/test/resources/tlstest.p12 | Bin 10155 -> 2645 bytes .../apache/pinot/spi/env/PinotConfiguration.java | 3 + 13 files changed, 647 insertions(+), 357 deletions(-) delete mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasicAuthTlsRealtimeIntegrationTest.java create mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java create mode 100644 pinot-integration-tests/src/test/resources/empty.jks create mode 100644 pinot-integration-tests/src/test/resources/empty.p12 - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch listener-tls-customization updated (1148538 -> 29e4860)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch listener-tls-customization in repository https://gitbox.apache.org/repos/asf/pinot.git. from 1148538 allow null defaults add 29e4860 java8 optional method No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch listener-tls-customization updated (88f040a -> 1148538)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch listener-tls-customization in repository https://gitbox.apache.org/repos/asf/pinot.git. from 88f040a update POM add 1148538 allow null defaults No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java | 3 +++ 1 file changed, 3 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch listener-tls-customization updated (3633704 -> 88f040a)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch listener-tls-customization in repository https://gitbox.apache.org/repos/asf/pinot.git. discard 3633704 controller working discard 7a18a16 prototype add c8dde9b allow JSON forward indexes (#8073) add 4d10e03 Improve the proactive segment clean-up for REVERTED (#8071) add 18777b7 Adding timestamp into schema test (#8080) add 0fe7ef8 Remove PQL response parser in quickstart (#8081) add 3f11902 also add schema test in fields for default values and others (#8088) add dd73ee7 add logic to instant delete segment (#8077) add 41dec36 prototype add 794d976 controller working add 0689778 extend tls integration tests, depraecate BasicAuthTlsRealtimeIntegrationTest add 52f058c checkstyle add 9d4950c avoid caching keystore if local file add 88f040a update POM This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (3633704) \ N -- N -- N refs/heads/listener-tls-customization (88f040a) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../org/apache/pinot/common/data/SchemaTest.java | 29 ++- pinot-common/src/test/resources/schemaTest.schema | 6 + .../helix/core/PinotHelixResourceManager.java | 25 +- .../helix/core/SegmentDeletionManager.java | 58 +++-- .../helix/core/PinotHelixResourceManagerTest.java | 64 - .../core/util/SegmentDeletionManagerTest.java | 4 +- .../apache/pinot/core/util/ListenerConfigUtil.java | 20 +- .../java/org/apache/pinot/core/util/TlsUtils.java | 32 ++- pinot-integration-tests/pom.xml| 2 +- .../tests/BasicAuthRealtimeIntegrationTest.java| 4 + .../tests/BasicAuthTlsRealtimeIntegrationTest.java | 261 - .../integration/tests/TlsIntegrationTest.java | 210 +++-- .../src/test/resources/empty.p12 | Bin 103 -> 88 bytes .../creator/impl/SegmentColumnarIndexCreator.java | 7 + .../java/org/apache/pinot/tools/Quickstart.java| 61 - 15 files changed, 381 insertions(+), 402 deletions(-) delete mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BasicAuthTlsRealtimeIntegrationTest.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/02: prototype
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch listener-tls-customization
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 7a18a16e9fe160130f9a72864dbb7938963e0d4f
Author: Alexander Pucher
AuthorDate: Thu Jan 27 15:32:11 2022 -0800
prototype
---
.../org/apache/pinot/core/transport/TlsConfig.java | 23 +-
.../apache/pinot/core/util/ListenerConfigUtil.java | 59 +++-
.../java/org/apache/pinot/core/util/TlsUtils.java | 77 +++--
.../tests/BasicAuthTlsRealtimeIntegrationTest.java | 83 ++---
.../integration/tests/TlsIntegrationTest.java | 354 +
.../CertBasedTlsChannelAccessControlFactory.java | 3 +-
.../src/test/resources/empty.jks | Bin 0 -> 32 bytes
.../src/test/resources/empty.p12 | Bin 0 -> 103 bytes
.../src/test/resources/tlstest.jks | Bin 2277 -> 2283 bytes
.../src/test/resources/tlstest.p12 | Bin 10155 -> 2645 bytes
10 files changed, 487 insertions(+), 112 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/transport/TlsConfig.java
b/pinot-core/src/main/java/org/apache/pinot/core/transport/TlsConfig.java
index c8de43b..9b86c34 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/transport/TlsConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/TlsConfig.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.core.transport;
+import io.netty.handler.ssl.SslProvider;
+import java.security.KeyStore;
import org.apache.commons.lang3.StringUtils;
@@ -26,13 +28,28 @@ import org.apache.commons.lang3.StringUtils;
*/
public class TlsConfig {
private boolean _clientAuthEnabled;
- private String _keyStoreType;
+ private String _keyStoreType = KeyStore.getDefaultType();
private String _keyStorePath;
private String _keyStorePassword;
- private String _trustStoreType;
+ private String _trustStoreType = KeyStore.getDefaultType();
private String _trustStorePath;
private String _trustStorePassword;
- private String _sslProvider;
+ private String _sslProvider = SslProvider.JDK.toString();
+
+ public TlsConfig() {
+// left blank
+ }
+
+ public TlsConfig(TlsConfig tlsConfig) {
+_clientAuthEnabled = tlsConfig._clientAuthEnabled;
+_keyStoreType = tlsConfig._keyStoreType;
+_keyStorePath = tlsConfig._keyStorePath;
+_keyStorePassword = tlsConfig._keyStorePassword;
+_trustStoreType = tlsConfig._trustStoreType;
+_trustStorePath = tlsConfig._trustStorePath;
+_trustStorePassword = tlsConfig._trustStorePassword;
+_sslProvider = tlsConfig._sslProvider;
+ }
public boolean isClientAuthEnabled() {
return _clientAuthEnabled;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
index 0211c29..0fd1143 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
@@ -19,7 +19,14 @@
package org.apache.pinot.core.util;
import com.google.common.base.Preconditions;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.net.URI;
+import java.net.URL;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -28,6 +35,7 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.transport.TlsConfig;
@@ -76,9 +84,7 @@ public final class ListenerConfigUtil {
String[] protocols = config.getProperty(namespace +
DOT_ACCESS_PROTOCOLS).split(",");
-return Arrays.stream(protocols).peek(protocol -> Preconditions
-.checkArgument(SUPPORTED_PROTOCOLS.contains(protocol), "Unsupported
protocol '%s' in config namespace '%s'",
-protocol, namespace)).map(protocol -> buildListenerConfig(config,
namespace, protocol, tlsDefaults))
+return Arrays.stream(protocols).map(protocol ->
buildListenerConfig(config, namespace, protocol, tlsDefaults))
.collect(Collectors.toList());
}
@@ -167,24 +173,35 @@ public final class ListenerConfigUtil {
return listeners;
}
- private static ListenerConfig buildListenerConfig(PinotConfiguration config,
String namespace, String protocol,
+ private static ListenerConfig buildListenerConfig(PinotConfiguration config,
String namespace, String name,
TlsConfig tlsConfig) {
-String protocolNamespace = namespace + DOT_ACCESS_PROTOCOLS + "." +
protocol;
+String protocolNamespace = namespace
[pinot] 02/02: controller working
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch listener-tls-customization
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 363370412f6f477a3a85a432596c90fd0e796308
Author: Alexander Pucher
AuthorDate: Thu Jan 27 16:29:45 2022 -0800
controller working
---
.../apache/pinot/core/util/ListenerConfigUtil.java | 4 +-
.../integration/tests/TlsIntegrationTest.java | 89 +-
2 files changed, 39 insertions(+), 54 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
index 0fd1143..018072a 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
@@ -179,7 +179,7 @@ public final class ListenerConfigUtil {
return new ListenerConfig(name,
getHost(config.getProperty(protocolNamespace + ".host", DEFAULT_HOST)),
getPort(config.getProperty(protocolNamespace + ".port")),
getProtocol(config.getProperty(protocolNamespace + ".protocol"), name),
-TlsUtils.extractTlsConfig(config, namespace + ".tls", tlsConfig));
+TlsUtils.extractTlsConfig(config, protocolNamespace + ".tls",
tlsConfig));
}
private static String getHost(String configuredHost) {
@@ -263,7 +263,7 @@ public final class ListenerConfigUtil {
private static File cacheInTempFile(String sourceUrl) {
try {
- File tempFile = Files.createTempFile("keystore", "cache").toFile();
+ File tempFile = Files.createTempFile("pinot-keystore-", null).toFile();
tempFile.deleteOnExit();
try (InputStream is = TlsUtils.makeKeyStoreUrl(sourceUrl).openStream();
OutputStream os = new FileOutputStream(tempFile)) {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
index a6e3c5c..edc8dd2 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TlsIntegrationTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.integration.tests;
+import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
import java.net.URL;
@@ -43,6 +44,7 @@ import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
@@ -59,8 +61,8 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
private static final char[] PASSWORD_CHAR = PASSWORD.toCharArray();
private static final Header CLIENT_HEADER = new BasicHeader("Authorization",
AUTH_TOKEN);
- private static final int INTERNAL_CONTROLLER_PORT = DEFAULT_CONTROLLER_PORT
+ 1;
- private static final int INTERNAL_BROKER_PORT = DEFAULT_BROKER_PORT + 1;
+ private static final int EXTERNAL_CONTROLLER_PORT = DEFAULT_CONTROLLER_PORT
+ 1;
+ private static final int EXTERNAL_BROKER_PORT = DEFAULT_BROKER_PORT + 1;
private static final String PKCS_12 = "PKCS12";
private static final String JKS = "JKS";
@@ -93,10 +95,6 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
// Push data into Kafka
pushAvroIntoKafka(avroFiles);
waitForAllDocsLoaded(600_000L);
-
-System.out.println("hello world!");
-
-Thread.sleep(60);
}
@AfterClass(alwaysRun = true)
@@ -121,20 +119,19 @@ public class TlsIntegrationTest extends
BaseClusterIntegrationTest {
prop.put("controller.tls.truststore.password", PASSWORD);
prop.put("controller.tls.truststore.type", PKCS_12);
-//prop.put("controller.access.protocols", "https");
-//prop.put("controller.access.protocols.https.port",
DEFAULT_CONTROLLER_PORT);
-prop.put("controller.access.protocols", "https,internal");
-prop.put("controller.access.protocols.https.port",
DEFAULT_CONTROLLER_PORT);
-prop.put("controller.access.protocols.https.tls.keystore.path",
_tlsStoreJKS);
-prop.put("controller.access.protocols.https.tls.keystore.type", JKS);
-prop.put("controller.access.protocols.https.tls.truststore.path",
_tlsStoreJKS);
-prop.put("controller.access.protocols.https.tls.truststore.type", JKS);
+// CAUTI
[pinot] branch listener-tls-customization created (now 3633704)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch listener-tls-customization in repository https://gitbox.apache.org/repos/asf/pinot.git. at 3633704 controller working This branch includes the following new commits: new 7a18a16 prototype new 3633704 controller working The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated (0df8492 -> 3c4285b)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git. from 0df8492 faster metric scans (#7920) add 3c4285b Implement OIDC auth workflow in UI (#7121) No new revisions were added by this update. Summary of changes: pinot-controller/src/main/resources/app/App.tsx| 179 - .../src/main/resources/app/app_state.ts| 4 + .../src/main/resources/app/interfaces/types.d.ts | 6 + .../src/main/resources/app/pages/LoginPage.tsx | 2 + .../main/resources/app/utils/PinotMethodUtils.ts | 23 ++- .../src/main/resources/app/utils/axios-config.ts | 15 +- 6 files changed, 184 insertions(+), 45 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch generator-sh-gracefully-handle-uppercase created (now 96ba6b6)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch generator-sh-gracefully-handle-uppercase in repository https://gitbox.apache.org/repos/asf/pinot.git. at 96ba6b6 generator.sh - gracefully handle uppercase format This branch includes the following new commits: new 96ba6b6 generator.sh - gracefully handle uppercase format The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: generator.sh - gracefully handle uppercase format
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch generator-sh-gracefully-handle-uppercase
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 96ba6b68f1f78ec283770e4eb4289f3d24efd33b
Author: Alexander Pucher
AuthorDate: Thu Dec 2 15:11:36 2021 -0800
generator.sh - gracefully handle uppercase format
---
.../org/apache/pinot/tools/admin/command/GenerateDataCommand.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java
index e73d8a1..b5cc1a8 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java
@@ -79,7 +79,7 @@ public class GenerateDataCommand extends
AbstractBaseAdminCommand implements Com
private boolean _help = false;
@CommandLine.Option(names = {"-format"}, required = false, help = true,
- description = "Output format ('avro' or 'csv').")
+ description = "Output format ('AVRO' or 'CSV').")
private String _format = FORMAT_AVRO;
@Override
@@ -147,9 +147,9 @@ public class GenerateDataCommand extends
AbstractBaseAdminCommand implements Com
final DataGenerator gen = new DataGenerator();
gen.init(spec);
-if (FORMAT_AVRO.equals(_format)) {
+if (FORMAT_AVRO.equalsIgnoreCase(_format)) {
gen.generateAvro(_numRecords, _numFiles);
-} else if (FORMAT_CSV.equals(_format)) {
+} else if (FORMAT_CSV.equalsIgnoreCase(_format)) {
gen.generateCsv(_numRecords, _numFiles);
} else {
throw new IllegalArgumentException(String.format("Invalid output format
'%s'", _format));
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: use uppercase format
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch generator-sh-fix-for-cli-parswer-changes
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 034ac6024778049f316fb884539fb06aca31899f
Author: Alexander Pucher
AuthorDate: Tue Nov 30 15:45:22 2021 -0800
use uppercase format
---
docker/images/pinot/bin/generator.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docker/images/pinot/bin/generator.sh
b/docker/images/pinot/bin/generator.sh
index 03a11ff..a0b2bcc 100755
--- a/docker/images/pinot/bin/generator.sh
+++ b/docker/images/pinot/bin/generator.sh
@@ -131,7 +131,7 @@ fi
echo "Generating data for ${TEMPLATE_NAME} in ${DATA_DIR}"
${ADMIN_PATH} GenerateData \
--numFiles 1 -numRecords $NUM_RECORDS -format csv \
+-numFiles 1 -numRecords $NUM_RECORDS -format CSV \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-schemaAnnotationFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_generator.json" \
-outDir "$DATA_DIR"
@@ -144,7 +144,7 @@ fi
echo "Creating segment for ${TEMPLATE_NAME} in ${SEGMENT_DIR}"
${ADMIN_PATH} CreateSegment \
--format csv \
+-format CSV \
-tableConfigFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json" \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-dataDir "${DATA_DIR}" \
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch generator-sh-fix-for-cli-parswer-changes created (now 034ac60)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch generator-sh-fix-for-cli-parswer-changes in repository https://gitbox.apache.org/repos/asf/pinot.git. at 034ac60 use uppercase format This branch includes the following new commits: new 034ac60 use uppercase format The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: fix obfuscator not capturing secretkey and keytab (#7794)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7c695e9 fix obfuscator not capturing secretkey and keytab (#7794)
7c695e9 is described below
commit 7c695e9adc6da23d7373bcea7f80d7ac3cbf22e3
Author: Rong Rong
AuthorDate: Thu Nov 18 12:36:59 2021 -0800
fix obfuscator not capturing secretkey and keytab (#7794)
---
.../main/java/org/apache/pinot/spi/utils/Obfuscator.java | 6 --
.../java/org/apache/pinot/spi/utils/ObfuscatorTest.java| 14 ++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
index 35d2494..cd36245 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
@@ -58,7 +58,8 @@ import java.util.stream.Stream;
public final class Obfuscator {
private static final String DEFAULT_MASKED_VALUE = "*";
private static final List DEFAULT_PATTERNS =
- Stream.of("(?i).*secret$", "(?i).*password$",
"(?i).*token$").map(Pattern::compile).collect(Collectors.toList());
+ Stream.of("(?i).*secret$", "(?i).*secret[\\s_-]*key$",
"(?i).*password$", "(?i).*keytab$", "(?i).*token$")
+ .map(Pattern::compile).collect(Collectors.toList());
private final String _maskedValue;
private final List _patterns;
@@ -93,7 +94,8 @@ public final class Obfuscator {
public JsonNode toJson(Object object) {
// NOTE: jayway json path 2.4.0 seems to have issues with '@.name' so
we'll do this manually
// as determined by a cursory and purely subjective investigation by alex
-// "$..[?(@.name =~ /password$/i || @.name =~ /secret$/i || @.name =~
/token$/i)]"
+// "$..[?(@.name =~ /password$/i || @.name =~ /secret$/i || @.name =~
/secret[\\s_-]*key$/i || @.name =~ /keytab$/i
+// || @.name =~ /token$/i)]"
try {
JsonNode node;
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
index 8d720a6..7b7f508 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
@@ -48,12 +48,26 @@ public class ObfuscatorTest {
_map.put("secret", "SECRET");
_map.put("a.secret", "SECRET");
_map.put("mysecret", "SECRET");
+_map.put("mySecret", "SECRET");
_map.put("password", "SECRET");
_map.put("a.password", "SECRET");
_map.put("mypassword", "SECRET");
+_map.put("myPassword", "SECRET");
+_map.put("keytab", "SECRET");
+_map.put("a.keytab", "SECRET");
+_map.put("mykeytab", "SECRET");
+_map.put("myKeytab", "SECRET");
_map.put("token", "SECRET");
_map.put("a.token", "SECRET");
_map.put("mytoken", "SECRET");
+_map.put("myToken", "SECRET");
+
+_map.put("secretKey", "SECRET");
+_map.put("secretkey", "SECRET");
+_map.put("secret_key", "SECRET");
+_map.put("mysecretKey", "SECRET");
+_map.put("mySecretKey", "SECRET");
+_map.put("a.secretKey", "SECRET");
_nestedMap = new HashMap<>();
_nestedMap.put("value", "VALUE");
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: correct way to send headers via url conn (#7728)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2075cbe correct way to send headers via url conn (#7728)
2075cbe is described below
commit 2075cbef39008e0faa3b4f78d9f0a2cb47b3deb0
Author: Xiaobing <61892277+klsi...@users.noreply.github.com>
AuthorDate: Mon Nov 8 22:37:06 2021 -0800
correct way to send headers via url conn (#7728)
---
.../main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
index 5c1781f..b3af753 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java
@@ -430,13 +430,12 @@ public class PerfBenchmarkDriver {
URLConnection conn = new URL(queryUrl).openConnection();
conn.setDoOutput(true);
-
+for (Map.Entry header : headers.entrySet()) {
+ conn.setRequestProperty(header.getKey(), header.getValue());
+}
try (BufferedWriter writer = new BufferedWriter(
new OutputStreamWriter(conn.getOutputStream(),
StandardCharsets.UTF_8))) {
String requestString = requestJson.toString();
- for (Map.Entry header : headers.entrySet()) {
-writer.write(String.format("%s: %s\n", header.getKey(),
header.getValue()));
- }
writer.write(requestString);
writer.flush();
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated (d8961b0 -> f20a364)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git. from d8961b0 Cleanup SegmentDirectoryLoader load (#7682) add f20a364 allow headers for query runner (#7696) No new revisions were added by this update. Summary of changes: .../java/org/apache/pinot/tools/perf/PerfBenchmarkDriver.java | 8 1 file changed, 8 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: fix segemnt uploader property in auth quickstart (#7479)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8a05ad8 fix segemnt uploader property in auth quickstart (#7479)
8a05ad8 is described below
commit 8a05ad82b36b121778c0943f3529af81511a0d1e
Author: Alexander Pucher
AuthorDate: Fri Sep 24 11:17:42 2021 -0700
fix segemnt uploader property in auth quickstart (#7479)
---
pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
index 0e40603..edc2034 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
@@ -36,7 +36,7 @@ public class AuthQuickstart extends Quickstart {
Map properties = new HashMap<>();
// controller
-properties.put("controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
+properties.put("pinot.controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
properties.put("controller.admin.access.control.factory.class",
"org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory");
properties.put("controller.admin.access.control.principals", "admin,
user");
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: fix segemnt uploader property in auth quickstart
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch auth-quickstart-property-fix
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit c147ca1440b17cf7d6b19778af87f0b9b4e4bfd1
Author: Alexander Pucher
AuthorDate: Thu Sep 23 17:29:17 2021 -0700
fix segemnt uploader property in auth quickstart
---
pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
index 0e40603..edc2034 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
@@ -36,7 +36,7 @@ public class AuthQuickstart extends Quickstart {
Map properties = new HashMap<>();
// controller
-properties.put("controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
+properties.put("pinot.controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
properties.put("controller.admin.access.control.factory.class",
"org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory");
properties.put("controller.admin.access.control.principals", "admin,
user");
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch auth-quickstart-property-fix created (now c147ca1)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch auth-quickstart-property-fix in repository https://gitbox.apache.org/repos/asf/pinot.git. at c147ca1 fix segemnt uploader property in auth quickstart This branch includes the following new commits: new c147ca1 fix segemnt uploader property in auth quickstart The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: hotfix for #7459 (#7471)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fcae6e0 hotfix for #7459 (#7471)
fcae6e0 is described below
commit fcae6e058c60a702b8d0feee7d8df0fb1a220421
Author: Rong Rong
AuthorDate: Wed Sep 22 20:17:51 2021 -0700
hotfix for #7459 (#7471)
---
.../main/java/org/apache/pinot/controller/util/FileIngestionHelper.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
index 4268ea5..7347f9c 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
@@ -106,7 +106,7 @@ public class FileIngestionHelper {
// Copy file to local working dir
File inputFile = new File(inputDir, String.format(
- "%s.%s", DATA_FILE_PREFIX,
_batchConfigMap.get(BatchConfigProperties.INPUT_DIR_URI).toLowerCase()));
+ "%s.%s", DATA_FILE_PREFIX,
_batchConfigMap.get(BatchConfigProperties.INPUT_FORMAT).toLowerCase()));
if (payload._payloadType == PayloadType.URI) {
copyURIToLocal(_batchConfigMap, payload._uri, inputFile);
LOGGER.info("Copied from URI: {} to local file: {}", payload._uri,
inputFile.getAbsolutePath());
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: fix auth token injection in SegmentGenerationAndPushTaskExecutor (#7464)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 72d05a7 fix auth token injection in
SegmentGenerationAndPushTaskExecutor (#7464)
72d05a7 is described below
commit 72d05a7729448099a5352a19be28c1890178550e
Author: Alexander Pucher
AuthorDate: Wed Sep 22 13:20:29 2021 -0700
fix auth token injection in SegmentGenerationAndPushTaskExecutor (#7464)
---
.../SegmentGenerationAndPushTaskExecutor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
index bde5938..59e8e05 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
@@ -216,6 +216,8 @@ public class SegmentGenerationAndPushTaskExecutor extends
BaseTaskExecutor {
spec.setPushJobSpec(pushJobSpec);
spec.setTableSpec(tableSpec);
spec.setPinotClusterSpecs(pinotClusterSpecs);
+spec.setAuthToken(taskConfigs.get(BatchConfigProperties.AUTH_TOKEN));
+
return spec;
}
@@ -274,7 +276,7 @@ public class SegmentGenerationAndPushTaskExecutor extends
BaseTaskExecutor {
recordReaderSpec.setConfigClassName(taskConfigs.get(BatchConfigProperties.RECORD_READER_CONFIG_CLASS));
taskSpec.setRecordReaderSpec(recordReaderSpec);
-String authToken = taskConfigs.get(BatchConfigProperties.AUTH_TOKEN); //
TODO
+String authToken = taskConfigs.get(BatchConfigProperties.AUTH_TOKEN);
String tableNameWithType =
taskConfigs.get(BatchConfigProperties.TABLE_NAME);
Schema schema;
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: fix auth token injection in SegmentGenerationAndPushTaskExecutor
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch fix-segment-gen-push-auth-token
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 65dd5767d6fcc80d3720d964021747cda01acb3b
Author: Alexander Pucher
AuthorDate: Wed Sep 22 11:23:17 2021 -0700
fix auth token injection in SegmentGenerationAndPushTaskExecutor
---
.../SegmentGenerationAndPushTaskExecutor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
index bde5938..59e8e05 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskExecutor.java
@@ -216,6 +216,8 @@ public class SegmentGenerationAndPushTaskExecutor extends
BaseTaskExecutor {
spec.setPushJobSpec(pushJobSpec);
spec.setTableSpec(tableSpec);
spec.setPinotClusterSpecs(pinotClusterSpecs);
+spec.setAuthToken(taskConfigs.get(BatchConfigProperties.AUTH_TOKEN));
+
return spec;
}
@@ -274,7 +276,7 @@ public class SegmentGenerationAndPushTaskExecutor extends
BaseTaskExecutor {
recordReaderSpec.setConfigClassName(taskConfigs.get(BatchConfigProperties.RECORD_READER_CONFIG_CLASS));
taskSpec.setRecordReaderSpec(recordReaderSpec);
-String authToken = taskConfigs.get(BatchConfigProperties.AUTH_TOKEN); //
TODO
+String authToken = taskConfigs.get(BatchConfigProperties.AUTH_TOKEN);
String tableNameWithType =
taskConfigs.get(BatchConfigProperties.TABLE_NAME);
Schema schema;
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch fix-segment-gen-push-auth-token created (now 65dd576)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch fix-segment-gen-push-auth-token in repository https://gitbox.apache.org/repos/asf/pinot.git. at 65dd576 fix auth token injection in SegmentGenerationAndPushTaskExecutor This branch includes the following new commits: new 65dd576 fix auth token injection in SegmentGenerationAndPushTaskExecutor The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated (fabda2b -> 2c74f20)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git. from fabda2b create pinot-integration-test-base module test-jar (#7399) add 2c74f20 exclude healthcheck endpoint from auth (#7455) No new revisions were added by this update. Summary of changes: .../org/apache/pinot/controller/api/access/AuthenticationFilter.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch exclude-healthcheck-from-auth created (now 72e1ca5)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch exclude-healthcheck-from-auth in repository https://gitbox.apache.org/repos/asf/pinot.git. at 72e1ca5 exclude healthcheck endpoint from auth This branch includes the following new commits: new 72e1ca5 exclude healthcheck endpoint from auth The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: exclude healthcheck endpoint from auth
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch exclude-healthcheck-from-auth
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 72e1ca5ee2b2a005a5189de3c834d9d625123bf4
Author: Alexander Pucher
AuthorDate: Mon Sep 20 13:43:14 2021 -0700
exclude healthcheck endpoint from auth
---
.../org/apache/pinot/controller/api/access/AuthenticationFilter.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
index 8a9d9fb..35733dc 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/access/AuthenticationFilter.java
@@ -48,7 +48,7 @@ import org.glassfish.grizzly.http.server.Request;
@javax.ws.rs.ext.Provider
public class AuthenticationFilter implements ContainerRequestFilter {
private static final Set UNPROTECTED_PATHS =
- new HashSet<>(Arrays.asList("", "help", "auth/info", "auth/verify"));
+ new HashSet<>(Arrays.asList("", "help", "auth/info", "auth/verify",
"health"));
@Inject
Provider _requestProvider;
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated (dc80d67 -> 514f469)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git. from dc80d67 avoid creating stateless ParseContextImpl once per jsonpath evaluation, avoid varargs allocation (#7412) add 514f469 Obfuscator base implementation (#7407) No new revisions were added by this update. Summary of changes: .../apache/pinot/spi/env/PinotConfiguration.java | 6 + .../org/apache/pinot/spi/utils/Obfuscator.java | 140 + .../apache/pinot/spi/config/ConfigUtilsTest.java | 17 +++ .../org/apache/pinot/spi/utils/ObfuscatorTest.java | 119 ++ 4 files changed, 282 insertions(+) create mode 100644 pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java create mode 100644 pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch config-obfuscator updated (beb7aa0 -> cfb41e9)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch config-obfuscator in repository https://gitbox.apache.org/repos/asf/pinot.git. from beb7aa0 dedicated JsonNode path add cfb41e9 handle json string separately No new revisions were added by this update. Summary of changes: .../org/apache/pinot/spi/utils/Obfuscator.java | 25 +++--- .../org/apache/pinot/spi/utils/ObfuscatorTest.java | 7 ++ 2 files changed, 24 insertions(+), 8 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch config-obfuscator updated (1f996ba -> beb7aa0)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch config-obfuscator in repository https://gitbox.apache.org/repos/asf/pinot.git. from 1f996ba license headers add beb7aa0 dedicated JsonNode path No new revisions were added by this update. Summary of changes: .../apache/pinot/spi/env/PinotConfiguration.java | 2 +- .../org/apache/pinot/spi/utils/Obfuscator.java | 31 +- .../org/apache/pinot/spi/utils/ObfuscatorTest.java | 23 +++- 3 files changed, 42 insertions(+), 14 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch config-obfuscator updated (fc7af0c -> 1f996ba)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch config-obfuscator in repository https://gitbox.apache.org/repos/asf/pinot.git. from fc7af0c basic obfuscator config add 1f996ba license headers No new revisions were added by this update. Summary of changes: .../java/org/apache/pinot/spi/utils/Obfuscator.java| 18 ++ .../org/apache/pinot/spi/utils/ObfuscatorTest.java | 18 ++ 2 files changed, 36 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch config-obfuscator created (now fc7af0c)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch config-obfuscator in repository https://gitbox.apache.org/repos/asf/pinot.git. at fc7af0c basic obfuscator config This branch includes the following new commits: new fc7af0c basic obfuscator config The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: basic obfuscator config
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch config-obfuscator
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit fc7af0c8ea94d0aee364e85815b597c6be3dc01e
Author: Alexander Pucher
AuthorDate: Tue Sep 7 16:49:43 2021 -0700
basic obfuscator config
---
.../apache/pinot/spi/env/PinotConfiguration.java | 6 ++
.../org/apache/pinot/spi/utils/Obfuscator.java | 96 ++
.../apache/pinot/spi/config/ConfigUtilsTest.java | 17
.../org/apache/pinot/spi/utils/ObfuscatorTest.java | 83 +++
4 files changed, 202 insertions(+)
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
index ce4dd8f..e50b13f 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
@@ -32,6 +32,7 @@ import
org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.MapConfiguration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.pinot.spi.ingestion.batch.spec.PinotFSSpec;
+import org.apache.pinot.spi.utils.Obfuscator;
/**
@@ -441,4 +442,9 @@ public class PinotConfiguration {
public Map toMap() {
return CommonsConfigurationUtils.toMap(_configuration);
}
+
+ @Override
+ public String toString() {
+return String.valueOf(new Obfuscator().obfuscateJson(this));
+ }
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
new file mode 100644
index 000..7e0fd21
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
@@ -0,0 +1,96 @@
+package org.apache.pinot.spi.utils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+
+/**
+ * Simple obfuscator for object trees and configuration containers with
key-value pairs. Matches a configurable set of
+ * patterns and replaces sensitive values with a pre-defined masked value for
output.
+ *
+ * Example input:
+ *
+ * {
+ * "type": "sample object",
+ * "nestedCredentials": {
+ * "user": "admin",
+ * "password": "verysecret"
+ * }
+ * }
+ *
+ *
+ * Example output
+ *
+ * {
+ * "type": "sample object",
+ * "nestedCredentials": {
+ * "user": "admin",
+ * "password": "*"
+ * }
+ * }
+ *
+ */
+public final class Obfuscator {
+ private static final String DEFAULT_MASKED_VALUE = "*";
+ private static final List DEFAULT_PATTERNS =
+ Stream.of("(?i).*secret$", "(?i).*password$",
"(?i).*token$").map(Pattern::compile).collect(Collectors.toList());
+
+ private final String _maskedValue;
+ private final List _patterns;
+
+ /**
+ * Obfuscator with default behavior matching (ignore case) "secret",
"password", and "token" suffixes. Masks any
+ * values with '*'
+ */
+ public Obfuscator() {
+_maskedValue = DEFAULT_MASKED_VALUE;
+_patterns = DEFAULT_PATTERNS;
+ }
+
+ /**
+ * Obfuscator with customized masking behavior. Defaults do not apply!
Please ensure case-insensitive regex matching.
+ *
+ * @param maskedValue replacement value
+ * @param patterns key patterns to obfuscate
+ */
+ public Obfuscator(String maskedValue, List patterns) {
+_maskedValue = maskedValue;
+_patterns = patterns;
+ }
+
+ /**
+ * Serialize an object tree as JSON and obfuscate matching keys.
+ *
+ * @param object input value
+ * @return obfuscated JSON tree
+ */
+ public JsonNode obfuscateJson(Object object) {
+// NOTE: jayway json path 2.4.0 seems to have issues with '@.name' so
we'll do this manually
+// as determined by a cursory and purely subjective investigation by alex
+// "$..[?(@.name =~ /password$/i || @.name =~ /secret$/i || @.name =~
/token$/i)]"
+
+return obfuscateJsonRec(JsonUtils.objectToJsonNode(object));
+ }
+
+ private JsonNode obfuscateJsonRec(JsonNode node) {
+if (node.isObject()) {
+ node.fieldNames().forEachRemaining(field -> {
+if (_patterns.stream().anyMatch(pattern ->
pattern.matcher(field).matches())) {
+ ((ObjectNode) node).put(field, _maskedValue);
+} else if (node.isArray()) {
+ IntStream.range(0, node.size()).forEach(i -> ((ArrayNode)
node).set(i, obfuscateJsonRe
[pinot] branch master updated (df9f5ef -> 95a6ae8)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git. from df9f5ef moving the map of pinot data type and java type to common util (#7231) add 95a6ae8 add cli args for auth tokens and user/password to ingestion job command (#7233) No new revisions were added by this update. Summary of changes: .../tools/admin/command/LaunchDataIngestionJobCommand.java| 11 +++ 1 file changed, 11 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch ingestion-job-cli-auth-tokens created (now c34582b)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch ingestion-job-cli-auth-tokens in repository https://gitbox.apache.org/repos/asf/pinot.git. at c34582b add cli args for auth tokens and user/password to ingestion job command This branch includes the following new commits: new c34582b add cli args for auth tokens and user/password to ingestion job command The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] 01/01: add cli args for auth tokens and user/password to ingestion job command
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch ingestion-job-cli-auth-tokens
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit c34582bf9eebf800dafd4a6272ce0146d56668ea
Author: Alexander Pucher
AuthorDate: Fri Jul 30 00:54:12 2021 -0700
add cli args for auth tokens and user/password to ingestion job command
---
.../tools/admin/command/LaunchDataIngestionJobCommand.java| 11 +++
1 file changed, 11 insertions(+)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java
index 62347e6..6f1fe3c 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/LaunchDataIngestionJobCommand.java
@@ -20,6 +20,7 @@ package org.apache.pinot.tools.admin.command;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.core.util.TlsUtils;
import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher;
import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
@@ -49,6 +50,12 @@ public class LaunchDataIngestionJobCommand extends
AbstractBaseAdminCommand impl
private List _values;
@Option(name = "-propertyFile", required = false, metaVar = "", usage = "A property file contains context values to set the job
spec template")
private String _propertyFile;
+ @Option(name = "-user", required = false, metaVar = "", usage =
"Username for basic auth.")
+ private String _user;
+ @Option(name = "-password", required = false, metaVar = "", usage =
"Password for basic auth.")
+ private String _password;
+ @Option(name = "-authToken", required = false, metaVar = "", usage =
"Http auth token.")
+ private String _authToken;
public static void main(String[] args) {
PluginManager.get().init();
@@ -128,6 +135,10 @@ public class LaunchDataIngestionJobCommand extends
AbstractBaseAdminCommand impl
tlsSpec.getTrustStorePath(), tlsSpec.getTrustStorePassword());
}
+if (StringUtils.isBlank(spec.getAuthToken())) {
+ spec.setAuthToken(makeAuthToken(_authToken, _user, _password));
+}
+
try {
IngestionJobLauncher.runIngestionJob(spec);
} catch (Exception e) {
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[pinot] branch master updated: 7228 /sql REST call now compatible with auth (#7230)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5e0dcff 7228 /sql REST call now compatible with auth (#7230)
5e0dcff is described below
commit 5e0dcff779faa290a9d5285b039bfef2002b49a5
Author: Gaurav Joshi <68965367+joshigaur...@users.noreply.github.com>
AuthorDate: Thu Jul 29 16:54:28 2021 -0700
7228 /sql REST call now compatible with auth (#7230)
---
pinot-controller/src/main/resources/app/utils/axios-config.ts | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pinot-controller/src/main/resources/app/utils/axios-config.ts
b/pinot-controller/src/main/resources/app/utils/axios-config.ts
index 0a34237..108f62a 100644
--- a/pinot-controller/src/main/resources/app/utils/axios-config.ts
+++ b/pinot-controller/src/main/resources/app/utils/axios-config.ts
@@ -52,4 +52,6 @@ export const baseApi = axios.create({ baseURL: '/' });
baseApi.interceptors.request.use(handleConfig, handleError);
baseApi.interceptors.response.use(handleResponse, handleError);
-export const transformApi = axios.create({baseURL: '/', transformResponse:
[data => data]});
\ No newline at end of file
+export const transformApi = axios.create({baseURL: '/', transformResponse:
[data => data]});
+transformApi.interceptors.request.use(handleConfig, handleError);
+transformApi.interceptors.response.use(handleResponse, handleError);
\ No newline at end of file
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: add groovy unit test for sha256 data masking (#7179)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b6907fc add groovy unit test for sha256 data masking (#7179)
b6907fc is described below
commit b6907fc717a9c8f66b2288b1afb16b3ee1abcd1e
Author: Alexander Pucher
AuthorDate: Mon Jul 19 18:23:19 2021 -0700
add groovy unit test for sha256 data masking (#7179)
---
.../apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java | 4
1 file changed, 4 insertions(+)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
index c5086c3..46f1209 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
@@ -83,6 +83,10 @@ public class GroovyFunctionEvaluatorTest {
genericRow7.putValue("eventType", "CLICK");
entries.add(new Object[]{"Groovy({eventType == 'IMPRESSION' ? 1: 0},
eventType)", Lists.newArrayList("eventType"), genericRow7, 0});
+GenericRow genericRow8 = new GenericRow();
+genericRow8.putValue("ssn", "123-45-6789");
+entries.add(new
Object[]{"Groovy({org.apache.commons.codec.digest.DigestUtils.sha256Hex(ssn)},
ssn)", Lists.newArrayList("ssn"), genericRow8,
"01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0"});
+
return entries.toArray(new Object[entries.size()][]);
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch groovy-data-masking-unit-test created (now a72ecd7)
This is an automated email from the ASF dual-hosted git repository. apucher pushed a change to branch groovy-data-masking-unit-test in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at a72ecd7 add groovy unit test for sha256 data masking This branch includes the following new commits: new a72ecd7 add groovy unit test for sha256 data masking The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: add groovy unit test for sha256 data masking
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch groovy-data-masking-unit-test
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit a72ecd71839379a4abdd55ab9af5a166dd1e9ee0
Author: Alexander Pucher
AuthorDate: Mon Jul 19 12:43:19 2021 -0700
add groovy unit test for sha256 data masking
---
.../apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java | 4
1 file changed, 4 insertions(+)
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
index c5086c3..46f1209 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/GroovyFunctionEvaluatorTest.java
@@ -83,6 +83,10 @@ public class GroovyFunctionEvaluatorTest {
genericRow7.putValue("eventType", "CLICK");
entries.add(new Object[]{"Groovy({eventType == 'IMPRESSION' ? 1: 0},
eventType)", Lists.newArrayList("eventType"), genericRow7, 0});
+GenericRow genericRow8 = new GenericRow();
+genericRow8.putValue("ssn", "123-45-6789");
+entries.add(new
Object[]{"Groovy({org.apache.commons.codec.digest.DigestUtils.sha256Hex(ssn)},
ssn)", Lists.newArrayList("ssn"), genericRow8,
"01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0"});
+
return entries.toArray(new Object[entries.size()][]);
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
