This is an automated email from the ASF dual-hosted git repository.
duong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 1a274b6f5a HDDS-8679. Add dedicated, configurable thread pool for OM
gRPC server (#4771)
add 74d885287a HDDS-7734. Implement symmetric SecretKeys lifescycle
management in SCM (#4194)
add aa24a638a7 HDDS-7830. SCM API for OM and Datanode to get secret keys
(#4345)
add a83668c0f6 HDDS-7831. Use symmetric secret key to sign and verify
token (#4417)
add c623e942c1 HDDS-8003. E2E integration test cases for block tokens
(#4547)
add 75d46b04ef HDDS-7945. Integrate secret keys to SCM snapshot (#4549)
add 5257f74250 HDDS-8164. Authorize secret key APIs (#4597)
add 7bfa04e7cf HDDS-8677. Ozone admin OM CLI command for block tokens
(#4760)
No new revisions were added by this update.
Summary of changes:
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 34 ++
.../security/exception/SCMSecretKeyException.java | 47 +++
.../security/token/ContainerTokenIdentifier.java | 17 +-
.../security/token/OzoneBlockTokenIdentifier.java | 26 +-
.../security/token/ShortLivedTokenIdentifier.java | 24 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 4 +
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 1 +
.../org/apache/hadoop/util/ProtobufUtils.java} | 54 +--
.../common/src/main/resources/ozone-default.xml | 75 ++++
.../token/TestOzoneBlockTokenIdentifier.java | 306 ----------------
.../hadoop/hdds/utils/TestProtobufUtils.java | 48 +++
.../apache/hadoop/ozone/HddsDatanodeService.java | 23 +-
.../common/statemachine/DatanodeStateMachine.java | 12 +-
.../ECReconstructionCoordinator.java | 10 +-
.../container/ec/reconstruction/TokenHelper.java | 47 +--
.../ozone/container/ozoneimpl/OzoneContainer.java | 17 +-
.../ozone/container/common/ContainerTestUtils.java | 2 +-
.../container/common/TestDatanodeStateMachine.java | 11 +-
.../container/ozoneimpl/TestOzoneContainer.java | 6 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 3 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 10 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 8 +-
.../hadoop/hdds/protocol/SCMSecurityProtocol.java | 1 +
.../hadoop/hdds/protocol/SecretKeyProtocol.java | 55 +++
.../hdds/protocol/SecretKeyProtocolDatanode.java | 34 ++
.../hadoop/hdds/protocol/SecretKeyProtocolOm.java | 32 ++
.../hadoop/hdds/protocol/SecretKeyProtocolScm.java | 31 ++
.../SecretKeyProtocolClientSideTranslatorPB.java | 165 +++++++++
.../protocolPB/SecretKeyProtocolDatanodePB.java | 40 +++
.../hdds/protocolPB/SecretKeyProtocolOmPB.java | 39 ++
.../hdds/protocolPB/SecretKeyProtocolScmPB.java | 38 ++
.../SecretKeyProtocolFailoverProxyProvider.java | 303 ++++++++++++++++
.../SingleSecretKeyProtocolProxyProvider.java | 56 +++
.../security/symmetric/DefaultSecretKeyClient.java | 72 ++++
.../symmetric/DefaultSecretKeySignerClient.java | 131 +++++++
.../symmetric/DefaultSecretKeyVerifierClient.java | 105 ++++++
.../security/symmetric/LocalSecretKeyStore.java | 199 +++++++++++
.../hdds/security/symmetric/ManagedSecretKey.java | 154 ++++++++
.../hdds/security/symmetric/SecretKeyClient.java | 26 ++
.../hdds/security/symmetric/SecretKeyConfig.java | 109 ++++++
.../hdds/security/symmetric/SecretKeyManager.java | 173 +++++++++
.../security/symmetric/SecretKeySignerClient.java | 46 +++
.../hdds/security/symmetric/SecretKeyState.java | 60 ++++
.../security/symmetric/SecretKeyStateImpl.java | 139 ++++++++
.../hdds/security/symmetric/SecretKeyStore.java | 35 ++
.../symmetric/SecretKeyVerifierClient.java | 32 ++
.../hdds/security/symmetric/package-info.java | 63 ++++
.../hdds/security/token/BlockTokenVerifier.java | 7 +-
.../token/ContainerTokenSecretManager.java | 14 +-
.../security/token/ContainerTokenVerifier.java | 6 +-
.../token/OzoneBlockTokenSecretManager.java | 66 +---
.../token/ShortLivedTokenSecretManager.java | 75 ++--
.../security/token/ShortLivedTokenVerifier.java | 66 ++--
.../hadoop/hdds/security/token/TokenVerifier.java | 9 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 58 +++
.../symmetric/LocalSecretKeyStoreTest.java | 188 ++++++++++
.../security/symmetric/ManagedSecretKeyTest.java | 75 ++++
.../security/symmetric/SecretKeyManagerTest.java | 206 +++++++++++
.../hdds/security/symmetric/SecretKeyTestUtil.java | 52 +++
.../security/token/TestBlockTokenVerifier.java | 18 +-
.../security/token/TestContainerTokenVerifier.java | 12 +-
.../token/TestOzoneBlockTokenIdentifier.java | 103 ++++++
.../token/TestOzoneBlockTokenSecretManager.java | 245 +++----------
.../hdds/security/token/TokenVerifierTests.java | 168 +++++----
.../org.mockito.plugins.MockMaker | 16 +
.../src/main/proto/ScmAdminProtocol.proto | 3 +-
.../interface-client/src/main/proto/hdds.proto | 3 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../src/main/proto/ScmSecretKeyProtocol.proto | 111 ++++++
.../src/main/proto/ScmServerSecurityProtocol.proto | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 8 +
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 26 ++
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 6 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 28 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +
.../apache/hadoop/hdds/scm/ha/io/ListCodec.java | 6 +-
.../hdds/scm/ha/io/ManagedSecretKeyCodec.java | 44 +++
.../SecretKeyProtocolServerSideTranslatorPB.java | 165 +++++++++
.../scm/security/ScmSecretKeyStateBuilder.java | 60 ++++
.../hdds/scm/security/SecretKeyManagerService.java | 163 +++++++++
.../hadoop/hdds/scm/security/package-info.java | 22 ++
.../hadoop/hdds/scm/server/SCMPolicyProvider.java | 15 +
.../hdds/scm/server/SCMSecurityProtocolServer.java | 76 +++-
.../hdds/scm/server/StorageContainerManager.java | 74 ++--
.../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 4 +-
.../scm/server/TestSCMSecurityProtocolServer.java | 2 +-
.../ozone/container/common/TestEndPoint.java | 12 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 17 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 3 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 17 +
.../src/main/compose/ozonesecure-ha/docker-config | 5 +
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 32 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 7 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 11 +
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 11 +
.../org/apache/hadoop/ozone/TestBlockTokens.java | 394 +++++++++++++++++++++
.../apache/hadoop/ozone/TestBlockTokensCLI.java | 237 +++++++++++++
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 8 +-
.../org/apache/hadoop/ozone/TestSecretKeysApi.java | 366 +++++++++++++++++++
.../hadoop/ozone/TestSecureOzoneCluster.java | 263 --------------
.../hadoop/ozone/client/SecretKeyTestClient.java | 73 ++++
.../client/rpc/TestContainerStateMachine.java | 2 +
.../rpc/TestContainerStateMachineFlushDelay.java | 2 +
.../client/rpc/TestOzoneAtRestEncryption.java | 2 +
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 8 +-
.../container/ozoneimpl/TestOzoneContainer.java | 5 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 20 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 18 +-
.../server/TestSecureContainerServer.java | 17 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 4 +-
.../hadoop/ozone/scm/TestSecretKeySnapshot.java | 289 +++++++++++++++
.../src/main/proto/OmClientProtocol.proto | 11 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 77 ++--
.../protocolPB/OzoneManagerRequestHandler.java | 14 +
.../ozone/security/TestOzoneManagerBlockToken.java | 251 -------------
.../hadoop/ozone/admin/om/FetchKeySubCommand.java | 56 +++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +-
119 files changed, 5789 insertions(+), 1578 deletions(-)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecretKeyException.java
copy hadoop-hdds/{interface-server/src/main/proto/SCMRatisProtocol.proto =>
common/src/main/java/org/apache/hadoop/util/ProtobufUtils.java} (50%)
delete mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestProtobufUtils.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocol.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocolDatanode.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocolOm.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocolScm.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolClientSideTranslatorPB.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolDatanodePB.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolOmPB.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolScmPB.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SecretKeyProtocolFailoverProxyProvider.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SingleSecretKeyProtocolProxyProvider.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyClient.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeySignerClient.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/DefaultSecretKeyVerifierClient.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStore.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKey.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyClient.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyConfig.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeySignerClient.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyState.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStateImpl.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyStore.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyVerifierClient.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/package-info.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStoreTest.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/ManagedSecretKeyTest.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManagerTest.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyTestUtil.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
create mode 100644
hadoop-hdds/framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
create mode 100644
hadoop-hdds/interface-server/src/main/proto/ScmSecretKeyProtocol.proto
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SecretKeyProtocolServerSideTranslatorPB.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/ScmSecretKeyStateBuilder.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/package-info.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestBlockTokens.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestBlockTokensCLI.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecretKeysApi.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/SecretKeyTestClient.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSecretKeySnapshot.java
delete mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneManagerBlockToken.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FetchKeySubCommand.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
