This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a change to branch HDDS-3698-nonrolling-upgrade
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 4506883 Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
add c0eb347 HDDS-5237. Add SSL support to the Ozone streaming API (#2315)
add 698456b HDDS-5363. Datanode shutdown due to too many bad volumes in
CI (#2352)
add b545e69 HDDS-4692. Handle CRLStatusReport got from DN heartbeats and
persist them (#2334)
add 51df153 HDDS-5356. [SCM-HA] SCM start failed with
PipelineNotFoundException (#2346)
add b9caec3 HDDS-5342. HTML report missing from acceptance results (#2332)
add bb4a5b2 HDDS-5361. Suppress logging of ServerNotLeaderException.
(#2351)
add 76d9a10 HDDS-5347. Wrong cache key for integration tests (#2336)
add eb6c1b6 HDDS-5314. Show number of Open containers per Node in Recon
UI (#2321)
add 5c3f9aa HDDS-5355. In ContainerStateMachine, share the executor
threads between the containers. (#2350)
add 5a9257e HDDS-5357. Move SCMUpdateProtocol to hdds interface-server
package. (#2348)
add fa9616d HDDS-5352. java.lang.ClassNotFoundException:
org/eclipse/jetty/alpn/ALPN (#2347)
add 5e1308a HDDS-5367. Update modification time when updating
quota/storageType/versioning (#2355)
add 2cd55d7 HDDS-5269. Datandoe with low ratis log volume space should
not be considered for new pipeline allocation. (#2344)
add dc02775 HDDS-4926. Support start/stop for container balancer via
command line (#2278)
add b563669 HDDS-5350 : Add allocate block support in MockOmTransport
(#2341). Contributed by Uma Maheswara Rao G.
add 93fb714 HDDS-5353. Avoid unnecessary executeBatch call in
insertAudits (#2342)
add 0766d2c HDDS-5381. SCM terminated with exit status 1: null. (#2362)
add bf42bea HDDS-5369. Cleanup unused configuration related to SCM HA
(#2359)
add e251bab HDDS-5289. Update container's deleteTransactionId on creation
of the transaction in SCM. (#2361)
add 3232fb0 HDDS-5387. ProfileServlet to move the default output location
to an ozone specific directory (#2368)
add 450c375 HDDS-5341. Container report processing is single threaded
(#2338)
add be7264d HDDS-5380. Get more accurate space info for
DedicatedDiskSpaceUsage. (#2365)
add b7ecf8b HDDS-5383. Eliminate expensive string creation in debug log
messages (#2372)
add 474457c HDDS-5389. Include ozoneserviceid in fs.defaultFS when
configuring o3fs (#2370)
add 1e5c708 HDDS-5395. Avoid unnecessary numKeyOps.incr() call in
OMMetrics (#2374)
add 88383d1 HDDS-5396. Fix negligence issue conditional expressions in
MockCRLStore.java (#2380)
add 1d8f972 HDDS-5398. Avoid object creation in ReplicationManger debug
log statements (#2379)
new 039afb3 Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 8 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 6 +-
.../hadoop/hdds/fs/AbstractSpaceUsageSource.java | 7 +-
.../hadoop/hdds/fs/DedicatedDiskSpaceUsage.java | 8 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 86 +----
.../apache/hadoop/hdds/scm/client/ScmClient.java | 21 +
.../hadoop/hdds/scm/container/ContainerInfo.java | 5 +
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 21 +
.../hadoop/hdds/security/x509/crl/CRLStatus.java | 87 +++++
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +
.../common/src/main/resources/ozone-default.xml | 141 +------
.../server/ratis/ContainerStateMachine.java | 74 ++--
.../common/volume/StorageVolumeChecker.java | 20 +-
.../container/stream/DirectoryServerSource.java | 6 +-
.../container/stream/DirstreamClientHandler.java | 5 +
.../container/stream/DirstreamServerHandler.java | 43 ++-
.../ozone/container/stream/StreamingClient.java | 28 +-
...ingDestination.java => StreamingException.java} | 20 +-
.../ozone/container/stream/StreamingServer.java | 68 ++--
.../stream/TestDirstreamClientHandler.java | 139 +++++++
.../container/stream/TestStreamingServer.java | 48 +++
.../ozone/container/stream}/package-info.java | 7 +-
hadoop-hdds/docs/content/interface/O3fs.md | 8 +
hadoop-hdds/docs/content/interface/O3fs.zh.md | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 73 ++++
.../SCMBlockLocationFailoverProxyProvider.java | 8 +-
.../SCMSecurityProtocolFailoverProxyProvider.java | 8 +-
.../scm/update/client/CRLClientUpdateHandler.java | 2 +-
.../certificate/authority/CertificateStore.java | 6 +
.../authority/PKIProfiles/DefaultProfile.java | 2 +-
.../hadoop/hdds/security/x509/crl/CRLCodec.java | 0
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 0
.../hdds/security/x509/crl/CRLInfoCodec.java | 0
.../hadoop/hdds/server/events/EventExecutor.java | 5 +
.../hadoop/hdds/server/events/EventQueue.java | 44 ++-
...dExecutor.java => FixedThreadPoolExecutor.java} | 42 +-
.../hdds/server/events/SingleThreadExecutor.java | 9 +
.../hadoop/hdds/server/http/ProfileServlet.java | 2 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 9 +
.../x509/certificate/authority/MockCAStore.java | 12 +
.../hadoop/hdds/server/events/TestEventQueue.java | 62 ++-
.../src/main/proto/ScmAdminProtocol.proto | 50 ++-
hadoop-hdds/interface-client/pom.xml | 13 -
hadoop-hdds/interface-server/pom.xml | 2 +
.../src/main/proto/SCMUpdateProtocol.proto | 0
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 31 +-
.../block/DatanodeDeletedBlockTransactions.java | 6 -
.../hdds/scm/block/DeletedBlockLogImplV2.java | 1 +
.../scm/block/DeletedBlockLogStateManagerImpl.java | 26 +-
.../hdds/scm/block/PendingDeleteHandler.java | 41 --
.../hdds/scm/block/PendingDeleteStatusList.java | 85 ----
.../hdds/scm/block/SCMBlockDeletingService.java | 24 --
.../hdds/scm/container/ContainerReportHandler.java | 38 --
.../scm/container/ContainerStateManagerImpl.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 11 +-
.../scm/container/balancer/ContainerBalancer.java | 101 +++--
.../balancer/ContainerBalancerConfiguration.java | 28 ++
.../hdds/scm/crl/CRLStatusReportHandler.java | 87 +++++
.../apache/hadoop/hdds/scm/crl}/package-info.java | 7 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 19 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 8 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 14 +
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 15 +-
.../scm/pipeline/PipelineStateManagerV2Impl.java | 4 +-
...inerLocationProtocolServerSideTranslatorPB.java | 70 ++++
.../scm/server/OzoneStorageContainerManager.java | 3 +
.../hdds/scm/server/SCMBlockProtocolServer.java | 2 +
.../hadoop/hdds/scm/server/SCMCertStore.java | 20 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 65 ++++
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 15 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 3 +
.../hdds/scm/server/StorageContainerManager.java | 90 +++--
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 113 ++++--
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 96 ++++-
.../hadoop/hdds/scm/container/MockNodeManager.java | 8 +
.../container/balancer/TestContainerBalancer.java | 20 +-
.../algorithms/TestContainerPlacementFactory.java | 11 +
.../TestSCMContainerPlacementCapacity.java | 16 +-
.../TestSCMContainerPlacementRackAware.java | 22 +-
.../TestSCMContainerPlacementRandom.java | 19 +-
.../hdds/scm/crl/TestCRLStatusReportHandler.java | 137 +++++++
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 49 ++-
.../hdds/scm/node/TestNodeReportHandler.java | 21 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 25 +-
.../hdds/scm/node/TestSCMNodeStorageStatMap.java | 11 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 14 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 33 +-
.../hdds/scm/update/server/MockCRLStore.java | 6 +-
.../ozone/container/common/TestEndPoint.java | 25 +-
.../hdds/scm/cli/ContainerBalancerCommands.java | 108 ++++++
....java => ContainerBalancerStartSubcommand.java} | 40 +-
...java => ContainerBalancerStatusSubcommand.java} | 21 +-
...d.java => ContainerBalancerStopSubcommand.java} | 18 +-
.../hdds/scm/cli/ContainerOperationClient.java | 20 +
...d.java => TestContainerBalancerSubCommand.java} | 114 +++---
.../hadoop/ozone/client/MockOmTransport.java | 12 +
.../hadoop/ozone/client/TestOzoneClient.java | 34 +-
.../src/main/compose/compatibility/docker-config | 1 +
.../dist/src/main/compose/ozone-csi/docker-config | 1 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../dist/src/main/compose/ozone-mr/common-config | 1 +
.../dist/src/main/compose/ozone-mr/test.sh | 6 +-
.../src/main/compose/ozone-om-ha/docker-config | 1 +
.../src/main/compose/ozone-topology/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../src/main/compose/ozones3-haproxy/docker-config | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 1 +
.../src/main/compose/ozonesecure-mr/docker-config | 2 +
.../dist/src/main/compose/restart/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/test-all.sh | 8 +-
.../compose/upgrade/compose/non-ha/docker-config | 3 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 3 +
.../dist/src/main/compose/xcompat/docker-config | 1 +
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 3 +
...s.java => TestContainerBalancerOperations.java} | 82 ++--
.../commandhandler/TestBlockDeletion.java | 47 ---
.../commandhandler/TestCloseContainerHandler.java | 4 +
.../commandhandler/TestDeleteContainerHandler.java | 4 +
.../TestDatanodeHddsVolumeFailureDetection.java | 4 +
.../TestDatanodeHddsVolumeFailureToleration.java | 4 +
.../hadoop/ozone/fsck/TestContainerMapper.java | 5 +
.../src/main/proto/OmClientProtocol.proto | 1 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 1 -
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 20 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 19 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 10 +
hadoop-ozone/ozonefs-shaded/pom.xml | 3 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 17 +-
.../ozone/recon/api/types/DatanodeMetadata.java | 15 +
.../ozone/recon/scm/ReconContainerManager.java | 18 +
.../scm/ReconStorageContainerManagerFacade.java | 6 +
.../webapps/recon/ozone-recon-web/api/db.json | 20 +
.../src/views/datanodes/datanodes.tsx | 11 +
.../hadoop/ozone/recon/api/TestEndpoints.java | 3 +-
.../ozone/recon/api/TestOpenContainerCount.java | 427 +++++++++++++++++++++
.../ozone/audit/parser/common/DatabaseHelper.java | 7 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 23 +-
140 files changed, 2840 insertions(+), 980 deletions(-)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLStatus.java
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/stream/{StreamingDestination.java
=> StreamingException.java} (71%)
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/stream/TestDirstreamClientHandler.java
copy {hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/conf =>
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/stream}/package-info.java
(90%)
rename hadoop-hdds/{common =>
framework}/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLCodec.java
(100%)
rename hadoop-hdds/{common =>
framework}/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java
(100%)
rename hadoop-hdds/{common =>
framework}/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfoCodec.java
(100%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/{SingleThreadExecutor.java
=> FixedThreadPoolExecutor.java} (67%)
rename hadoop-hdds/{interface-client =>
interface-server}/src/main/proto/SCMUpdateProtocol.proto (100%)
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/PendingDeleteHandler.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/PendingDeleteStatusList.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl/CRLStatusReportHandler.java
copy
{hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils
=>
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/crl}/package-info.java
(89%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/crl/TestCRLStatusReportHandler.java
create mode 100644
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java
copy
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/{ReplicationManagerStartSubcommand.java
=> ContainerBalancerStartSubcommand.java} (50%)
copy
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/{ReplicationManagerStartSubcommand.java
=> ContainerBalancerStatusSubcommand.java} (74%)
copy
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/{ReplicationManagerStartSubcommand.java
=> ContainerBalancerStopSubcommand.java} (73%)
copy
hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/{TestMaintenanceSubCommand.java
=> TestContainerBalancerSubCommand.java} (50%)
copy
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{TestContainerOperations.java
=> TestContainerBalancerOperations.java} (53%)
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
