This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from dba2d54 HDDS-5421. SCM throws NPE during JMX call. (#2398)
add 3e21d75 HDDS-3829. Introduce Layout Feature interface in Ozone.
(#1322)
add 3fe83ce HDDS-4173. Implement HDDS Version management using the
LayoutVersionManager interface. (#1392)
add 9b55f69 HDDS-4141. Implement Finalize command in Ozone Manager
client. (#1400)
add 95b8712 HDDS-4174. Add current HDDS layout version to Datanode
heartbeat/registration (#1421)
add 6fa5fa1 HDDS-4143. Implement a factory for OM Requests that returns
an instance based on layout version. (#1405)
add b96834f HDDS-4252. Add the current layout versions to DN - SCM proto
payload. (#1432)
add 08d8d36 HDDS-4227. Implement a 'Prepare For Upgrade' step in OM that
applies all committed Ratis transactions. (#1430)
add 8fe8a1f HDDS-4253. Add LayoutVersion request/response for DN
registration. (#1457)
add 515c619 HDDS-4319. Compile error with Java 11 (#1483)
add bfce8b9 HDDS-4172. Implement Finalize command in Ozone Manager
server. (#1456)
add 0d014c0 HDDS-4314. OM Layout Version Manager init throws silent CNF
error in integration tests. (#1492)
add b8dcf6b Merge remote-tracking branch 'upstream/master' into
HDDS-3698-upgrade
add d4212a7 Fix copilation issue after mergee
add 5186dec HDDS-4401. Fix compilation issue in HDDS-3698-upgrade branch.
(#1529)
add 8e29229 HDDS-4296. SCM changes to process Layout Info in heartbeat
request/response (#1486)
add 18bef92 HDDS-4142. Expose upgrade related state through JMX. (#1534)
add 28ecd8d Merge remote-tracking branch 'upstream/master' into
HDDS-3698-upgrade
add 7a26250 HDDS-4342. Add DataNode state and transitions for a node
going through upgrade. (#1508)
add 4384789 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-upgrade
add bf7624b Merge remote-tracking branch 'upstream/master' into
HDDS-3698-upgrade
add 6b9ed19 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-upgrade
add ad9d019 HDDS-4536. Add more unit tests for OM layout version manager.
(#1646)
add 9c4f2b4 HDDS-4463. Verify that OM/SCM start fails when Software
Layout Version < Metadata Layout Version. (#1645)
add c3e3b35 HDDS-4179. Implement post-finalize SCM logic. (#1611)
add a7797aa HDDS-4480. Implement OM Prepare Request/Response. (#1613)
add 2489968 HDDS-4540. Add a new OM admin operation to submit the
OMPrepareRequest. (#1664)
add 6519142 HDDS-4178. SCM Finalize client command implementation (#1651)
add bdd00a7 HDDS-4564. Prepare client should check every OM individually
for the prepared check based on Txn ID. (#1692)
add 17a126d HDDS-4569. Add pre append gate and marker file to OM prepare
state. (#1705)
add 72c215c HDDS-4612. Create OMCancelPrepareRequest and Response to
cancel the prepared state of an OM. (#1765)
add d36b87d HDDS-4175. Implement Datanode Finalization. (#1720)
add 81c1dee HDDS-4219. Revisit 'static' nature of OM Layout Version
Manager. (#1773)
add 1de63f4 HDDS-4645. Starting OM with the --upgrade flag should delete
the prepare marker file. (#1798)
add a81b36e HDDS-4614. Add Integration test for HDDS upgrade (happy path
cases). (#1726)
add ee8d276 HDDS-4611. Add an admin command to cancel "preparation" of an
OM quorum. (#1830)
add 1e187f7 Merge remote-tracking branch 'upstream/master' into
upgrade-branch-merge-candidate
add 7ae013e Fix spotbugs issues.
add ba4aeca trigger new CI check
add dd14670 trigger new CI check
add 5720ebf HDDS-4182. Onboard HDDS-3869 into Layout version management.
(#1913)
add 277cc0b HDDS-4610. Fix issues in 'prepare' operation with one OM
down. (#1836)
add 9a078c4 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade
add 8a9be0d trigger new CI check
add 447024b HDDS-4817. Fresh deploy of Ozone must use the highest layout
version by default. (#1933)
add 485ec25 HDDS-4902 Add Layout version information to Recon datanode
info API. (#1993)
add df57ebc HDDS-4946. Do not wait one heartbeat to move newly registered
datanodes that match SCM's MLV from HEALTHY_READONLY to HEALTHY (#2015)
add 7eefee7 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade-merge-candidate
add f6a302a HDDS-4908. Layout version should be available in DB for an
un-finalized OM to be finalized through a Ratis snapshot. (#2017)
add f6ecb1d Merge remote-tracking branch 'ssh-upstream/master' into
HDDS-3698-nonrolling-upgrade
add d55a714 HDDS-5023. Datanodes should always use MLV 0 when no VERSION
file is present. (#2080)
add ed00db7 HDDS-4990. Introduce First upgrade startup action and
Pre-finalized state validation in Layout Feature. (#2040)
add 2f96d94 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade
add 1730b0d Allow layout version to be specified through config on --init
(for MiniOzone cluster testing).
add de3d735 Fix SCM admin command issues after merge.
add 666f33b Ignore failing upgrade integration test.
add 8f3b5ae Fix rat failure.
add dfc4d99 trigger new CI check
add f16092e Intialize DN layout version before security init.
add 805d93d Checkstyle fix.
add 32d0834 Merge remote-tracking branch 'upstream/master' into
HDDS-5046-2
add 9e2e43f Merge remote-tracking branch 'upstream/master' into
HDDS-5046-2
add 9363434 Fix ozone-ha acceptance test.
add 3ea88e1 HDDS-5014. Add non-rolling upgrade design docs.
add a55f657 Merge remote-tracking branch 'upstream/master' into
ssh-upstream-upgrade-branch
add baeb46d Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
add c4b7e7b HDDS-4828. SCM should go into "safe mode" until there is at
least 1 pipeline to work with after finalization. (#2101)
add 5c6c858 HDDS-4181. Add acceptance tests for upgrade, finalization and
downgrade. (#2056)
add 6e027a1 HDDS-5014. Upgrade usage primer documentation. (#2133)
add e8e7ced HDDS-5014. Move upgrade user flow to 'feature' folder.
add 7266f32 HDDS-4914. Failure injection and validating HDDS upgrade.
(#1998)
add 58a87fb HDDS-5086. Add pre-finalize validation action for SCM HA.
(#2143)
add d2da5a4 HDDS-4992. SCM should not use pipelines with HEALTHY_READONLY
datanodes (#2142)
add 3221362 Update upgrade-dev-primer.md
add 2823320 HDDS-5108. Attempt to remove state from *UpgradeFinalizer
classes. (#2160)
add 40223ec HDDS-4818. Add command to get status of Finalize in OM & SCM
(#2083)
add 5927bd2 HDDS-5118. Recover from failure during upgrade action. (#2179)
add 9168e07 HDDS-4959. NoSuchMethodException when wrapping RpcException
on downgrade (#2178)
add bef4aed HDDS-5137. Update licenses for non-rolling upgrade new
dependencies. (#2201)
add cdb1471 HDDS-5170. Race condition in NodestateManager#addNode allows
datanodes with lower MLV to be used in pipelines. (#2204)
add 253cc55 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade-master-merge2
add 1686e56 Fix compilation errors afte merge Update javassist in recon
pom Fix changes introduced in merge that failed TestSCMNodeManager upgrade
tests Fix checkstyle Fix intermittent test failure
TestSCMNodeManager#testSetNodeOpStateAndCommandFired after merge Skip scm init
default layout version in TestOzoneConfigurationFields
add ea239ee HDDS-5226. Do not fail SCM HA pre-finalize validation if SCM
HA was already being used. (#2257)
add 5f046fa HDDS-5138. Upgrade related RPC calls should be allowed only
for admins. (#2217)
add 92b5106 HDDS-5244. Allow multiple OM request versions to be supported
at same layout version. (#2265)
add 1d18f47 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade-master-merge
add a1b50ad HDDS-5321. Merge master branch at 12e2918 into upgrade branch
(#2316)
add 2c0adac HDDS-5109. Track OM prepare intermittent integration test
failure. (#2288)
add 1290b87 Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
add 4c8b62f HDDS-5333. Remove getRequestType method from OM request
classes.
add b335431 HDDS-5336. Fix datanode capacity related race condition.
(#2327)
add 92eb6a2 Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
add 4b4e475 Fix checkstyle after merge.
add 4506883 Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
add 039afb3 Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
add 01e8811 Merge remote-tracking branch 'upstream/master' into
HDDS-3698-nonrolling-upgrade
add aaeae47 HDDS-5415. Fix TestSCMNodeManager after merge of master at
1d8f972 into upgrade branch (#2388)
add 240af4d Merge branch 'master' into HDDS-3698-nonrolling-upgrade
new 5af5a33 Merge remote-tracking branch 'ssh-upstream/master' into
ssh-upstream-upgrade-branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 2 +-
hadoop-hdds/common/pom.xml | 12 +
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 17 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +
.../protocol/StorageContainerLocationProtocol.java | 8 +
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 76 ++
.../hdds/upgrade/HDDSLayoutVersionManager.java | 97 ++
.../hadoop/hdds/upgrade/HDDSUpgradeAction.java | 15 +-
.../apache/hadoop/hdds/upgrade}/package-info.java | 5 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 1 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 +-
.../org/apache/hadoop/ozone/common/Storage.java | 41 +-
.../apache/hadoop/ozone/common/StorageInfo.java | 33 +-
.../upgrade/AbstractLayoutVersionManager.java | 162 +++
.../ozone/upgrade/BasicUpgradeFinalizer.java | 367 +++++++
.../DefaultUpgradeFinalizationExecutor.java | 66 ++
.../apache/hadoop/ozone/upgrade/LayoutFeature.java | 79 ++
.../upgrade/LayoutVersionInstanceFactory.java | 253 +++++
.../hadoop/ozone/upgrade/LayoutVersionManager.java | 76 ++
.../ozone/upgrade/LayoutVersionManagerMXBean.java | 18 +-
.../hadoop/ozone/upgrade/UpgradeActionHdds.java | 39 +-
.../hadoop/ozone/upgrade/UpgradeException.java | 159 +--
.../hadoop/ozone/upgrade/UpgradeFinalizer.java | 195 ++++
.../hadoop/ozone/upgrade/VersionFactoryKey.java | 70 ++
.../apache/hadoop/ozone/upgrade}/package-info.java | 12 +-
.../hdds/upgrade/TestHDDSLayoutVersionManager.java | 102 ++
.../hadoop/hdds/upgrade/test/MockComponent.java | 67 ++
.../InjectedUpgradeFinalizationExecutor.java | 128 +++
.../upgrade/TestAbstractLayoutVersionManager.java | 203 ++++
.../ozone/upgrade/TestBasicUpgradeFinalizer.java | 126 +++
.../TestDefaultUpgradeFinalizationExecutor.java | 99 ++
.../upgrade/TestLayoutVersionInstanceFactory.java | 198 ++++
.../ozone/upgrade/TestUpgradeFinalizerActions.java | 210 ++++
.../hadoop/ozone/upgrade/TestUpgradeUtils.java | 54 +
.../hadoop/hdds/conf/ConfigFileGenerator.java | 17 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +
.../container/common/DatanodeLayoutStorage.java | 98 ++
...utVersion.java => HDDSVolumeLayoutVersion.java} | 16 +-
.../common/statemachine/DatanodeStateMachine.java | 49 +
.../common/statemachine/StateContext.java | 3 +-
.../FinalizeNewLayoutVersionCommandHandler.java | 121 +++
.../states/endpoint/HeartbeatEndpointTask.java | 65 +-
.../states/endpoint/RegisterEndpointTask.java | 60 +-
.../server/ratis/ContainerStateMachine.java | 4 +-
.../container/common/utils/HddsVolumeUtil.java | 6 +-
.../ozone/container/common/volume/HddsVolume.java | 7 +-
.../container/keyvalue/KeyValueContainer.java | 4 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 95 ++
.../upgrade/DatanodeMetadataFeatures.java | 53 +
.../upgrade/DatanodeSchemaV2FinalizeAction.java | 48 +
.../ScmHAFirstUpgradeLayoutChangeAction.java | 45 +
.../ozone/container/upgrade/UpgradeUtils.java | 40 +-
.../ozone/container/upgrade}/package-info.java | 13 +-
.../protocol/StorageContainerDatanodeProtocol.java | 6 +-
.../protocol/StorageContainerNodeProtocol.java | 12 +-
.../commands/FinalizeNewLayoutVersionCommand.java | 80 ++
...inerDatanodeProtocolClientSideTranslatorPB.java | 11 +-
...inerDatanodeProtocolServerSideTranslatorPB.java | 16 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 11 +-
.../common/TestDatanodeLayOutVersion.java | 10 +-
.../common/helpers/TestDatanodeVersionFile.java | 4 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 12 +-
.../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 96 ++
.../docs/content/design/nonrolling-upgrade.md | 32 +
hadoop-hdds/docs/content/design/omprepare.md | 60 ++
.../docs/content/design/upgrade-dev-primer.md | 111 ++
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 +
...inerLocationProtocolClientSideTranslatorPB.java | 54 +-
.../src/main/proto/ScmAdminProtocol.proto | 21 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 23 +
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 14 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 5 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 43 +-
...andler.java => HealthyReadOnlyNodeHandler.java} | 41 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 39 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 216 +++-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 9 +-
...va => ReadOnlyHealthyToHealthyNodeHandler.java} | 15 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 170 +++-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 10 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 15 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 12 +-
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 25 +
.../hdds/scm/pipeline/SCMPipelineManager.java | 29 +-
...inerLocationProtocolServerSideTranslatorPB.java | 63 ++
.../hdds/scm/server/SCMClientProtocolServer.java | 38 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 31 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 24 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 9 +-
.../hdds/scm/server/StorageContainerManager.java | 101 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 161 +++
.../ScmHAUnfinalizedStateValidationAction.java | 72 ++
.../ScmOnFinalizeActionForDatanodeSchemaV2.java | 46 +
.../hdds/scm/server/upgrade}/package-info.java | 5 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 19 +
.../hadoop/hdds/scm/container/MockNodeManager.java | 26 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 40 +-
.../TestIncrementalContainerReportHandler.java | 14 +-
.../algorithms/TestContainerPlacementFactory.java | 4 +-
.../TestSCMContainerPlacementCapacity.java | 4 +-
.../TestSCMContainerPlacementRackAware.java | 7 +-
.../TestSCMContainerPlacementRandom.java | 4 +-
.../hdds/scm/node/TestContainerPlacement.java | 23 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 2 +-
.../hdds/scm/node/TestNodeReportHandler.java | 16 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 57 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 532 +++++++++-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 17 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 14 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 10 +
.../TestSCMHAUnfinalizedStateValidationAction.java | 128 +++
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 75 ++
.../hadoop/hdds/scm/upgrade/package-info.java | 21 +
.../ozone/container/common/TestEndPoint.java | 20 +-
.../testutils/ReplicationNodeManagerMock.java | 23 +-
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 63 +-
.../hdds/scm/cli/ContainerOperationClient.java | 16 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 13 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 31 +
.../apache/hadoop/ozone/om/OmUpgradeConfig.java | 82 ++
.../hadoop/ozone/om/exceptions/OMException.java | 10 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 12 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 105 ++
...OzoneManagerProtocolClientSideTranslatorPB.java | 110 +-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 31 +
.../compose/{upgrade => ozone-om-prepare}/.env | 10 +-
.../docker-compose.yaml | 75 +-
.../{upgrade => ozone-om-prepare}/docker-config | 28 +-
.../dist/src/main/compose/ozone-om-prepare/test.sh | 74 ++
hadoop-ozone/dist/src/main/compose/testlib.sh | 22 +-
.../dist/src/main/compose/upgrade/README.md | 95 +-
.../src/main/compose/upgrade/{ => compose/ha}/.env | 7 +-
.../upgrade/{ => compose/ha}/docker-compose.yaml | 120 ++-
.../compose/upgrade/{ => compose/ha}/docker-config | 30 +-
.../upgrade/{1.0.0/test.sh => compose/ha/load.sh} | 14 +-
.../main/compose/upgrade/{ => compose/non-ha}/.env | 2 +-
.../{ => compose/non-ha}/docker-compose.yaml | 99 +-
.../upgrade/{ => compose/non-ha}/docker-config | 13 +-
.../{0.5.0/test.sh => compose/non-ha/load.sh} | 13 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 30 +-
.../dist/src/main/compose/upgrade/testlib.sh | 135 ++-
.../src/main/compose/upgrade/upgrade_to_current.sh | 51 -
.../manual-upgrade/0.5.0-1.1.0/callback.sh} | 26 +-
.../manual-upgrade/driver.sh} | 63 +-
.../non-rolling-upgrade/1.0.0-1.1.0/callback.sh | 79 ++
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 101 ++
.../dist/src/main/compose/versions/0.5.0.sh | 4 -
.../dist/src/main/compose/versions/1.0.0.sh | 4 -
.../dist/src/main/compose/versions/1.1.0.sh | 4 -
.../dist/src/main/compose/versions/README.md | 3 -
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 8 +
.../license/bin/licenses/LICENSE-org.aspectj.html | 91 ++
.../LICENSE-org.reflections-reflections.txt | 14 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 69 +-
.../main/smoketest/omha/om-cancel-prepare.robot | 43 +
.../dist/src/main/smoketest/omha/om-prepare.robot | 49 +
.../dist/src/main/smoketest/omha/om-prepared.robot | 36 +
.../src/main/smoketest/upgrade/check-mlv.robot | 41 +
.../dist/src/main/smoketest/upgrade/finalize.robot | 31 +
.../dist/src/main/smoketest/upgrade/generate.robot | 33 +
.../dist/src/main/smoketest/upgrade/prepare.robot | 26 +
.../dist/src/main/smoketest/upgrade/validate.robot | 30 +
.../dist/src/shell/upgrade/{v1.sh => 1.0.0.sh} | 2 +-
.../upgrade/{v1 => 1.0.0}/01-migrate-scm-db.sh | 0
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 1067 ++++++++++++++++++++
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 20 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 45 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 35 +-
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 2 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 2 +
.../hadoop/ozone/TestStorageContainerManager.java | 27 +-
.../apache/hadoop/ozone/container/TestHelper.java | 4 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 254 +++++
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 1 -
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 80 ++
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 510 ++++++++++
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 1 +
.../src/main/proto/OmClientProtocol.proto | 80 ++
.../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-ozone/ozone-manager/pom.xml | 43 +-
.../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 7 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 30 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 238 ++++-
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 311 ++++++
.../hadoop/ozone/om/OzoneManagerStarter.java | 32 +-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 15 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 28 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 57 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 9 +
.../hadoop/ozone/om/request/OMClientRequest.java | 7 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 1 +
.../request/bucket/OMBucketSetPropertyRequest.java | 1 +
.../request/bucket/acl/OMBucketAddAclRequest.java | 5 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 1 +
.../om/request/file/OMDirectoryCreateRequest.java | 1 +
.../ozone/om/request/file/OMFileCreateRequest.java | 7 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 1 +
.../ozone/om/request/key/OMKeyDeleteRequest.java | 1 +
.../ozone/om/request/key/OMKeyPurgeRequest.java | 1 +
.../ozone/om/request/key/OMKeyRenameRequest.java | 1 +
.../om/request/key/acl/OMKeyAddAclRequest.java | 1 +
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 1 +
.../om/request/key/acl/OMKeySetAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixAddAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 1 +
.../key/acl/prefix/OMPrefixSetAclRequest.java | 1 +
.../S3InitiateMultipartUploadRequest.java | 1 +
.../S3MultipartUploadCompleteRequest.java | 1 +
.../om/request/s3/security/S3GetSecretRequest.java | 1 +
.../security/OMCancelDelegationTokenRequest.java | 2 +-
.../security/OMGetDelegationTokenRequest.java | 2 +-
.../security/OMRenewDelegationTokenRequest.java | 1 +
.../om/request/upgrade/OMCancelPrepareRequest.java | 97 ++
.../request/upgrade/OMFinalizeUpgradeRequest.java | 116 +++
.../ozone/om/request/upgrade/OMPrepareRequest.java | 257 +++++
.../security => request/upgrade}/package-info.java | 15 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 2 +
.../om/request/volume/OMVolumeSetOwnerRequest.java | 1 +
.../om/request/volume/OMVolumeSetQuotaRequest.java | 1 +
.../hadoop/ozone/om/response/CleanupTableInfo.java | 10 +-
.../om/response/key/OMOpenKeysDeleteRequest.java | 1 +
.../om/response/s3/security/package-info.java | 2 +-
.../response/upgrade/OMCancelPrepareResponse.java | 46 +
.../upgrade/OMFinalizeUpgradeResponse.java | 58 ++
.../om/response/upgrade/OMPrepareResponse.java | 60 ++
.../{s3/security => upgrade}/package-info.java | 15 +-
.../BelongsToLayoutVersion.java} | 23 +-
.../DisallowedUntilLayoutVersion.java} | 31 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 80 ++
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 114 +++
.../ozone/om/upgrade/OMLayoutVersionManager.java | 210 ++++
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 50 +
.../OmUpgradeAction.java} | 16 +-
.../hadoop/ozone/om/upgrade/UpgradeActionOm.java | 37 +
.../s3/security => upgrade}/package-info.java | 7 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 13 +-
.../protocolPB/OzoneManagerRequestHandler.java | 62 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 9 +
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 28 +
.../om/ratis/TestOzoneManagerStateMachine.java | 108 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 5 +
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +
.../upgrade/TestOMCancelPrepareRequest.java | 110 ++
.../request/volume/TestOMVolumeDeleteRequest.java | 2 +-
.../om/request/volume/TestOMVolumeRequest.java | 9 +-
.../ozone/om/response/TestCleanupTableInfo.java | 26 +-
.../hadoop/ozone/om/upgrade/MockOmRequest.java} | 20 +-
.../ozone/om/upgrade/OMLayoutFeatureUtil.java | 66 ++
.../om/upgrade/TestOMLayoutFeatureAspect.java | 99 ++
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 290 ++++++
.../ozone/om/upgrade/TestOMVersionManager.java | 195 ++++
.../TestOmVersionManagerRequestFactory.java | 111 ++
.../om/upgrade/TestOzoneManagerPrepareState.java | 283 ++++++
hadoop-ozone/recon/pom.xml | 5 +
.../ozone/recon/api/ClusterStateEndpoint.java | 3 +-
.../hadoop/ozone/recon/api/NodeEndpoint.java | 5 +
.../ozone/recon/api/types/DatanodeMetadata.java | 13 +
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 35 +-
.../scm/ReconStorageContainerManagerFacade.java | 10 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 13 +-
.../ozone/recon/api/TestOpenContainerCount.java | 7 +-
.../scm/AbstractReconContainerManagerTest.java | 13 +-
...TestReconIncrementalContainerReportHandler.java | 14 +-
.../ozone/recon/scm/TestReconNodeManager.java | 19 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 23 +-
hadoop-ozone/s3gateway/pom.xml | 5 +
.../ozone/admin/om/CancelPrepareSubCommand.java | 53 +
.../admin/om/FinalizationStatusSubCommand.java | 64 ++
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 171 ++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 44 +-
.../hadoop/ozone/admin/om/PrepareSubCommand.java | 173 ++++
.../admin/scm/FinalizationScmStatusSubcommand.java | 51 +
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 156 +++
.../admin/scm/FinalizeUpgradeCommandUtil.java | 92 ++
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 4 +-
281 files changed, 14203 insertions(+), 1130 deletions(-)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSUpgradeAction.java
(73%)
copy
{hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade}/package-info.java
(87%)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/DefaultUpgradeFinalizationExecutor.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionInstanceFactory.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManager.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManagerMXBean.java
(73%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/CleanupTableInfo.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionHdds.java
(63%)
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java
(52%)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalizer.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/VersionFactoryKey.java
copy
{hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade}/package-info.java
(80%)
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/test/MockComponent.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/InjectedUpgradeFinalizationExecutor.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestBasicUpgradeFinalizer.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestDefaultUpgradeFinalizationExecutor.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestLayoutVersionInstanceFactory.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeUtils.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeLayoutStorage.java
rename
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/{DataNodeLayoutVersion.java
=> HDDSVolumeLayoutVersion.java} (78%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeNewLayoutVersionCommandHandler.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeMetadataFeatures.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFirstUpgradeLayoutChangeAction.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/CleanupTableInfo.java
=>
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java
(51%)
copy
{hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security
=>
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade}/package-info.java
(80%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeNewLayoutVersionCommand.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDataNodeStartupSlvLessThanMlv.java
create mode 100644 hadoop-hdds/docs/content/design/nonrolling-upgrade.md
create mode 100644 hadoop-hdds/docs/content/design/omprepare.md
create mode 100644 hadoop-hdds/docs/content/design/upgrade-dev-primer.md
create mode 100644
hadoop-hdds/docs/content/feature/how-to-do-a-nonrolling-upgrade.md
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/{NonHealthyToHealthyNodeHandler.java
=> HealthyReadOnlyNodeHandler.java} (54%)
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/{NonHealthyToHealthyNodeHandler.java
=> ReadOnlyHealthyToHealthyNodeHandler.java} (78%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmHAUnfinalizedStateValidationAction.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java
copy
{hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security
=>
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade}/package-info.java
(87%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestSCMHAUnfinalizedStateValidationAction.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/package-info.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmUpgradeConfig.java
copy hadoop-ozone/dist/src/main/compose/{upgrade => ozone-om-prepare}/.env
(73%)
copy hadoop-ozone/dist/src/main/compose/{upgrade =>
ozone-om-prepare}/docker-compose.yaml (65%)
copy hadoop-ozone/dist/src/main/compose/{upgrade =>
ozone-om-prepare}/docker-config (60%)
create mode 100755 hadoop-ozone/dist/src/main/compose/ozone-om-prepare/test.sh
copy hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/ha}/.env (79%)
copy hadoop-ozone/dist/src/main/compose/upgrade/{ =>
compose/ha}/docker-compose.yaml (71%)
copy hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/ha}/docker-config
(73%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{1.0.0/test.sh =>
compose/ha/load.sh} (73%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{ => compose/non-ha}/.env
(97%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{ =>
compose/non-ha}/docker-compose.yaml (78%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{ =>
compose/non-ha}/docker-config (89%)
rename hadoop-ozone/dist/src/main/compose/upgrade/{0.5.0/test.sh =>
compose/non-ha/load.sh} (75%)
delete mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh
copy hadoop-ozone/dist/src/main/compose/{versions/0.5.0.sh =>
upgrade/upgrades/manual-upgrade/0.5.0-1.1.0/callback.sh} (64%)
mode change 100644 => 100755
rename hadoop-ozone/dist/src/main/compose/upgrade/{upgrade_to_release.sh =>
upgrades/manual-upgrade/driver.sh} (50%)
create mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.0.0-1.1.0/callback.sh
create mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh
create mode 100644
hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.aspectj.html
create mode 100644
hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.reflections-reflections.txt
create mode 100644
hadoop-ozone/dist/src/main/smoketest/omha/om-cancel-prepare.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/om-prepare.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/om-prepared.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/check-mlv.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/finalize.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/generate.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/prepare.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/validate.robot
rename hadoop-ozone/dist/src/shell/upgrade/{v1.sh => 1.0.0.sh} (85%)
rename hadoop-ozone/dist/src/shell/upgrade/{v1 => 1.0.0}/01-migrate-scm-db.sh
(100%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmStartupSlvLessThanMlv.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerPrepareState.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMCancelPrepareRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMPrepareRequest.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/s3/security
=> request/upgrade}/package-info.java (73%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCancelPrepareResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMPrepareResponse.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{s3/security
=> upgrade}/package-info.java (73%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/CleanupTableInfo.java
=> upgrade/BelongsToLayoutVersion.java} (66%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/CleanupTableInfo.java
=> upgrade/DisallowedUntilLayoutVersion.java} (61%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizer.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/s3/security/package-info.java
=> upgrade/OmUpgradeAction.java} (67%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/UpgradeActionOm.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{response/s3/security
=> upgrade}/package-info.java (86%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMCancelPrepareRequest.java
copy
hadoop-ozone/ozone-manager/src/{main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java
=> test/java/org/apache/hadoop/ozone/om/upgrade/MockOmRequest.java} (65%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizer.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOmVersionManagerRequestFactory.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOzoneManagerPrepareState.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/CancelPrepareSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizeUpgradeSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/PrepareSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/FinalizationScmStatusSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/FinalizeScmUpgradeSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/FinalizeUpgradeCommandUtil.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
