This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-6517-Snapshot in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 106b193c2ae9e111ba273b1f9dcb1c1a6e9057cb Merge: aea5edfe99 6389d01605 Author: Siyao Meng <[email protected]> AuthorDate: Tue Jan 31 15:05:46 2023 -0800 Merge remote-tracking branch 'asf/master' into HDDS-6517-Snapshot Conflicts: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java pom.xml Change-Id: Ia2415f4046f917ac3b5f911f12fda8f6f372a8d9 .../hadoop/ozone/client/io/ECStreamTestUtil.java | 3 +- .../org/apache/hadoop/hdds/client/BlockID.java | 5 + .../hadoop/hdds/client/ContainerBlockID.java | 3 + .../hdds/client/DefaultReplicationConfig.java | 94 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 5 +- .../java/org/apache/hadoop/hdds/utils/IOUtils.java | 17 + .../common/src/main/resources/ozone-default.xml | 16 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 2 +- .../container/common/helpers/ContainerUtils.java | 20 + .../container/common/impl/ContainerDataYaml.java | 8 +- .../common/interfaces/ContainerPacker.java | 3 +- .../common/statemachine/DatanodeStateMachine.java | 65 +- .../commandhandler/DeleteBlocksCommandHandler.java | 13 +- .../ReconstructECContainersCommandHandler.java | 19 +- .../ReplicateContainerCommandHandler.java | 24 +- .../container/common/volume/StorageVolume.java | 4 + .../ECReconstructionCoordinatorTask.java | 66 +- .../reconstruction/ECReconstructionSupervisor.java | 111 - .../container/ec/reconstruction/TokenHelper.java | 16 +- .../container/keyvalue/KeyValueContainer.java | 86 +- .../container/keyvalue/KeyValueContainerCheck.java | 10 +- .../container/keyvalue/KeyValueContainerData.java | 27 +- .../KeyValueContainerMetadataInspector.java | 14 +- .../ozone/container/keyvalue/KeyValueHandler.java | 15 +- .../container/keyvalue/TarContainerPacker.java | 146 +- .../helpers/KeyValueContainerLocationUtil.java | 41 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 10 +- .../container/keyvalue/impl/BlockManagerImpl.java | 14 +- .../background/BlockDeletingService.java | 7 +- .../container/ozoneimpl/ContainerController.java | 3 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 6 +- .../replication/AbstractReplicationTask.java | 123 + .../container/replication/ContainerDownloader.java | 2 +- .../container/replication/ContainerImporter.java | 126 + .../container/replication/ContainerUploader.java} | 22 +- .../replication/CopyContainerResponseStream.java | 47 + .../replication/DownloadAndImportReplicator.java | 101 +- .../replication/GrpcContainerUploader.java | 100 + .../container/replication/GrpcOutputStream.java | 41 +- .../replication/GrpcReplicationClient.java | 21 +- .../replication/GrpcReplicationService.java | 20 +- .../container/replication/MeasuredReplicator.java | 23 +- .../container/replication/PushReplicator.java | 76 + .../container/replication/ReplicationServer.java | 7 +- .../replication/ReplicationSupervisor.java | 170 +- .../replication/ReplicationSupervisorMetrics.java | 7 +- .../container/replication/ReplicationTask.java | 88 +- .../replication/SendContainerOutputStream.java | 43 + .../replication/SendContainerRequestHandler.java | 131 + .../replication/SimpleContainerDownloader.java | 34 +- .../commands/ReplicateContainerCommand.java | 76 +- .../hadoop/ozone/TestHddsSecureDatanodeInit.java | 2 +- .../container/common/TestBlockDeletingService.java | 19 +- .../TestSchemaOneBackwardsCompatibility.java | 27 +- .../TestSchemaTwoBackwardsCompatibility.java | 4 +- .../common/helpers/TestContainerUtils.java | 10 + .../common/statemachine/TestStateContext.java | 9 +- .../TestECReconstructionSupervisor.java | 158 - .../container/keyvalue/TestKeyValueContainer.java | 10 +- .../TestKeyValueContainerIntegrityChecks.java | 4 +- .../TestKeyValueContainerMetadataInspector.java | 8 +- .../container/keyvalue/TestTarContainerPacker.java | 77 +- .../container/ozoneimpl/TestContainerReader.java | 13 +- .../container/ozoneimpl/TestOzoneContainer.java | 7 +- ...OutputStream.java => GrpcOutputStreamTest.java} | 46 +- .../ReplicationSupervisorScheduling.java | 85 +- .../TestCopyContainerResponseStream.java | 50 + .../replication/TestMeasuredReplicator.java | 32 +- .../container/replication/TestPushReplicator.java | 140 + .../replication/TestReplicationSupervisor.java | 242 +- .../replication/TestSendContainerOutputStream.java | 50 + .../replication/TestSimpleContainerDownloader.java | 50 +- .../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +- hadoop-hdds/docs/content/design/trash.md | 5 - hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +- hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 4 +- .../hadoop/hdds/security/OzoneSecretKey.java | 9 +- .../hadoop/hdds/security/OzoneSecretManager.java | 54 +- .../security/ssl/PemFileBasedKeyStoresFactory.java | 2 +- .../token/ContainerTokenSecretManager.java | 5 +- .../token/OzoneBlockTokenSecretManager.java | 5 +- .../token/ShortLivedTokenSecretManager.java | 9 +- .../x509/certificate/authority/BaseApprover.java | 4 +- .../certificate/authority/DefaultApprover.java | 2 +- .../certificate/authority/DefaultCAServer.java | 10 +- .../{PKIProfiles => profile}/DefaultCAProfile.java | 2 +- .../{PKIProfiles => profile}/DefaultProfile.java | 2 +- .../{PKIProfiles => profile}/PKIProfile.java | 2 +- .../{PKIProfiles => profile}/package-info.java | 2 +- .../x509/certificate/client/CertificateClient.java | 10 +- .../client/CertificateNotification.java} | 19 +- .../client/CommonCertificateClient.java | 4 +- .../certificate/client/DNCertificateClient.java | 8 +- .../client/DefaultCertificateClient.java | 44 +- .../certificate/client/ReconCertificateClient.java | 8 +- .../certificate/client/SCMCertificateClient.java | 4 +- .../utils/CertificateSignRequest.java | 4 +- .../utils/SelfSignedCertificate.java | 4 +- .../CertificateException.java | 2 +- .../{exceptions => exception}/package-info.java | 2 +- .../hdds/security/x509/keys/SecurityUtil.java | 2 +- .../hadoop/hdds/utils/DBCheckpointServlet.java | 69 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 18 +- .../token/TestOzoneBlockTokenSecretManager.java | 2 +- .../hdds/security/token/TokenVerifierTests.java | 2 +- .../hdds/security/x509/CertificateClientTest.java | 9 +- .../x509/certificate/authority/MockApprover.java | 2 +- .../certificate/authority/TestDefaultCAServer.java | 10 +- .../certificate/authority/TestDefaultProfile.java | 4 +- .../client/TestDefaultCertificateClient.java | 4 +- .../x509/certificate/utils/TestCRLCodec.java | 1 - .../certificate/utils/TestCertificateCodec.java | 1 - .../utils}/TestCertificateSignRequest.java | 3 +- .../utils}/TestRootCertificate.java | 8 +- .../src/main/proto/DatanodeClientProtocol.proto | 11 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 7 + hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 20 - .../hdds/scm/container/ContainerReportHandler.java | 6 +- .../AbstractOverReplicationHandler.java | 34 +- .../replication/ContainerHealthResult.java | 21 +- .../replication/ContainerReplicaCount.java | 4 +- .../replication/ECContainerReplicaCount.java | 15 +- .../replication/ECMisReplicationHandler.java | 16 +- .../replication/ECOverReplicationHandler.java | 132 +- .../replication/ECUnderReplicationHandler.java | 60 +- .../LegacyRatisContainerReplicaCount.java | 51 + .../replication/LegacyReplicationManager.java | 831 +++-- .../replication/MisReplicationHandler.java | 36 +- .../replication/OverReplicatedProcessor.java | 5 +- .../replication/RatisContainerReplicaCount.java | 170 +- .../replication/RatisMisReplicationHandler.java | 12 +- .../replication/RatisOverReplicationHandler.java | 66 +- .../replication/RatisUnderReplicationHandler.java | 58 +- .../container/replication/ReplicationManager.java | 51 +- .../replication/UnderReplicatedProcessor.java | 5 +- .../replication/UnhealthyReplicationHandler.java | 5 +- .../replication/UnhealthyReplicationProcessor.java | 13 +- .../health/ClosedWithUnhealthyReplicasHandler.java | 2 +- .../health/DeletingContainerHandler.java | 3 +- .../replication/health/EmptyContainerHandler.java | 2 +- ...Handler.java => MismatchedReplicasHandler.java} | 44 +- .../health/RatisReplicationCheckHandler.java | 82 +- .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 8 +- .../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 8 +- .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 3 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 14 +- .../hdds/scm/server/StorageContainerManager.java | 28 +- .../container/replication/ReplicationTestUtil.java | 10 + .../replication/TestECMisReplicationHandler.java | 3 +- .../replication/TestECOverReplicationHandler.java | 130 +- .../replication/TestECUnderReplicationHandler.java | 62 +- .../replication/TestLegacyReplicationManager.java | 3586 +++++++++++--------- .../replication/TestMisReplicationHandler.java | 14 +- .../replication/TestOverReplicatedProcessor.java | 9 +- .../TestRatisContainerReplicaCount.java | 46 + .../TestRatisMisReplicationHandler.java | 3 +- .../TestRatisOverReplicationHandler.java | 76 +- .../TestRatisUnderReplicationHandler.java | 14 +- .../replication/TestReplicationManager.java | 69 +- .../replication/TestUnderReplicatedProcessor.java | 17 +- .../TestClosedWithUnhealthyReplicasHandler.java | 2 +- .../health/TestDeletingContainerHandler.java | 2 +- .../health/TestEmptyContainerHandler.java | 2 +- ...ler.java => TestMismatchedReplicasHandler.java} | 56 +- .../health/TestRatisReplicationCheckHandler.java | 131 + .../org/apache/ozone/test/SpyInputStream.java} | 37 +- .../org/apache/ozone/test/SpyOutputStream.java} | 37 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 7 +- .../client/checksum/ECFileChecksumHelper.java | 19 +- .../hadoop/ozone/client/io/ECKeyOutputStream.java | 12 +- .../hadoop/ozone/client/MockOmTransport.java | 22 +- .../hadoop/ozone/client/TestOzoneECClient.java | 11 +- .../main/java/org/apache/hadoop/ozone/OFSPath.java | 20 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +- .../hadoop/ozone/om/helpers/OmBucketArgs.java | 4 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 18 +- .../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 - .../ozone/security/OzoneTokenIdentifier.java | 3 +- .../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 2 +- .../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 11 +- .../src/main/compose/ozonesecure/docker-config | 7 + hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 3 - hadoop-ozone/dist/src/main/license/jar-report.txt | 5 - .../dist/src/main/smoketest/createbucketenv.robot | 2 +- .../dist/src/main/smoketest/createmrenv.robot | 2 +- .../main/smoketest/debug/ozone-debug-tests.robot | 2 +- .../dist/src/main/smoketest/gdpr/gdpr.robot | 8 +- .../dist/src/main/smoketest/ozonefs/setup.robot | 12 +- .../dist/src/main/smoketest/recon/recon-api.robot | 8 +- .../src/main/smoketest/s3/MultipartUpload.robot | 1 + .../main/smoketest/security/ozone-secure-fs.robot | 51 + .../smoketest/security/ozone-secure-tenant.robot | 7 + hadoop-ozone/integration-test/pom.xml | 5 - .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 3 +- .../hadoop/fs/ozone/TestOzoneFileChecksum.java | 2 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 156 +- .../hdds/scm/TestSCMDatanodeProtocolServer.java | 4 +- .../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 8 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 - .../hadoop/ozone/TestSecureOzoneCluster.java | 187 +- .../ozone/TestStorageContainerManagerHelper.java | 2 +- .../ozone/client/CertificateClientTestImpl.java | 104 +- .../ozone/client/rpc/TestECKeyOutputStream.java | 5 +- .../client/rpc/TestFailureHandlingByClient.java | 4 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 2 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 74 +- .../client/rpc/TestOzoneRpcClientWithRatis.java | 3 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 9 +- .../client/rpc/TestValidateBCSIDOnRestart.java | 2 +- .../ozone/container/TestECContainerRecovery.java | 30 +- .../commandhandler/TestBlockDeletion.java | 7 +- .../ozoneimpl/TestOzoneContainerWithTLS.java | 9 +- .../ozoneimpl/TestSecureOzoneContainer.java | 4 +- .../server/TestSecureContainerServer.java | 6 +- .../hadoop/ozone/fsck/TestContainerMapper.java | 8 +- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 58 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 24 +- .../apache/hadoop/ozone/om/TestObjectStore.java | 4 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 + .../hadoop/ozone/om/TestOzoneManagerRestart.java | 4 + .../ozone/recon/TestReconWithOzoneManagerFSO.java | 14 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 58 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 64 +- .../java/org/apache/hadoop/ozone/om/OMStorage.java | 184 +- .../apache/hadoop/ozone/om/OmMetadataReader.java | 2 +- .../apache/hadoop/ozone/om/OzoneConfigUtil.java | 25 +- .../hadoop/ozone/om/OzoneListStatusHelper.java | 49 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 76 +- .../hadoop/ozone/om/TrashOzoneFileSystem.java | 22 +- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 6 +- .../ozone/om/lock/OBSKeyPathLockStrategy.java | 14 +- .../ozone/om/lock/RegularBucketLockStrategy.java | 20 +- .../BucketLayoutAwareOMKeyRequestFactory.java | 2 +- .../om/request/bucket/OMBucketCreateRequest.java | 123 +- .../request/bucket/OMBucketSetPropertyRequest.java | 10 +- .../ozone/om/request/file/OMFileRequest.java | 15 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 34 - .../om/request/volume/OMVolumeSetQuotaRequest.java | 16 + .../S3InitiateMultipartUploadResponseWithFSO.java | 3 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +- .../hadoop/ozone/security/OMCertificateClient.java | 6 +- .../OzoneDelegationTokenSecretManager.java | 17 +- .../org/apache/hadoop/ozone/om/TestOMStorage.java | 315 +- .../hadoop/ozone/om/TestOzoneConfigUtil.java | 14 +- .../ozone/om/request/bucket/TestBucketRequest.java | 4 + .../request/bucket/TestOMBucketCreateRequest.java | 26 +- .../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +- .../bucket/TestOMBucketSetPropertyRequest.java | 53 +- .../om/request/file/TestOMFileCreateRequest.java | 23 - .../om/request/key/TestOMAllocateBlockRequest.java | 24 - .../om/request/key/TestOMKeyCommitRequest.java | 71 +- .../request/key/TestOMKeyCommitRequestWithFSO.java | 11 +- .../om/request/key/TestOMKeyCreateRequest.java | 23 - .../volume/TestOMVolumeSetQuotaRequest.java | 23 + .../fs/ozone/BasicOzoneClientAdapterImpl.java | 2 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 44 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 35 +- .../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 +- hadoop-ozone/ozonefs-hadoop2/pom.xml | 20 + hadoop-ozone/pom.xml | 6 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 24 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 6 +- .../recon/api/handlers/BucketEntityHandler.java | 36 +- .../ozone/recon/api/handlers/BucketHandler.java | 4 + .../recon/api/handlers/DirectoryEntityHandler.java | 26 +- .../ozone/recon/api/handlers/FSOBucketHandler.java | 20 + .../ozone/recon/api/handlers/KeyEntityHandler.java | 23 +- .../recon/api/handlers/LegacyBucketHandler.java | 14 + .../recon/api/handlers/RootEntityHandler.java | 30 +- .../recon/api/handlers/UnknownEntityHandler.java | 8 +- .../recon/api/handlers/VolumeEntityHandler.java | 32 +- .../ozone/recon/api/types/BucketObjectDBInfo.java | 168 + ...mespaceSummaryResponse.java => CountStats.java} | 70 +- .../ozone/recon/api/types/KeyObjectDBInfo.java | 154 + .../recon/api/types/NamespaceSummaryResponse.java | 138 +- .../hadoop/ozone/recon/api/types/ObjectDBInfo.java | 140 + .../ozone/recon/api/types/VolumeObjectDBInfo.java | 78 + .../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +- .../impl/StorageContainerServiceProviderImpl.java | 2 +- .../ozone/recon/tasks/OMDBUpdatesHandler.java | 15 +- .../webapps/recon/ozone-recon-web/api/db.json | 135 +- .../src/views/diskUsage/diskUsage.tsx | 131 +- .../recon/api/TestNSSummaryEndpointWithFSO.java | 61 +- .../recon/api/TestNSSummaryEndpointWithLegacy.java | 62 +- .../hadoop/ozone/recon/common/CommonUtils.java | 225 ++ .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 12 +- hadoop-ozone/tools/pom.xml | 5 +- .../ozone/admin/nssummary/NSSummaryAdmin.java | 9 +- .../ozone/debug/container/ExportSubcommand.java | 2 +- .../ozone/freon/ClosedContainerReplicator.java | 20 +- pom.xml | 139 +- 294 files changed, 9290 insertions(+), 4714 deletions(-) diff --cc hadoop-hdds/rocksdb-checkpoint-differ/pom.xml index 8e372ffc3c,0000000000..842994cc28 mode 100644,000000..100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml +++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml @@@ -1,213 -1,0 +1,193 @@@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 +https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.ozone</groupId> + <artifactId>hdds</artifactId> + <version>1.4.0-SNAPSHOT</version> + </parent> + + <artifactId>rocksdb-checkpoint-differ</artifactId> + <version>1.4.0-SNAPSHOT</version> + <description>RocksDB Checkpoint Differ</description> + <name>RocksDB Checkpoint Differ</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.rocksdb</groupId> + <artifactId>rocksdbjni</artifactId> + </dependency> + <dependency> + <groupId>org.apache.ozone</groupId> + <artifactId>hdds-common</artifactId> + <version>${hdds.version}</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> - <dependency> - <groupId>com.github.vlsi.mxgraph</groupId> - <artifactId>jgraphx</artifactId> - <version>4.2.2</version> - </dependency> - <dependency> - <groupId>org.jgrapht</groupId> - <artifactId>jgrapht-core</artifactId> - <version>1.5.0</version> - </dependency> - <dependency> - <groupId>org.jgrapht</groupId> - <artifactId>jgrapht-guava</artifactId> - <version>1.5.0</version> - </dependency> - <dependency> - <groupId>org.jgrapht</groupId> - <artifactId>jgrapht-ext</artifactId> - <version>1.4.0</version> - </dependency> + <dependency> + <groupId>com.github.spotbugs</groupId> + <artifactId>spotbugs-annotations</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-reload4j</artifactId> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.ozone</groupId> + <artifactId>hdds-test-utils</artifactId> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <resources> + <resource> + <directory>${basedir}/src/main/resources</directory> + <excludes> + <exclude>ozone-version-info.properties</exclude> + </excludes> + <filtering>false</filtering> + </resource> + <resource> + <directory>${basedir}/src/main/resources</directory> + <includes> + <include>ozone-version-info.properties</include> + </includes> + <filtering>true</filtering> + </resource> + </resources> + <plugins> + <plugin> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> + <executions> + <execution> + <id>version-info</id> + <phase>generate-resources</phase> + <goals> + <goal>version-info</goal> + </goals> + <configuration> + <source> + <directory>${basedir}/../</directory> + <includes> + <include>*/src/main/java/**/*.java</include> + <include>*/src/main/proto/*.proto</include> + </includes> + </source> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.github.spotbugs</groupId> + <artifactId>spotbugs-maven-plugin</artifactId> + <configuration> + <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile> + </configuration> + </plugin> + <plugin> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>depcheck</id> + <phase></phase> + </execution> + <execution> + <id>banned-rocksdb-imports</id> + <phase>process-sources</phase> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <RestrictImports> + <includeTestCode>false</includeTestCode> + <reason>Use managed RocksObjects under org.apache.hadoop.hdds.utils.db.managed instead.</reason> + <!-- By default, ban all the classes in org.rocksdb --> + <bannedImport>org.rocksdb.**</bannedImport> + <allowedImports> + <allowedImport>org.rocksdb.AbstractEventListener</allowedImport> + <allowedImport>org.rocksdb.Checkpoint</allowedImport> + <allowedImport>org.rocksdb.ColumnFamilyDescriptor</allowedImport> + <allowedImport>org.rocksdb.ColumnFamilyHandle</allowedImport> + <allowedImport>org.rocksdb.ColumnFamilyOptions</allowedImport> + <allowedImport>org.rocksdb.CompactionJobInfo</allowedImport> + <allowedImport>org.rocksdb.CompressionType</allowedImport> + <allowedImport>org.rocksdb.DBOptions</allowedImport> + <allowedImport>org.rocksdb.FlushOptions</allowedImport> + <allowedImport>org.rocksdb.LiveFileMetaData</allowedImport> + <allowedImport>org.rocksdb.Options</allowedImport> + <allowedImport>org.rocksdb.RocksDB</allowedImport> + <allowedImport>org.rocksdb.RocksDBException</allowedImport> + <allowedImport>org.rocksdb.SstFileReader</allowedImport> + <allowedImport>org.rocksdb.TableProperties</allowedImport> + <allowedImport>org.rocksdb.ReadOptions</allowedImport> + <allowedImport>org.rocksdb.SstFileReaderIterator</allowedImport> + </allowedImports> + <exclusion>org.apache.hadoop.hdds.utils.db.managed.*</exclusion> + </RestrictImports> + </rules> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <profiles> + <profile> + <id>k8s-dev</id> + <build> + <plugins> + <plugin> + <groupId>io.fabric8</groupId> + <artifactId>docker-maven-plugin</artifactId> + <configuration> + <images> + <image> + <name>${user.name}/ozone:${project.version}</name> + <build> + <dockerFileDir>${project.basedir}</dockerFileDir> + </build> + </image> + </images> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> diff --cc hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index 83ae64aaea,828dc7e27a..f62e2cf563 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@@ -460,25 -459,6 +460,22 @@@ MI org.slf4j:slf4j-reload4j +EPL 2.0 +===================== + + jakarta.annotation:jakarta.annotation-api + jakarta.ws.rs:jakarta.ws.rs-api - org.jgrapht:jgrapht-core - org.jgrapht:jgrapht-guava - org.jgrapht:jgrapht-ext + + +CDDL + GPLv2 with classpath exception +===================== + + javax.annotation:javax.annotation-api + javax.el:javax.el-api + javax.interceptor:javax.interceptor-api + javax.servlet:javax.servlet-api + + Public Domain ===================== diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index 06b8e48b6a,0000000000..e8258adfa4 mode 100644,000000..100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@@ -1,583 -1,0 +1,583 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.om; + +import java.io.IOException; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.server.OzoneAdmins; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.audit.AuditAction; +import org.apache.hadoop.ozone.audit.AuditEventStatus; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.audit.Auditor; +import org.apache.hadoop.ozone.audit.OMAction; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; +import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; +import org.apache.hadoop.ozone.security.acl.RequestContext; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_MAX; +import static org.apache.hadoop.ozone.om.KeyManagerImpl.getRemoteUser; +import static org.apache.hadoop.ozone.om.OzoneManager.getS3Auth; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; +import org.apache.hadoop.ozone.security.acl.OzoneAccessAuthorizer; +import org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer; +import org.apache.hadoop.ozone.security.acl.OzoneObj; +import org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType; +import org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import static org.apache.hadoop.util.MetricUtil.captureLatencyNs; + +/** + * OM Metadata Reading class for the OM and Snapshot managers. + * + * This abstraction manages all the metadata key/acl reading + * from a rocksDb instance, for both the OM and OM snapshots. + */ +public class OmMetadataReader implements IOmMetadataReader, Auditor { + private final KeyManager keyManager; + private final PrefixManager prefixManager; + private final VolumeManager volumeManager; + private final BucketManager bucketManager; + private final OzoneManager ozoneManager; + private final boolean isAclEnabled; + private final IAccessAuthorizer accessAuthorizer; + private final boolean isNativeAuthorizerEnabled; + private final OmMetadataReaderMetrics metrics; + private final Logger log; + private final AuditLogger audit; + private final OMPerformanceMetrics perfMetrics; + + public OmMetadataReader(KeyManager keyManager, + PrefixManager prefixManager, + OzoneManager ozoneManager, + Logger log, + AuditLogger audit, + OmMetadataReaderMetrics omMetadataReaderMetrics) { + this.keyManager = keyManager; + this.bucketManager = ozoneManager.getBucketManager(); + this.volumeManager = ozoneManager.getVolumeManager(); + this.prefixManager = prefixManager; + OzoneConfiguration configuration = ozoneManager.getConfiguration(); + this.ozoneManager = ozoneManager; + this.isAclEnabled = ozoneManager.getAclsEnabled(); + this.log = log; + this.audit = audit; + boolean allowListAllVolumes = ozoneManager.getAllowListAllVolumes(); + this.metrics = omMetadataReaderMetrics; + this.perfMetrics = ozoneManager.getPerfMetrics(); + if (isAclEnabled) { + accessAuthorizer = getACLAuthorizerInstance(configuration); + if (accessAuthorizer instanceof OzoneNativeAuthorizer) { + OzoneNativeAuthorizer authorizer = + (OzoneNativeAuthorizer) accessAuthorizer; + isNativeAuthorizerEnabled = true; + authorizer.setVolumeManager(volumeManager); + authorizer.setBucketManager(bucketManager); + authorizer.setKeyManager(keyManager); + authorizer.setPrefixManager(prefixManager); + authorizer.setOzoneAdmins( + new OzoneAdmins(ozoneManager.getOmAdminUsernames())); + authorizer.setAllowListAllVolumes(allowListAllVolumes); + } else { + isNativeAuthorizerEnabled = false; + } + } else { + accessAuthorizer = null; + isNativeAuthorizerEnabled = false; + } + } + + /** + * Lookup a key. + * + * @param args - attributes of the key. + * @return OmKeyInfo - the info about the requested key. + * @throws IOException + */ + @Override + public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException { + long start = Time.monotonicNowNanos(); + ResolvedBucket bucket = captureLatencyNs( + perfMetrics.getLookupResolveBucketLatencyNs(), + () -> ozoneManager.resolveBucketLink(args)); + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + OmKeyArgs resolvedArgs = bucket.update(args); + + try { + if (isAclEnabled) { + captureLatencyNs(perfMetrics.getLookupAclCheckLatencyNs(), + () -> checkAcls(ResourceType.KEY, StoreType.OZONE, + ACLType.READ, bucket.realVolume(), bucket.realBucket(), + args.getKeyName()) + ); + } + metrics.incNumKeyLookups(); + return keyManager.lookupKey(resolvedArgs, getClientAddress()); + } catch (Exception ex) { + metrics.incNumKeyLookupFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_KEY, + auditMap)); + } + + perfMetrics.addLookupLatency(Time.monotonicNowNanos() - start); + } + } + + @Override + public KeyInfoWithVolumeContext getKeyInfo(final OmKeyArgs args, + boolean assumeS3Context) + throws IOException { + long start = Time.monotonicNowNanos(); + + java.util.Optional<S3VolumeContext> s3VolumeContext = + java.util.Optional.empty(); + + final OmKeyArgs resolvedVolumeArgs; + if (assumeS3Context) { + S3VolumeContext context = ozoneManager.getS3VolumeContext(); + s3VolumeContext = java.util.Optional.of(context); + resolvedVolumeArgs = args.toBuilder() + .setVolumeName(context.getOmVolumeArgs().getVolume()) + .build(); + } else { + resolvedVolumeArgs = args; + } + + final ResolvedBucket bucket = captureLatencyNs( + perfMetrics.getGetKeyInfoResolveBucketLatencyNs(), + () -> ozoneManager.resolveBucketLink(resolvedVolumeArgs)); + + boolean auditSuccess = true; - OmKeyArgs resolvedArgs = bucket.update(args); ++ OmKeyArgs resolvedArgs = bucket.update(resolvedVolumeArgs); + + try { + if (isAclEnabled) { + captureLatencyNs(perfMetrics.getGetKeyInfoAclCheckLatencyNs(), () -> + checkAcls(ResourceType.KEY, + StoreType.OZONE, ACLType.READ, + bucket.realVolume(), bucket.realBucket(), args.getKeyName()) + ); + } + + metrics.incNumGetKeyInfo(); + OmKeyInfo keyInfo = + keyManager.getKeyInfo(resolvedArgs, + OmMetadataReader.getClientAddress()); + KeyInfoWithVolumeContext.Builder builder = KeyInfoWithVolumeContext + .newBuilder() + .setKeyInfo(keyInfo); + s3VolumeContext.ifPresent(context -> { + builder.setVolumeArgs(context.getOmVolumeArgs()); + builder.setUserPrincipal(context.getUserPrincipal()); + }); + return builder.build(); + } catch (Exception ex) { + metrics.incNumGetKeyInfoFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY, + bucket.audit(resolvedVolumeArgs.toAuditMap()), ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_KEY, + bucket.audit(resolvedVolumeArgs.toAuditMap()))); + } + perfMetrics.addGetKeyInfoLatencyNs(Time.monotonicNowNanos() - start); + } + } + + @Override + public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, boolean allowPartialPrefixes) + throws IOException { + + long maxListingPageSize = ozoneManager.getConfiguration().getInt( + OZONE_FS_LISTING_PAGE_SIZE_MAX, + OZONE_FS_LISTING_PAGE_SIZE_DEFAULT); + maxListingPageSize = OzoneConfigUtil.limitValue(numEntries, + OZONE_FS_LISTING_PAGE_SIZE, OZONE_FS_LISTING_PAGE_SIZE_MAX, + maxListingPageSize); + + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + args = bucket.update(args); + + try { + if (isAclEnabled) { + checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ, + bucket.realVolume(), bucket.realBucket(), args.getKeyName()); + } + metrics.incNumListStatus(); + return keyManager.listStatus(args, recursive, startKey, + maxListingPageSize, getClientAddress(), allowPartialPrefixes); + } catch (Exception ex) { + metrics.incNumListStatusFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_STATUS, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess( + OMAction.LIST_STATUS, auditMap)); + } + } + } + + @Override + public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + args = bucket.update(args); + + try { + metrics.incNumGetFileStatus(); + return keyManager.getFileStatus(args, getClientAddress()); + } catch (IOException ex) { + metrics.incNumGetFileStatusFails(); + auditSuccess = false; + audit.logReadFailure( + buildAuditMessageForFailure(OMAction.GET_FILE_STATUS, auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess( + buildAuditMessageForSuccess(OMAction.GET_FILE_STATUS, auditMap)); + } + } + } + + @Override + public OmKeyInfo lookupFile(OmKeyArgs args) throws IOException { + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(args.toAuditMap()); + + args = bucket.update(args); + + try { + if (isAclEnabled) { + checkAcls(ResourceType.KEY, StoreType.OZONE, ACLType.READ, + bucket.realVolume(), bucket.realBucket(), args.getKeyName()); + } + metrics.incNumLookupFile(); + return keyManager.lookupFile(args, getClientAddress()); + } catch (Exception ex) { + metrics.incNumLookupFileFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.LOOKUP_FILE, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess( + OMAction.LOOKUP_FILE, auditMap)); + } + } + } + + @Override + public List<OmKeyInfo> listKeys(String volumeName, String bucketName, + String startKey, String keyPrefix, int maxKeys) throws IOException { + + ResolvedBucket bucket = ozoneManager.resolveBucketLink( + Pair.of(volumeName, bucketName)); + + boolean auditSuccess = true; + Map<String, String> auditMap = bucket.audit(); + auditMap.put(OzoneConsts.START_KEY, startKey); + auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys)); + auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix); + + try { + if (isAclEnabled) { + checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, + bucket.realVolume(), bucket.realBucket(), keyPrefix); + } + metrics.incNumKeyLists(); + return keyManager.listKeys(bucket.realVolume(), bucket.realBucket(), + startKey, keyPrefix, maxKeys); + } catch (IOException ex) { + metrics.incNumKeyListFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_KEYS, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_KEYS, + auditMap)); + } + } + } + + /** + * Returns list of ACLs for given Ozone object. + * + * @param obj Ozone object. + * @throws IOException if there is error. + */ + public List<OzoneAcl> getAcl(OzoneObj obj) throws IOException { + boolean auditSuccess = true; + + try { + if (isAclEnabled) { + checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.READ_ACL, + obj.getVolumeName(), obj.getBucketName(), obj.getKeyName()); + } + metrics.incNumGetAcl(); + switch (obj.getResourceType()) { + case VOLUME: + return volumeManager.getAcl(obj); + case BUCKET: + return bucketManager.getAcl(obj); + case KEY: + return keyManager.getAcl(obj); + case PREFIX: + return prefixManager.getAcl(obj); + + default: + throw new OMException("Unexpected resource type: " + + obj.getResourceType(), INVALID_REQUEST); + } + } catch (Exception ex) { + auditSuccess = false; + audit.logReadFailure( + buildAuditMessageForFailure(OMAction.GET_ACL, obj.toAuditMap(), ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess( + buildAuditMessageForSuccess(OMAction.GET_ACL, obj.toAuditMap())); + } + } + } + + /** + * Checks if current caller has acl permissions. + * + * @param resType - Type of ozone resource. Ex volume, bucket. + * @param store - Store type. i.e Ozone, S3. + * @param acl - type of access to be checked. + * @param vol - name of volume + * @param bucket - bucket name + * @param key - key + * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied. + */ + void checkAcls(ResourceType resType, StoreType store, + ACLType acl, String vol, String bucket, String key) + throws IOException { + UserGroupInformation user; + if (getS3Auth() != null) { + String principal = + OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId()); + user = UserGroupInformation.createRemoteUser(principal); + } else { + user = ProtobufRpcEngine.Server.getRemoteUser(); + } + + InetAddress remoteIp = ProtobufRpcEngine.Server.getRemoteIp(); + String volumeOwner = ozoneManager.getVolumeOwner(vol, acl, resType); + String bucketOwner = ozoneManager.getBucketOwner(vol, bucket, acl, resType); + + OzoneAclUtils.checkAllAcls(this, resType, store, acl, + vol, bucket, key, volumeOwner, bucketOwner, + user != null ? user : getRemoteUser(), + remoteIp != null ? remoteIp : + ozoneManager.getOmRpcServerAddr().getAddress(), + remoteIp != null ? remoteIp.getHostName() : + ozoneManager.getOmRpcServerAddr().getHostName()); + } + + + /** + * CheckAcls for the ozone object. + * + * @return true if permission granted, false if permission denied. + * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied + * and throwOnPermissionDenied set to true. + */ + @SuppressWarnings("parameternumber") + public boolean checkAcls(ResourceType resType, StoreType storeType, + ACLType aclType, String vol, String bucket, String key, + UserGroupInformation ugi, InetAddress remoteAddress, String hostName, + boolean throwIfPermissionDenied, String owner) + throws OMException { + OzoneObj obj = OzoneObjInfo.Builder.newBuilder() + .setResType(resType) + .setStoreType(storeType) + .setVolumeName(vol) + .setBucketName(bucket) + .setKeyName(key).build(); + RequestContext context = RequestContext.newBuilder() + .setClientUgi(ugi) + .setIp(remoteAddress) + .setHost(hostName) + .setAclType(ACLIdentityType.USER) + .setAclRights(aclType) + .setOwnerName(owner) + .build(); + + return checkAcls(obj, context, throwIfPermissionDenied); + } + + /** + * CheckAcls for the ozone object. + * + * @return true if permission granted, false if permission denied. + * @throws OMException ResultCodes.PERMISSION_DENIED if permission denied + * and throwOnPermissionDenied set to true. + */ + public boolean checkAcls(OzoneObj obj, RequestContext context, + boolean throwIfPermissionDenied) + throws OMException { + + if (!accessAuthorizer.checkAccess(obj, context)) { + if (throwIfPermissionDenied) { + String volumeName = obj.getVolumeName() != null ? + "Volume:" + obj.getVolumeName() + " " : ""; + String bucketName = obj.getBucketName() != null ? + "Bucket:" + obj.getBucketName() + " " : ""; + String keyName = obj.getKeyName() != null ? + "Key:" + obj.getKeyName() : ""; + log.warn("User {} doesn't have {} permission to access {} {}{}{}", + context.getClientUgi().getUserName(), context.getAclRights(), + obj.getResourceType(), volumeName, bucketName, keyName); + throw new OMException("User " + context.getClientUgi().getUserName() + + " doesn't have " + context.getAclRights() + + " permission to access " + obj.getResourceType() + " " + + volumeName + bucketName + keyName, ResultCodes.PERMISSION_DENIED); + } + return false; + } else { + return true; + } + } + + /** + * Returns an instance of {@link IAccessAuthorizer}. + * Looks up the configuration to see if there is custom class specified. + * Constructs the instance by passing the configuration directly to the + * constructor to achieve thread safety using final fields. + * + * @param conf + * @return IAccessAuthorizer + */ + private IAccessAuthorizer getACLAuthorizerInstance(OzoneConfiguration conf) { + Class<? extends IAccessAuthorizer> clazz = conf.getClass( + OZONE_ACL_AUTHORIZER_CLASS, OzoneAccessAuthorizer.class, + IAccessAuthorizer.class); + return ReflectionUtils.newInstance(clazz, conf); + } + + static String getClientAddress() { + String clientMachine = Server.getRemoteAddress(); + if (clientMachine == null) { //not a RPC client + clientMachine = ""; + } + return clientMachine; + } + + @Override + public AuditMessage buildAuditMessageForSuccess(AuditAction op, + Map<String, String> auditMap) { + + return new AuditMessage.Builder() + .setUser(getRemoteUserName()) + .atIp(Server.getRemoteAddress()) + .forOperation(op) + .withParams(auditMap) + .withResult(AuditEventStatus.SUCCESS) + .build(); + } + + @Override + public AuditMessage buildAuditMessageForFailure(AuditAction op, + Map<String, String> auditMap, Throwable throwable) { + + return new AuditMessage.Builder() + .setUser(getRemoteUserName()) + .atIp(Server.getRemoteAddress()) + .forOperation(op) + .withParams(auditMap) + .withResult(AuditEventStatus.FAILURE) + .withException(throwable) + .build(); + } + + /** + * Returns true if OzoneNativeAuthorizer is enabled and false if otherwise. + * + * @return if native authorizer is enabled. + */ + public boolean isNativeAuthorizerEnabled() { + return isNativeAuthorizerEnabled; + } + + public IAccessAuthorizer getAccessAuthorizer() { + return accessAuthorizer; + } + + private ResourceType getResourceType(OmKeyArgs args) { + if (args.getKeyName() == null || args.getKeyName().length() == 0) { + return ResourceType.BUCKET; + } + return ResourceType.KEY; + } + + +} diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 8eef324523,15b579c9c9..169e87fa39 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -71,10 -73,8 +73,11 @@@ import org.apache.hadoop.hdds.utils.db. import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.service.OMRangerBGSyncService; + import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport; import org.apache.hadoop.ozone.util.OzoneNetUtils; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; diff --cc hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 9dcd365154,3eddc01d6c..f36dad9a96 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@@ -637,13 -636,4 +637,13 @@@ public class BasicOzoneClientAdapterImp length, combineMode, ozoneClient.getObjectStore().getClientProxy()); } + + @Override + public String createSnapshot(String pathStr, String snapshotName) + throws IOException { - OFSPath ofsPath = new OFSPath(pathStr); ++ OFSPath ofsPath = new OFSPath(pathStr, config); + return objectStore.createSnapshot(ofsPath.getVolumeName(), + ofsPath.getBucketName(), + snapshotName); + } } diff --cc hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 6eddf06dac,2606bd4fdf..c1d882b2e4 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@@ -1157,13 -1159,4 +1159,13 @@@ public class BasicRootedOzoneClientAdap length, combineMode, ozoneClient.getObjectStore().getClientProxy()); } + + @Override + public String createSnapshot(String pathStr, String snapshotName) + throws IOException { - OFSPath ofsPath = new OFSPath(pathStr); ++ OFSPath ofsPath = new OFSPath(pathStr, config); + return proxy.createSnapshot(ofsPath.getVolumeName(), + ofsPath.getBucketName(), + snapshotName); + } } diff --cc pom.xml index 9a2bc44173,c330243178..5101bf85da --- a/pom.xml +++ b/pom.xml @@@ -194,7 -198,18 +198,18 @@@ xsi:schemaLocation="http://maven.apache <findbugs.version>3.0.0</findbugs.version> <spotbugs.version>3.1.12</spotbugs.version> <dnsjava.version>2.1.7</dnsjava.version> + <jakarta.activation.version>1.2.2</jakarta.activation.version> + <okhttp3.version>4.9.3</okhttp3.version> + <stax2.version>4.2.1</stax2.version> + <nimbus.version>9.8.1</nimbus.version> + <checker.version>3.12.0</checker.version> + <jakarta.inject.version>2.6.1</jakarta.inject.version> + <jakarta.annotation.version>1.3.5</jakarta.annotation.version> + <joda.time.version>2.10.6</joda.time.version> + <commons.lang.version>2.6</commons.lang.version> + <zookeeper.version>3.5.6</zookeeper.version> + <snappy.java.version>1.1.8.2</snappy.java.version> - + <compile-testing.version>0.19</compile-testing.version> <errorprone-annotations.version>2.2.0</errorprone-annotations.version> <guava.version>31.1-jre</guava.version> @@@ -1444,11 -1478,96 +1479,101 @@@ <artifactId>sqlite-jdbc</artifactId> <version>${sqlite.version}</version> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <version>${awaitility.version}</version> + </dependency> + <dependency> + <groupId>jakarta.activation</groupId> + <artifactId>jakarta.activation-api</artifactId> + <version>${jakarta.activation.version}</version> + </dependency> + <dependency> + <groupId>com.squareup.okhttp3</groupId> + <artifactId>okhttp</artifactId> + <version>${okhttp3.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.woodstox</groupId> + <artifactId>stax2-api</artifactId> + <version>${stax2.version}</version> + </dependency> + <dependency> + <groupId>com.nimbusds</groupId> + <artifactId>nimbus-jose-jwt</artifactId> + <version>${nimbus.version}</version> + </dependency> + <dependency> + <groupId>org.checkerframework</groupId> + <artifactId>checker-qual</artifactId> + <version>${checker.version}</version> + </dependency> + <dependency> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + <version>${java.dev.jna.version}</version> + </dependency> + <dependency> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna-platform</artifactId> + <version>${java.dev.jna.version}</version> + </dependency> + <dependency> + <groupId>org.glassfish.hk2.external</groupId> + <artifactId>jakarta.inject</artifactId> + <version>${jakarta.inject.version}</version> + </dependency> + <dependency> + <groupId>jakarta.annotation</groupId> + <artifactId>jakarta.annotation-api</artifactId> + <version>${jakarta.annotation.version}</version> + </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>${joda.time.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + <version>${codehaus.jackson.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + <version>${codehaus.jackson.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + <version>${codehaus.jackson.version}</version> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>${commons.lang.version}</version> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + <version>${apache.curator}</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-client</artifactId> + <version>${apache.curator}</version> + </dependency> + <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <version>${snappy.java.version}</version> + </dependency> </dependencies> </dependencyManagement> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
