Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e634d49 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e634d49 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e634d49 Branch: refs/heads/HDFS-7240 Commit: 1e634d49da09b22af845cede4951f867590db6a8 Parents: 7448475 451265a Author: Xiaoyu Yao <[email protected]> Authored: Mon Feb 26 11:32:26 2018 -0800 Committer: Xiaoyu Yao <[email protected]> Committed: Mon Feb 26 11:32:26 2018 -0800 ---------------------------------------------------------------------- dev-support/bin/verify-license-files | 145 +++ .../resources/assemblies/hadoop-yarn-dist.xml | 22 +- .../hadoop-client-minicluster/pom.xml | 1 + .../client/KerberosAuthenticator.java | 80 +- .../security/authentication/util/Signer.java | 22 +- .../client/TestKerberosAuthenticator.java | 29 + .../org/apache/hadoop/conf/ConfServlet.java | 11 +- .../org/apache/hadoop/conf/Configuration.java | 146 +-- .../org/apache/hadoop/conf/CorePropertyTag.java | 37 - .../org/apache/hadoop/conf/HDFSPropertyTag.java | 41 - .../apache/hadoop/conf/OzonePropertyTag.java | 49 - .../org/apache/hadoop/conf/PropertyTag.java | 30 - .../org/apache/hadoop/conf/YarnPropertyTag.java | 39 - .../apache/hadoop/crypto/key/KeyProvider.java | 6 +- .../hadoop/fs/CommonConfigurationKeys.java | 11 - .../fs/CommonConfigurationKeysPublic.java | 2 + .../apache/hadoop/io/compress/BZip2Codec.java | 33 +- .../java/org/apache/hadoop/log/Log4Json.java | 2 +- .../security/RuleBasedLdapGroupsMapping.java | 91 ++ .../hadoop/security/UserGroupInformation.java | 898 +++++++++---------- .../src/main/resources/core-default.xml | 26 +- .../site/markdown/AdminCompatibilityGuide.md | 278 ++++++ .../src/site/markdown/Compatibility.md | 33 +- .../conf/TestCommonConfigurationFields.java | 4 +- .../org/apache/hadoop/conf/TestConfServlet.java | 58 -- .../apache/hadoop/conf/TestConfiguration.java | 121 ++- .../apache/hadoop/fs/TestFileSystemCaching.java | 233 +++-- .../hadoop/http/TestHttpServerWithSpengo.java | 5 +- .../org/apache/hadoop/log/TestLogLevel.java | 18 +- .../TestRuleBasedLdapGroupsMapping.java | 99 ++ .../hadoop/security/TestUGILoginFromKeytab.java | 404 ++++++++- .../hadoop/security/TestUGIWithMiniKdc.java | 54 +- .../security/TestUserGroupInformation.java | 113 ++- .../delegation/web/TestWebDelegationToken.java | 4 +- .../hadoop/crypto/key/kms/server/KMSWebApp.java | 2 +- .../hadoop/hdfs/DFSOpsCountStatistics.java | 1 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 63 +- .../hadoop/hdfs/DistributedFileSystem.java | 87 ++ .../hadoop/hdfs/client/impl/LeaseRenewer.java | 2 +- .../hdfs/protocol/SnapshotDiffReport.java | 4 + .../ha/RequestHedgingProxyProvider.java | 8 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 49 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 15 + .../hadoop/hdfs/web/resources/GetOpParam.java | 3 +- .../protocol/TestHdfsFileStatusMethods.java | 2 +- .../ha/TestRequestHedgingProxyProvider.java | 45 + .../lib/service/security/DummyGroupMapping.java | 3 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 10 +- .../blockmanagement/DatanodeAdminManager.java | 27 +- .../server/datanode/BlockRecoveryWorker.java | 6 +- .../hdfs/server/datanode/ReplicaInfo.java | 1 - .../diskbalancer/command/PlanCommand.java | 16 +- .../connectors/DBNameNodeConnector.java | 2 - .../datamodel/DiskBalancerVolume.java | 4 +- .../federation/metrics/FederationRPCMBean.java | 2 + .../metrics/FederationRPCMetrics.java | 11 + .../FederationRPCPerformanceMonitor.java | 10 + .../resolver/NamenodeStatusReport.java | 8 + .../federation/router/RouterRpcClient.java | 71 +- .../federation/router/RouterRpcMonitor.java | 13 + .../federation/router/RouterRpcServer.java | 9 + .../store/driver/StateStoreDriver.java | 2 +- .../server/namenode/EncryptionZoneManager.java | 3 +- .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 28 +- .../hdfs/server/namenode/FSDirAttrOp.java | 54 +- .../hdfs/server/namenode/FSDirConcatOp.java | 5 +- .../hdfs/server/namenode/FSDirDeleteOp.java | 8 +- .../server/namenode/FSDirEncryptionZoneOp.java | 12 +- .../hdfs/server/namenode/FSDirMkdirOp.java | 3 +- .../hdfs/server/namenode/FSDirRenameOp.java | 11 +- .../hdfs/server/namenode/FSDirSnapshotOp.java | 38 +- .../server/namenode/FSDirStatAndListingOp.java | 35 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 35 +- .../hdfs/server/namenode/FSNamesystem.java | 190 ++-- .../web/resources/NamenodeWebHdfsMethods.java | 35 +- .../offlineImageViewer/XmlImageVisitor.java | 18 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 33 + .../src/main/resources/hdfs-default.xml | 21 +- .../apache/hadoop/hdfs/TestLeaseRecovery.java | 44 + .../java/org/apache/hadoop/hdfs/TestQuota.java | 107 +++ .../hdfs/qjournal/server/TestJournalNode.java | 26 +- .../router/TestRouterRPCClientRetries.java | 151 ++++ .../hdfs/server/namenode/NameNodeAdapter.java | 5 +- .../hdfs/server/namenode/TestAuditLogger.java | 3 +- .../namenode/TestAuditLoggerWithCommands.java | 4 +- .../namenode/TestTruncateQuotaUpdate.java | 10 +- .../snapshot/TestSnapshotDiffReport.java | 130 +++ .../web/resources/TestWebHdfsDataLocality.java | 2 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 72 +- .../hadoop/mapred/TaskAttemptListenerImpl.java | 15 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +- .../mapreduce/v2/app/TaskHeartbeatHandler.java | 67 +- .../mapred/TestTaskAttemptListenerImpl.java | 54 +- .../v2/app/TestTaskHeartbeatHandler.java | 38 + .../hadoop/mapreduce/v2/hs/JobHistory.java | 2 +- .../mapred/TestConcatenatedCompressedInput.java | 84 +- .../hadoop/mapred/TestTextInputFormat.java | 8 + .../src/test/resources/testdata/concat.bz2 | Bin 0 -> 208 bytes .../src/test/resources/testdata/concat.gz | Bin 0 -> 148 bytes .../testdata/testCompressThenConcat.txt.bz2 | Bin 0 -> 3056 bytes .../testdata/testCompressThenConcat.txt.gz | Bin 0 -> 3413 bytes .../testdata/testConcatThenCompress.txt.bz2 | Bin 0 -> 2567 bytes .../testdata/testConcatThenCompress.txt.gz | Bin 0 -> 2734 bytes hadoop-project/pom.xml | 16 +- hadoop-project/src/site/site.xml | 1 + .../hadoop/tools/HadoopArchiveLogsRunner.java | 26 +- .../tools/TestHadoopArchiveLogsRunner.java | 204 +++-- .../java/org/apache/hadoop/fs/s3a/Listing.java | 2 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 42 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 115 ++- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 4 +- .../fs/s3a/SimpleAWSCredentialsProvider.java | 10 +- .../fs/s3a/TemporaryAWSCredentialsProvider.java | 14 +- .../s3a/auth/AssumedRoleCredentialProvider.java | 7 +- .../src/site/markdown/tools/hadoop-aws/index.md | 62 +- .../hadoop/fs/s3a/TestSSEConfiguration.java | 171 +++- .../hadoop/fs/s3a/auth/ITestAssumeRole.java | 16 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 2 +- hadoop-tools/hadoop-sls/pom.xml | 2 + .../org/apache/hadoop/yarn/sls/SLSRunner.java | 137 ++- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 2 +- .../yarn/sls/appmaster/MRAMSimulator.java | 7 +- .../yarn/sls/appmaster/StreamAMSimulator.java | 273 ++++++ .../hadoop/yarn/sls/appmaster/package-info.java | 21 + .../yarn/sls/scheduler/RMNodeWrapper.java | 6 +- .../hadoop/yarn/sls/synthetic/SynthJob.java | 367 ++++---- .../yarn/sls/synthetic/SynthJobClass.java | 180 ---- .../sls/synthetic/SynthTraceJobProducer.java | 487 ++++++++-- .../yarn/sls/synthetic/SynthWorkload.java | 121 --- .../hadoop/yarn/sls/BaseSLSRunnerTest.java | 2 +- .../hadoop/yarn/sls/TestSLSGenericSynth.java | 76 ++ .../hadoop/yarn/sls/TestSLSStreamAMSynth.java | 76 ++ .../hadoop/yarn/sls/TestSynthJobGeneration.java | 213 ++++- .../yarn/sls/appmaster/TestAMSimulator.java | 2 +- .../src/test/resources/sls-runner.xml | 4 + .../hadoop-sls/src/test/resources/syn.json | 2 +- .../src/test/resources/syn_generic.json | 54 ++ .../src/test/resources/syn_stream.json | 46 + .../org/apache/hadoop/streaming/StreamJob.java | 2 +- .../api/protocolrecords/AllocateRequest.java | 2 +- .../api/protocolrecords/AllocateResponse.java | 2 +- .../hadoop/yarn/api/records/Container.java | 2 +- .../yarn/api/records/ProfileCapability.java | 173 ---- .../hadoop/yarn/api/records/QueueInfo.java | 35 + .../yarn/api/records/ResourceRequest.java | 43 +- .../hadoop/yarn/conf/YarnConfiguration.java | 54 +- .../yarn/util/resource/ResourceUtils.java | 20 +- .../src/main/proto/yarn_protos.proto | 7 +- .../distributedshell/ApplicationMaster.java | 22 +- .../applications/distributedshell/Client.java | 37 +- .../hadoop/yarn/service/webapp/ApiServer.java | 23 +- .../hadoop/yarn/service/TestApiServer.java | 69 +- .../hadoop/yarn/client/api/AMRMClient.java | 77 +- .../yarn/client/api/impl/AMRMClientImpl.java | 126 ++- .../client/api/impl/RemoteRequestsTable.java | 100 +-- .../apache/hadoop/yarn/client/cli/QueueCLI.java | 6 + .../hadoop/yarn/client/ProtocolHATestBase.java | 3 +- .../yarn/client/api/impl/TestAMRMClient.java | 73 +- .../impl/TestAMRMClientContainerRequest.java | 5 +- .../TestAMRMClientPlacementConstraints.java | 3 +- .../yarn/client/api/impl/TestNMClient.java | 37 +- ...TestOpportunisticContainerAllocationE2E.java | 25 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 96 +- .../impl/pb/ProfileCapabilityPBImpl.java | 126 --- .../api/records/impl/pb/QueueInfoPBImpl.java | 13 + .../records/impl/pb/ResourceRequestPBImpl.java | 40 +- .../RequestHedgingRMFailoverProxyProvider.java | 2 +- .../yarn/security/ContainerTokenIdentifier.java | 2 +- .../src/main/resources/yarn-default.xml | 10 +- .../hadoop/yarn/api/TestPBImplRecords.java | 12 +- .../hadoop/yarn/api/TestProfileCapability.java | 109 --- .../yarn/util/resource/TestResourceUtils.java | 28 +- .../resource-types/node-resources-2.xml | 11 - .../resource-types/node-resources-error-1.xml | 29 + .../resource-types/resource-types-error-2.xml | 4 +- .../registry/client/impl/zk/CuratorService.java | 3 + .../client/impl/zk/RegistrySecurity.java | 46 +- .../api/protocolrecords/NMContainerStatus.java | 2 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 1 - .../server/nodemanager/ContainerExecutor.java | 3 +- .../launcher/ContainerLaunch.java | 194 +++- .../resourceplugin/ResourcePluginManager.java | 3 +- .../impl/container-executor.c | 35 - .../launcher/TestContainerLaunch.java | 337 +++++++ .../ApplicationMasterService.java | 46 +- .../resourcemanager/DefaultAMSProcessor.java | 4 +- .../server/resourcemanager/RMServerUtils.java | 33 - .../AbstractPreemptableResourceCalculator.java | 38 +- .../capacity/AbstractPreemptionEntity.java | 4 + .../CapacitySchedulerPreemptionContext.java | 6 +- .../FifoIntraQueuePreemptionPlugin.java | 2 +- .../capacity/IntraQueueCandidatesSelector.java | 4 +- .../capacity/PreemptableResourceCalculator.java | 21 +- .../ProportionalCapacityPreemptionPolicy.java | 14 +- ...QueuePriorityContainerCandidateSelector.java | 4 +- .../monitor/capacity/TempQueuePerPartition.java | 106 ++- .../resourcemanager/recovery/RMStateStore.java | 34 +- .../rmapp/attempt/RMAppAttemptImpl.java | 2 +- .../resourcemanager/rmnode/RMNodeImpl.java | 2 +- .../rmnode/RMNodeStatusEvent.java | 3 +- .../scheduler/SchedulerUtils.java | 10 - .../scheduler/capacity/AbstractCSQueue.java | 70 +- .../scheduler/capacity/CSQueue.java | 16 +- .../scheduler/capacity/CapacityScheduler.java | 13 - .../CapacitySchedulerConfiguration.java | 20 +- .../processor/AbstractPlacementProcessor.java | 96 ++ .../constraint/processor/BatchedRequests.java | 2 +- .../processor/DisabledPlacementProcessor.java | 77 ++ .../processor/PlacementConstraintProcessor.java | 341 +++++++ .../processor/PlacementDispatcher.java | 4 +- .../processor/PlacementProcessor.java | 377 -------- .../processor/SchedulerPlacementProcessor.java | 55 ++ .../SingleConstraintAppPlacementAllocator.java | 3 +- .../webapp/CapacitySchedulerPage.java | 5 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 6 + .../webapp/dao/ResourceInfo.java | 5 +- .../TestApplicationMasterService.java | 6 - .../TestResourceTrackerService.java | 17 +- ...alCapacityPreemptionPolicyMockFramework.java | 2 +- .../recovery/RMStateStoreTestBase.java | 3 + .../recovery/TestZKRMStateStore.java | 66 ++ .../resource/MockResourceProfileManager.java | 79 -- .../TestConfigurationMutationACLPolicies.java | 2 +- .../TestSchedulerApplicationAttempt.java | 2 +- ...apacitySchedulerSchedulingRequestUpdate.java | 4 + ...CapacitySchedulerWithMultiResourceTypes.java | 110 --- .../scheduler/capacity/TestLeafQueue.java | 2 +- ...estSchedulingRequestContainerAllocation.java | 8 +- ...hedulingRequestContainerAllocationAsync.java | 4 +- .../scheduler/capacity/TestUtils.java | 4 +- .../constraint/TestPlacementProcessor.java | 12 +- .../webapp/TestRMWebServicesCapacitySched.java | 2 +- .../pom.xml | 26 +- ...stTimelineReaderWebServicesHBaseStorage.java | 8 +- .../storage/TestHBaseTimelineStorageApps.java | 50 +- .../TestHBaseTimelineStorageEntities.java | 89 +- .../storage/TestHBaseTimelineStorageSchema.java | 61 +- .../flow/TestHBaseStorageFlowActivity.java | 24 +- .../storage/flow/TestHBaseStorageFlowRun.java | 42 +- .../flow/TestHBaseStorageFlowRunCompaction.java | 58 +- .../pom.xml | 219 +++++ .../reader/filter/TimelineFilterUtils.java | 313 +++++++ .../reader/filter/package-info.java | 28 + .../storage/HBaseTimelineReaderImpl.java | 96 ++ .../storage/HBaseTimelineWriterImpl.java | 611 +++++++++++++ .../storage/TimelineSchemaCreator.java | 368 ++++++++ .../storage/application/ApplicationTableRW.java | 137 +++ .../storage/application/package-info.java | 28 + .../storage/apptoflow/AppToFlowTableRW.java | 92 ++ .../storage/apptoflow/package-info.java | 28 + .../storage/common/BaseTableRW.java | 167 ++++ .../storage/common/ColumnRWHelper.java | 487 ++++++++++ .../common/HBaseTimelineStorageUtils.java | 121 +++ .../common/TimelineHBaseSchemaConstants.java | 71 ++ .../storage/common/TypedBufferedMutator.java | 73 ++ .../storage/common/package-info.java | 28 + .../storage/entity/EntityTableRW.java | 136 +++ .../storage/entity/package-info.java | 28 + .../storage/flow/FlowActivityTableRW.java | 91 ++ .../storage/flow/FlowRunTableRW.java | 102 +++ .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../reader/AbstractTimelineStorageReader.java | 159 ++++ .../storage/reader/ApplicationEntityReader.java | 523 +++++++++++ .../storage/reader/EntityTypeReader.java | 175 ++++ .../reader/FlowActivityEntityReader.java | 186 ++++ .../storage/reader/FlowRunEntityReader.java | 298 ++++++ .../storage/reader/GenericEntityReader.java | 655 ++++++++++++++ .../reader/SubApplicationEntityReader.java | 489 ++++++++++ .../storage/reader/TimelineEntityReader.java | 464 ++++++++++ .../reader/TimelineEntityReaderFactory.java | 105 +++ .../storage/reader/package-info.java | 28 + .../subapplication/SubApplicationTableRW.java | 137 +++ .../storage/subapplication/package-info.java | 28 + .../common/TestHBaseTimelineStorageUtils.java | 33 + .../pom.xml | 132 +++ .../storage/application/ApplicationColumn.java | 101 +++ .../application/ApplicationColumnFamily.java | 65 ++ .../application/ApplicationColumnPrefix.java | 150 ++++ .../storage/application/ApplicationRowKey.java | 251 ++++++ .../application/ApplicationRowKeyPrefix.java | 69 ++ .../storage/application/ApplicationTable.java | 60 ++ .../storage/application/package-info.java | 28 + .../storage/apptoflow/AppToFlowColumn.java | 95 ++ .../apptoflow/AppToFlowColumnFamily.java | 51 ++ .../apptoflow/AppToFlowColumnPrefix.java | 105 +++ .../storage/apptoflow/AppToFlowRowKey.java | 58 ++ .../storage/apptoflow/AppToFlowTable.java | 60 ++ .../storage/apptoflow/package-info.java | 28 + .../storage/common/AppIdKeyConverter.java | 97 ++ .../storage/common/BaseTable.java | 27 + .../timelineservice/storage/common/Column.java | 56 ++ .../storage/common/ColumnFamily.java | 34 + .../storage/common/ColumnHelper.java | 101 +++ .../storage/common/ColumnPrefix.java | 71 ++ .../storage/common/EventColumnName.java | 63 ++ .../common/EventColumnNameConverter.java | 99 ++ .../storage/common/GenericConverter.java | 48 + .../common/HBaseTimelineSchemaUtils.java | 156 ++++ .../storage/common/KeyConverter.java | 41 + .../storage/common/KeyConverterToString.java | 38 + .../storage/common/LongConverter.java | 94 ++ .../storage/common/LongKeyConverter.java | 68 ++ .../storage/common/NumericValueConverter.java | 39 + .../timelineservice/storage/common/Range.java | 62 ++ .../storage/common/RowKeyPrefix.java | 42 + .../storage/common/Separator.java | 575 ++++++++++++ .../storage/common/StringKeyConverter.java | 54 ++ .../storage/common/TimestampGenerator.java | 116 +++ .../storage/common/ValueConverter.java | 47 + .../storage/common/package-info.java | 28 + .../storage/entity/EntityColumn.java | 105 +++ .../storage/entity/EntityColumnFamily.java | 65 ++ .../storage/entity/EntityColumnPrefix.java | 162 ++++ .../storage/entity/EntityRowKey.java | 299 ++++++ .../storage/entity/EntityRowKeyPrefix.java | 77 ++ .../storage/entity/EntityTable.java | 61 ++ .../storage/entity/package-info.java | 28 + .../flow/AggregationCompactionDimension.java | 63 ++ .../storage/flow/AggregationOperation.java | 94 ++ .../timelineservice/storage/flow/Attribute.java | 39 + .../storage/flow/FlowActivityColumnFamily.java | 55 ++ .../storage/flow/FlowActivityColumnPrefix.java | 133 +++ .../storage/flow/FlowActivityRowKey.java | 247 +++++ .../storage/flow/FlowActivityRowKeyPrefix.java | 60 ++ .../storage/flow/FlowActivityTable.java | 45 + .../storage/flow/FlowRunColumn.java | 112 +++ .../storage/flow/FlowRunColumnFamily.java | 54 ++ .../storage/flow/FlowRunColumnPrefix.java | 129 +++ .../storage/flow/FlowRunRowKey.java | 233 +++++ .../storage/flow/FlowRunRowKeyPrefix.java | 54 ++ .../storage/flow/FlowRunTable.java | 77 ++ .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../subapplication/SubApplicationColumn.java | 99 ++ .../SubApplicationColumnFamily.java | 68 ++ .../SubApplicationColumnPrefix.java | 163 ++++ .../subapplication/SubApplicationRowKey.java | 290 ++++++ .../SubApplicationRowKeyPrefix.java | 69 ++ .../subapplication/SubApplicationTable.java | 64 ++ .../storage/subapplication/package-info.java | 28 + .../TestCustomApplicationIdConversion.java | 39 + .../storage/common/TestKeyConverters.java | 134 +++ .../storage/common/TestRowKeys.java | 276 ++++++ .../storage/common/TestRowKeysAsString.java | 144 +++ .../storage/common/TestSeparator.java | 215 +++++ .../pom.xml | 161 ++++ .../src/assembly/coprocessor.xml | 37 + .../common/HBaseTimelineServerUtils.java | 135 +++ .../storage/common/package-info.java | 28 + .../storage/flow/FlowRunCoprocessor.java | 277 ++++++ .../storage/flow/FlowScanner.java | 723 +++++++++++++++ .../storage/flow/FlowScannerOperation.java | 46 + .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../pom.xml | 193 +--- .../reader/filter/TimelineFilterUtils.java | 308 ------- .../reader/filter/package-info.java | 28 - .../storage/HBaseTimelineReaderImpl.java | 96 -- .../storage/HBaseTimelineWriterImpl.java | 593 ------------ .../storage/TimelineSchemaCreator.java | 367 -------- .../storage/application/ApplicationColumn.java | 108 --- .../application/ApplicationColumnFamily.java | 65 -- .../application/ApplicationColumnPrefix.java | 236 ----- .../storage/application/ApplicationRowKey.java | 251 ------ .../application/ApplicationRowKeyPrefix.java | 69 -- .../storage/application/ApplicationTable.java | 170 ---- .../storage/application/package-info.java | 28 - .../storage/apptoflow/AppToFlowColumn.java | 101 --- .../apptoflow/AppToFlowColumnFamily.java | 51 -- .../apptoflow/AppToFlowColumnPrefix.java | 206 ----- .../storage/apptoflow/AppToFlowRowKey.java | 58 -- .../storage/apptoflow/AppToFlowTable.java | 125 --- .../storage/apptoflow/package-info.java | 28 - .../storage/common/AppIdKeyConverter.java | 97 -- .../storage/common/BaseTable.java | 167 ---- .../common/BufferedMutatorDelegator.java | 73 -- .../timelineservice/storage/common/Column.java | 80 -- .../storage/common/ColumnFamily.java | 34 - .../storage/common/ColumnHelper.java | 414 --------- .../storage/common/ColumnPrefix.java | 145 --- .../storage/common/EventColumnName.java | 63 -- .../common/EventColumnNameConverter.java | 99 -- .../storage/common/GenericConverter.java | 48 - .../common/HBaseTimelineStorageUtils.java | 354 -------- .../storage/common/KeyConverter.java | 41 - .../storage/common/KeyConverterToString.java | 38 - .../storage/common/LongConverter.java | 94 -- .../storage/common/LongKeyConverter.java | 68 -- .../storage/common/NumericValueConverter.java | 39 - .../timelineservice/storage/common/Range.java | 62 -- .../storage/common/RowKeyPrefix.java | 42 - .../storage/common/Separator.java | 575 ------------ .../storage/common/StringKeyConverter.java | 54 -- .../common/TimelineHBaseSchemaConstants.java | 71 -- .../storage/common/TimestampGenerator.java | 116 --- .../storage/common/TypedBufferedMutator.java | 28 - .../storage/common/ValueConverter.java | 47 - .../storage/common/package-info.java | 28 - .../storage/entity/EntityColumn.java | 112 --- .../storage/entity/EntityColumnFamily.java | 65 -- .../storage/entity/EntityColumnPrefix.java | 249 ----- .../storage/entity/EntityRowKey.java | 299 ------ .../storage/entity/EntityRowKeyPrefix.java | 77 -- .../storage/entity/EntityTable.java | 170 ---- .../storage/entity/package-info.java | 28 - .../flow/AggregationCompactionDimension.java | 63 -- .../storage/flow/AggregationOperation.java | 94 -- .../timelineservice/storage/flow/Attribute.java | 39 - .../storage/flow/FlowActivityColumnFamily.java | 55 -- .../storage/flow/FlowActivityColumnPrefix.java | 221 ----- .../storage/flow/FlowActivityRowKey.java | 247 ----- .../storage/flow/FlowActivityRowKeyPrefix.java | 60 -- .../storage/flow/FlowActivityTable.java | 109 --- .../storage/flow/FlowRunColumn.java | 131 --- .../storage/flow/FlowRunColumnFamily.java | 54 -- .../storage/flow/FlowRunColumnPrefix.java | 217 ----- .../storage/flow/FlowRunCoprocessor.java | 277 ------ .../storage/flow/FlowRunRowKey.java | 233 ----- .../storage/flow/FlowRunRowKeyPrefix.java | 54 -- .../storage/flow/FlowRunTable.java | 151 ---- .../storage/flow/FlowScanner.java | 729 --------------- .../storage/flow/FlowScannerOperation.java | 46 - .../storage/flow/package-info.java | 29 - .../timelineservice/storage/package-info.java | 28 - .../reader/AbstractTimelineStorageReader.java | 158 ---- .../storage/reader/ApplicationEntityReader.java | 520 ----------- .../storage/reader/EntityTypeReader.java | 175 ---- .../reader/FlowActivityEntityReader.java | 185 ---- .../storage/reader/FlowRunEntityReader.java | 294 ------ .../storage/reader/GenericEntityReader.java | 651 -------------- .../reader/SubApplicationEntityReader.java | 488 ---------- .../storage/reader/TimelineEntityReader.java | 459 ---------- .../reader/TimelineEntityReaderFactory.java | 105 --- .../storage/reader/package-info.java | 28 - .../subapplication/SubApplicationColumn.java | 108 --- .../SubApplicationColumnFamily.java | 68 -- .../SubApplicationColumnPrefix.java | 250 ------ .../subapplication/SubApplicationRowKey.java | 290 ------ .../SubApplicationRowKeyPrefix.java | 69 -- .../subapplication/SubApplicationTable.java | 174 ---- .../storage/subapplication/package-info.java | 28 - .../TestCustomApplicationIdConversion.java | 39 - .../common/TestHBaseTimelineStorageUtils.java | 33 - .../storage/common/TestKeyConverters.java | 134 --- .../storage/common/TestRowKeys.java | 276 ------ .../storage/common/TestRowKeysAsString.java | 144 --- .../storage/common/TestSeparator.java | 215 ----- .../src/site/markdown/CapacityScheduler.md | 3 +- .../src/site/markdown/PlacementConstraints.md | 136 +++ .../site/markdown/PlacementConstraints.md.vm | 149 --- .../src/site/markdown/ResourceManagerRest.md | 4 +- .../src/site/markdown/UsingGpus.md | 230 +++++ .../app/components/service-component-table.js | 2 +- .../main/webapp/app/models/yarn-servicedef.js | 10 +- .../serializers/yarn-queue/capacity-queue.js | 29 +- .../components/service-component-table.hbs | 2 +- 458 files changed, 23213 insertions(+), 19012 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e634d49/hadoop-client-modules/hadoop-client-minicluster/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e634d49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java ---------------------------------------------------------------------- diff --cc hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java index ac4efea,2128de7..b19e54d --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java @@@ -108,72 -88,6 +108,67 @@@ public class ConfServlet extends HttpSe out.close(); } + private void processConfigTagRequest(HttpServletRequest request, + Writer out) throws IOException { + String cmd = request.getParameter(COMMAND); + Gson gson = new Gson(); + Configuration config = getOzoneConfig(); + + config.get("ozone.enabled"); + switch (cmd) { - case "getOzoneTags": - LOG.debug( - "Sending json for tags:" + gson.toJson(OzonePropertyTag.values())); - out.write(gson.toJson(OzonePropertyTag.values())); - break; + case "getPropertyByTag": + String tags = request.getParameter("tags"); + String tagGroup = request.getParameter("group"); + LOG.debug("Getting all properties for tags:" + tags + " group:" + + tagGroup); - List<PropertyTag> tagList = new ArrayList<>(); ++ List<String> tagList = new ArrayList<>(); + for (String tag : tags.split(",")) { - if (config.getPropertyTag(tag, tagGroup) != null) { - tagList.add(config.getPropertyTag(tag, tagGroup)); ++ if (config.isPropertyTag(tag)) { ++ tagList.add(tag); + } + } + + Properties properties = config.getAllPropertiesByTags(tagList); + if (propertyMap == null) { + loadDescriptions(); + } + + List<OzoneConfiguration.Property> filteredProperties = new ArrayList<>(); + + properties.stringPropertyNames().stream().forEach(key -> { + if (config.get(key) != null) { + propertyMap.get(key).setValue(config.get(key)); + filteredProperties.add(propertyMap.get(key)); + } + }); + out.write(gson.toJsonTree(filteredProperties).toString()); + break; + default: + throw new IllegalArgumentException(cmd + " is not a valid command."); + } + + } + + private void loadDescriptions() { + OzoneConfiguration config = (OzoneConfiguration) getOzoneConfig(); + List<OzoneConfiguration.Property> propList = null; + propertyMap = new HashMap<>(); + try { + propList = config.readPropertyFromXml(config.getResource("ozone-site" + + ".xml")); + propList.stream().map(p -> propertyMap.put(p.getName(), p)); + propList = config.readPropertyFromXml(config.getResource("ozone-default" + + ".xml")); + propList.stream().forEach(p -> { + if (!propertyMap.containsKey(p.getName().trim())) { + propertyMap.put(p.getName().trim(), p); + } + }); + } catch (Exception e) { + LOG.error("Error while reading description from xml files", e); + } + } + @VisibleForTesting static String parseAcceptHeader(HttpServletRequest request) { String format = request.getHeader(HttpHeaders.ACCEPT); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e634d49/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e634d49/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e634d49/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e634d49/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
