[
https://issues.apache.org/jira/browse/HDFS-12345?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16872429#comment-16872429
]
Hudson commented on HDFS-12345:
-------------------------------
FAILURE: Integrated in Jenkins build Hadoop-trunk-Commit #16821 (See
[https://builds.apache.org/job/Hadoop-trunk-Commit/16821/])
HDFS-12345 Add Dynamometer to hadoop-tools, a tool for scale testing the
(xkrogen: rev ab0b180ddb5d0775a2452d5eeb7badd252aadb91)
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/test/java/org/apache/hadoop/tools/dynamometer/blockgenerator/TestBlockGen.java
* (edit) hadoop-tools/pom.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/java/org/apache/hadoop/tools/dynamometer/blockgenerator/XMLParserMapper.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/AuditReplayThread.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/bash/generate-block-lists.sh
* (add) hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/CreateFileMapper.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AllowAllImpersonationProvider.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/conf/etc/hadoop/log4j.properties
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/blocks/dn1-a-0-r-00001
* (add)
hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-blockgen.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/WorkloadMapper.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/hadoop_3_1/fsimage_0000000000000061740
* (edit) dev-support/bin/dist-layout-stitching
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/package-info.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/VirtualInputSplit.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/java/org/apache/hadoop/tools/dynamometer/blockgenerator/GenerateDNBlockInfosReducer.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/AuditReplayMapper.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/java/org/apache/hadoop/tools/dynamometer/blockgenerator/GenerateBlockImagesDriver.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AMOptions.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/AuditReplayCommand.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/BlockPlacementPolicyAlwaysSatisfied.java
* (add) hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/ApplicationMaster.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/NoSplitTextInputFormat.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/test/resources/audit_trace_direct/audit0
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/test/resources/fsimage_0000000000000061740.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/test/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/TestWorkloadGenerator.java
* (edit) hadoop-dist/pom.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/upload-fsimage.sh
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/SimulatedDataNodes.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/start-dynamometer-cluster.sh
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/WorkloadDriver.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/conf/etc/hadoop/hdfs-site.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/bash/parse-start-timestamp.sh
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java
* (add) hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/java/org/apache/hadoop/tools/dynamometer/blockgenerator/BlockInfo.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/AuditLogDirectParser.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/blocks/dn2-a-0-r-00002
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/hadoop_3_1/VERSION
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynoInfraUtils.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/conf/etc/hadoop/core-site.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh
* (add)
hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-infra.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/test/java/org/apache/hadoop/tools/dynamometer/blockgenerator/TestXMLParser.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/VirtualRecordReader.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/java/org/apache/hadoop/tools/dynamometer/blockgenerator/package-info.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/hadoop_3_1/fsimage_0000000000000061740.md5
* (edit) hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/bash/start-workload.sh
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/AuditCommandParser.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/VirtualInputFormat.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoResource.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/package-info.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/create-slim-hadoop-tar.sh
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/main/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/package-info.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/bash/parse-metrics.sh
* (add) hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml
* (add) hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/pom.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoConstants.java
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/test/java/org/apache/hadoop/tools/dynamometer/workloadgenerator/audit/TestAuditLogDirectParser.java
* (add) hadoop-tools/hadoop-dynamometer/pom.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/blocks/dn0-a-0-r-00000
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/src/test/resources/audit_trace_hive/audit0
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/java/org/apache/hadoop/tools/dynamometer/blockgenerator/XMLParser.java
* (add)
hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-workload.xml
* (add)
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/resources/yarn-site.xml
> Scale testing HDFS NameNode with real metadata and workloads (Dynamometer)
> --------------------------------------------------------------------------
>
> Key: HDFS-12345
> URL: https://issues.apache.org/jira/browse/HDFS-12345
> Project: Hadoop HDFS
> Issue Type: New Feature
> Components: namenode, test
> Reporter: Zhe Zhang
> Assignee: Erik Krogen
> Priority: Major
> Fix For: 3.3.0
>
> Attachments: HDFS-12345.000.patch, HDFS-12345.001.patch,
> HDFS-12345.002.patch, HDFS-12345.003.patch, HDFS-12345.004.patch,
> HDFS-12345.005.patch, HDFS-12345.006.patch, HDFS-12345.007.patch,
> HDFS-12345.008.patch, HDFS-12345.009.patch
>
>
> Dynamometer has now been open sourced on our [GitHub
> page|https://github.com/linkedin/dynamometer]. Read more at our [recent blog
> post|https://engineering.linkedin.com/blog/2018/02/dynamometer--scale-testing-hdfs-on-minimal-hardware-with-maximum].
> To encourage getting the tool into the open for others to use as quickly as
> possible, we went through our standard open sourcing process of releasing on
> GitHub. However we are interested in the possibility of donating this to
> Apache as part of Hadoop itself and would appreciate feedback on whether or
> not this is something that would be supported by the community.
> Also of note, previous [discussions on the dev mail
> lists|http://mail-archives.apache.org/mod_mbox/hadoop-hdfs-dev/201707.mbox/%[email protected]%3e]
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
