bgaborg commented on issue #1003: HADOOP-16384: Avoid inconsistencies between DDB and S3 URL: https://github.com/apache/hadoop/pull/1003#issuecomment-510511005 the latest commit seems good to me, one outstanding issue is the usage of withKeyCondition instead of withCondition in dynamo filtering in ITestDynamoDBMetadataStoreScale.java. I've created https://issues.apache.org/jira/browse/HADOOP-16422 to deal with AbstractITCommitMRJob.testMRJob test failures, as those are permanent errors on each test run. there was new test errors while I was running with -Dscale: ``` [ERROR] Tests run: 7, Failures: 1, Errors: 2, Skipped: 0, Time elapsed: 63.155 s <<< FAILURE! - in org.apache.hadoop.fs.s3a.commit.terasort.ITestTerasortMagicCommitter [ERROR] test_110_teragen(org.apache.hadoop.fs.s3a.commit.terasort.ITestTerasortMagicCommitter) Time elapsed: 16.4 s <<< FAILURE! java.lang.AssertionError: Teragen(1000, s3a://gabota-versioned-bucket-ireland/terasort-ITestTerasortMagicCommitter/sortin) failed expected:<0> but was:<1> at org.junit.Assert.fail(Assert.java:88) at org.junit.Assert.failNotEquals(Assert.java:834) at org.junit.Assert.assertEquals(Assert.java:645) at org.apache.hadoop.fs.s3a.commit.terasort.AbstractCommitTerasortIT.executeStage(AbstractCommitTerasortIT.java:143) at org.apache.hadoop.fs.s3a.commit.terasort.AbstractCommitTerasortIT.test_110_teragen(AbstractCommitTerasortIT.java:167) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:748) [ERROR] test_120_terasort(org.apache.hadoop.fs.s3a.commit.terasort.ITestTerasortMagicCommitter) Time elapsed: 1.535 s <<< ERROR! java.io.FileNotFoundException: Expected file: not found s3a://gabota-versioned-bucket-ireland/terasort-ITestTerasortMagicCommitter/sortin/_SUCCESS in s3a://gabota-versioned-bucket-ireland/terasort-ITestTerasortMagicCommitter/sortin at org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists(ContractTestUtils.java:942) at org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists(ContractTestUtils.java:920) at org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile(ContractTestUtils.java:828) at org.apache.hadoop.fs.s3a.commit.AbstractCommitITest.loadSuccessFile(AbstractCommitITest.java:499) at org.apache.hadoop.fs.s3a.commit.terasort.AbstractCommitTerasortIT.test_120_terasort(AbstractCommitTerasortIT.java:177) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:748) Caused by: java.io.FileNotFoundException: No such file or directory: s3a://gabota-versioned-bucket-ireland/terasort-ITestTerasortMagicCommitter/sortin/_SUCCESS at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:2818) at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:2707) at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:2601) at org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists(ContractTestUtils.java:936) ... 19 more [ERROR] test_130_teravalidate(org.apache.hadoop.fs.s3a.commit.terasort.ITestTerasortMagicCommitter) Time elapsed: 1.432 s <<< ERROR! java.io.FileNotFoundException: Expected file: not found s3a://gabota-versioned-bucket-ireland/terasort-ITestTerasortMagicCommitter/sortout/_SUCCESS in s3a://gabota-versioned-bucket-ireland/terasort-ITestTerasortMagicCommitter/sortout at org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists(ContractTestUtils.java:942) at org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists(ContractTestUtils.java:920) at org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile(ContractTestUtils.java:828) at org.apache.hadoop.fs.s3a.commit.AbstractCommitITest.loadSuccessFile(AbstractCommitITest.java:499) at org.apache.hadoop.fs.s3a.commit.terasort.AbstractCommitTerasortIT.test_130_teravalidate(AbstractCommitTerasortIT.java:192) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:748) Caused by: java.io.FileNotFoundException: No such file or directory: s3a://gabota-versioned-bucket-ireland/terasort-ITestTerasortMagicCommitter/sortout/_SUCCESS at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:2818) at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:2707) at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:2601) at org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists(ContractTestUtils.java:936) ... 19 more ```
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
