@Harsh J
I was following an explanation for Gridmix and it runs it using the java
command but here I have ran it as hadoop jar with the jar file that is located
in the contrib/gridmix directory that is in 1.0.4
as you can see before I run Gridmix the bench folder in hdfs is non existent
but I keep getting the FileAlreadyExistsException when I run it
SeanBarry-Macbook-Pro:gridmix seanbarry$ hadoop fs -ls /user/seanbarry
Found 5 items
drwxrwxrwx - seanbarry staff 0 2013-01-03 15:16 /user/seanbarry/Rumen
drwxrwxrwx - seanbarry staff 0 2013-01-03 15:37
/user/seanbarry/gutenberg
drwxrwxrwx - seanbarry staff 0 2013-01-03 12:05
/user/seanbarry/gutenberg-output
drwxrwxrwx - seanbarry staff 0 2013-01-03 12:06
/user/seanbarry/gutenberg-output-1
drwxrwxrwx - seanbarry staff 0 2013-01-03 12:07
/user/seanbarry/gutenberg-output-2
SeanBarry-Macbook-Pro:gridmix seanbarry$ hadoop jar hadoop-gridmix-1.0.4.jar
org.apache.hadoop.mapred.gridmix.Gridmix -generate 10m
hdfs://hostname:54310/user/seanbarry/bench
file:///Users/seanbarry/sfbarry/job-trace.json
13/01/07 10:12:11 INFO gridmix.SubmitterUserResolver: Current user resolver is
SubmitterUserResolver
13/01/07 10:12:11 WARN gridmix.Gridmix: Resource null ignored
13/01/07 10:12:11 INFO gridmix.Gridmix: Submission policy is STRESS
13/01/07 10:12:11 INFO gridmix.Gridmix: Generating 10.0m of test data...
13/01/07 10:12:11 INFO mapred.JobClient: Cleaning up the staging area
hdfs://hostname:54310/app/hadoop/tmp/mapred/staging/seanbarry/.staging/job_201301070957_0003
13/01/07 10:12:11 ERROR security.UserGroupInformation:
PriviledgedActionException as:seanbarry
cause:org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
hdfs://hostname:54310/user/seanbarry/bench already exists
13/01/07 10:12:11 ERROR security.UserGroupInformation:
PriviledgedActionException as:seanbarry
cause:org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
hdfs://hostname:54310/user/seanbarry/bench already exists
13/01/07 10:12:11 WARN gridmix.JobSubmitter: Failed to submit GRIDMIX_GENDATA
as seanbarry
org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
hdfs://hostname:54310/user/seanbarry/bench already exists
at
org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.checkOutputSpecs(FileOutputFormat.java:137)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:887)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:850)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
at
org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:850)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:500)
at
org.apache.hadoop.mapred.gridmix.GenerateData$1.run(GenerateData.java:116)
at
org.apache.hadoop.mapred.gridmix.GenerateData$1.run(GenerateData.java:101)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
at
org.apache.hadoop.mapred.gridmix.GenerateData.call(GenerateData.java:101)
at
org.apache.hadoop.mapred.gridmix.GenerateData.call(GenerateData.java:57)
at
org.apache.hadoop.mapred.gridmix.JobSubmitter$SubmitTask.run(JobSubmitter.java:106)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
13/01/07 10:12:11 INFO gridmix.JobMonitor: Job submission failed notify if
anyone is waiting org.apache.hadoop.mapreduce.Job@1b980630
13/01/07 10:12:21 INFO mapred.JobClient: Cleaning up the staging area
hdfs://hostname:54310/app/hadoop/tmp/mapred/staging/seanbarry/.staging/job_201301070957_0004
13/01/07 10:12:21 ERROR gridmix.Gridmix: Startup failed
java.lang.IllegalArgumentException: Wrong FS:
hdfs://hostname:54310/app/hadoop/tmp/mapred/staging/seanbarry/.staging/job_201301070957_0003/job.jar,
expected: file:///
at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:381)
at
org.apache.hadoop.fs.RawLocalFileSystem.pathToFile(RawLocalFileSystem.java:55)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:393)
at
org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:251)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:213)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1195)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1171)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1143)
at
org.apache.hadoop.mapred.JobClient.copyAndConfigureFiles(JobClient.java:779)
at
org.apache.hadoop.mapred.JobClient.copyAndConfigureFiles(JobClient.java:655)
at org.apache.hadoop.mapred.JobClient.access$300(JobClient.java:174)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:865)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:850)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
at
org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:850)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:500)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:530)
at
org.apache.hadoop.mapred.gridmix.Gridmix.writeInputData(Gridmix.java:118)
at org.apache.hadoop.mapred.gridmix.Gridmix.start(Gridmix.java:283)
at org.apache.hadoop.mapred.gridmix.Gridmix.runJob(Gridmix.java:263)
at org.apache.hadoop.mapred.gridmix.Gridmix.access$000(Gridmix.java:55)
at org.apache.hadoop.mapred.gridmix.Gridmix$1.run(Gridmix.java:217)
at org.apache.hadoop.mapred.gridmix.Gridmix$1.run(Gridmix.java:215)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
at org.apache.hadoop.mapred.gridmix.Gridmix.run(Gridmix.java:215)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)
at org.apache.hadoop.mapred.gridmix.Gridmix.main(Gridmix.java:390)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:156)
13/01/07 10:12:21 INFO gridmix.Gridmix: Exiting…
SeanBarry-Macbook-Pro:gridmix seanbarry$ hadoop fs -ls /user/seanbarry
Found 6 items
drwxrwxrwx - seanbarry staff 0 2013-01-03 15:16 /user/seanbarry/Rumen
drwxrwxrwx - seanbarry staff 0 2013-01-07 10:12 /user/seanbarry/bench
drwxrwxrwx - seanbarry staff 0 2013-01-03 15:37
/user/seanbarry/gutenberg
drwxrwxrwx - seanbarry staff 0 2013-01-03 12:05
/user/seanbarry/gutenberg-output
drwxrwxrwx - seanbarry staff 0 2013-01-03 12:06
/user/seanbarry/gutenberg-output-1
drwxrwxrwx - seanbarry staff 0 2013-01-03 12:07
/user/seanbarry/gutenberg-output-2
On Jan 4, 2013, at 11:45 PM, Harsh J wrote:
> Hi Sean,
>
> Two questions: Why are you running this in local mode? Placing a cluster's
> config directory on your java -cp will make it go distributed. And, does that
> reported output directory really exist? If so, you may want to delete it
> before you run GridMix.
>
>
> On Sat, Jan 5, 2013 at 3:55 AM, Sean Barry <[email protected]> wrote:
> Hi,
>
> I am trying to use grid mix but I keep getting the error that is shown below.
> Does anyone have some suggestions.
>
> Thanks in advance.
>
> Sean Barry
>
> hostname:gridmix seanbarry$ pwd
> /usr/local/hadoop-1.0.4/contrib/gridmix
> hostname:gridmix seanbarry$ java -cp
> /usr/local/hadoop-1.0.4/contrib/gridmix/hadoop-gridmix-1.0.4.jar:/usr/local/hadoop-1.0.4/*:/usr/local/hadoop-1.0.4/lib/*
> org.apache.hadoop.mapred.gridmix.Gridmix -generate 10m
> hdfs://hostname:54310/user/seanbarry/bench
> file:///Users/seanbarry/sfbarry/job-trace.json
> 2013-01-04 14:19:15.759 java[2416:1903] Unable to load realm info from
> SCDynamicStore
> 13/01/04 14:19:15 INFO gridmix.SubmitterUserResolver: Current user resolver
> is SubmitterUserResolver
> 13/01/04 14:19:15 WARN gridmix.Gridmix: Resource null ignored
> 13/01/04 14:19:16 INFO gridmix.Gridmix: Submission policy is STRESS
> 13/01/04 14:19:16 INFO gridmix.Gridmix: Generating 10.0m of test data...
> 13/01/04 14:19:16 WARN util.NativeCodeLoader: Unable to load native-hadoop
> library for your platform... using builtin-java classes where applicable
> 13/01/04 14:19:16 INFO mapred.JobClient: Cleaning up the staging area
> file:/tmp/hadoop-seanbarry/mapred/staging/seanbarry-957366090/.staging/job_local_0001
> 13/01/04 14:19:16 ERROR security.UserGroupInformation:
> PriviledgedActionException as:seanbarry
> cause:org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
> hdfs://hostname:54310/user/seanbarry/bench already exists
> 13/01/04 14:19:16 ERROR security.UserGroupInformation:
> PriviledgedActionException as:seanbarry
> cause:org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
> hdfs://hostname:54310/user/seanbarry/bench already exists
> 13/01/04 14:19:16 WARN gridmix.JobSubmitter: Failed to submit GRIDMIX_GENDATA
> as seanbarry
> org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
> hdfs://hostname:54310/user/seanbarry/bench already exists
> at
> org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.checkOutputSpecs(FileOutputFormat.java:137)
> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:887)
> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:850)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
> at
> org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:850)
> at org.apache.hadoop.mapreduce.Job.submit(Job.java:500)
> at
> org.apache.hadoop.mapred.gridmix.GenerateData$1.run(GenerateData.java:116)
> at
> org.apache.hadoop.mapred.gridmix.GenerateData$1.run(GenerateData.java:101)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
> at
> org.apache.hadoop.mapred.gridmix.GenerateData.call(GenerateData.java:101)
> at
> org.apache.hadoop.mapred.gridmix.GenerateData.call(GenerateData.java:57)
> at
> org.apache.hadoop.mapred.gridmix.JobSubmitter$SubmitTask.run(JobSubmitter.java:106)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:680)
> 13/01/04 14:19:16 INFO gridmix.JobMonitor: Job submission failed notify if
> anyone is waiting org.apache.hadoop.mapreduce.Job@79884a40
> 13/01/04 14:19:26 INFO mapred.JobClient: Cleaning up the staging area
> file:/tmp/hadoop-seanbarry/mapred/staging/seanbarry15872696/.staging/job_local_0002
> 13/01/04 14:19:26 ERROR security.UserGroupInformation:
> PriviledgedActionException as:seanbarry
> cause:org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
> hdfs://hostname:54310/user/seanbarry/bench already exists
> 13/01/04 14:19:26 ERROR gridmix.Gridmix: Startup failed
> org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory
> hdfs://hostname:54310/user/seanbarry/bench already exists
> at
> org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.checkOutputSpecs(FileOutputFormat.java:137)
> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:887)
> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:850)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
> at
> org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:850)
> at org.apache.hadoop.mapreduce.Job.submit(Job.java:500)
> at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:530)
> at
> org.apache.hadoop.mapred.gridmix.Gridmix.writeInputData(Gridmix.java:118)
> at org.apache.hadoop.mapred.gridmix.Gridmix.start(Gridmix.java:283)
> at org.apache.hadoop.mapred.gridmix.Gridmix.runJob(Gridmix.java:263)
> at
> org.apache.hadoop.mapred.gridmix.Gridmix.access$000(Gridmix.java:55)
> at org.apache.hadoop.mapred.gridmix.Gridmix$1.run(Gridmix.java:217)
> at org.apache.hadoop.mapred.gridmix.Gridmix$1.run(Gridmix.java:215)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)
> at org.apache.hadoop.mapred.gridmix.Gridmix.run(Gridmix.java:215)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)
> at org.apache.hadoop.mapred.gridmix.Gridmix.main(Gridmix.java:390)
> 13/01/04 14:19:26 INFO gridmix.Gridmix: Exiting...
> hostname:gridmix seanbarry$
>
>
>
>
> --
> Harsh J
