[
https://issues.apache.org/jira/browse/MAPREDUCE-2060?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12909043#action_12909043
]
Dan Adkins commented on MAPREDUCE-2060:
---------------------------------------
Here's the view from HDFS of job submission leading up to the "Filesystem
closed" error, the original fault reported (I apologize for the length. I
don't know what might be relevant).
10/09/13 23:17:43 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=mkdirs
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_20100913
2304_0001 dst=null perm=dadkins:supergroup:rwxr-xr-x
10/09/13 23:17:43 WARN security.ShellBasedUnixGroupsMapping: got exception
trying to get groups for user dadkins
org.apache.hadoop.util.Shell$ExitCodeException: id: dadkins: No such user
at org.apache.hadoop.util.Shell.runCommand(Shell.java:256)
at org.apache.hadoop.util.Shell.run(Shell.java:183)
at
org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:376)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:462)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:445)
at
org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getUnixGroups(ShellBasedUnixGroupsMapping.java:60)
at
org.apache.hadoop.security.ShellBasedUnixGroupsMapping.getGroups(ShellBasedUnixGroupsMapping.java:47)
at org.apache.hadoop.security.Groups.getGroups(Groups.java:88)
at
org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:996)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.<init>(FSPermissionChecker.java:51)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkPermission(FSNamesystem.java:4060)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkOwner(FSNamesystem.java:4023)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.setPermission(FSNamesystem.java:679)
at
org.apache.hadoop.hdfs.server.namenode.NameNode.setPermission(NameNode.java:792)
at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:349)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1380)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1376)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1105)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1374)
10/09/13 23:17:43 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=setPermission
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_
201009132304_0001 dst=null perm=dadkins:supergroup:rwx------
10/09/13 23:17:44 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=create
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_20100913
2304_0001/job.jar dst=null perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:44 INFO hdfs.StateChange: BLOCK* NameSystem.allocateBlock:
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.jar.
blk_441
168967748892583_13013{blockUCState=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.1:21100|RBW],
ReplicaUnderConstruction[10.
2.19.10:21100|RBW], ReplicaUnderConstruction[10.2.19.105:21100|RBW]]}
10/09/13 23:17:44 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.105:21100 is added to
blk_441168967748892583_13013{blockUCSt
ate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.1:21100|RBW],
ReplicaUnderConstruction[10.2.19.10:21100|RBW], ReplicaUnde
rConstruction[10.2.19.105:21100|RBW]]} size 0
10/09/13 23:17:44 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.10:21100 is added to
blk_441168967748892583_13013{blockUCSta
te=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.1:21100|RBW],
ReplicaUnderConstruction[10.2.19.10:21100|RBW], ReplicaUnder
Construction[10.2.19.105:21100|RBW]]} size 0
10/09/13 23:17:44 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.1:21100 is added to
blk_441168967748892583_13013{blockUCStat
e=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.1:21100|RBW],
ReplicaUnderConstruction[10.2.19.10:21100|RBW], ReplicaUnderC
onstruction[10.2.19.105:21100|RBW]]} size 0
10/09/13 23:17:44 INFO hdfs.StateChange: DIR* NameSystem.completeFile: file
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.jar
is clo
sed by DFSClient_-540240165
10/09/13 23:17:44 INFO namenode.FSNamesystem: Increasing replication for file
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.jar.
New
replication is 10
10/09/13 23:17:44 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=setReplication
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_
201009132304_0001/job.jar dst=null perm=null
10/09/13 23:17:44 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=setPermission
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_
201009132304_0001/job.jar dst=null
perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:44 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=create
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_20100913
2304_0001/job.split dst=null perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:44 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=setPermission
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_
201009132304_0001/job.split dst=null
perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:44 INFO namenode.FSNamesystem: Increasing replication for file
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.split.
N
ew replication is 10
10/09/13 23:17:44 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=setReplication
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_
201009132304_0001/job.split dst=null perm=null
10/09/13 23:17:44 INFO hdfs.StateChange: BLOCK* NameSystem.allocateBlock:
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.split.
blk_-
1148662313095098724_13014{blockUCState=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction
[10.2.20.109:21100|RBW], ReplicaUnderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100
|RBW], ReplicaUnderConstruction[10.2.20.103:21100|RBW],
ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW], ReplicaUn
derConstruction[10.2.19.5:21100|RBW],
ReplicaUnderConstruction[10.2.20.2:21100|RBW]]}
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.2:21100 is added to
blk_-1148662313095098724_13014{blockUCSt
ate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaUnd
erConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[10.
2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW
], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.5:21100 is added to
blk_-1148662313095098724_13014{blockUCSt
ate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaUnd
erConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[10.
2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW
], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.110:21100 is added to
blk_-1148662313095098724_13014{blockUC
State=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaU
nderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[1
0.2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|R
BW], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.9:21100 is added to
blk_-1148662313095098724_13014{blockUCSt
ate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaUnd
erConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[10.
2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW
], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.103:21100 is added to
blk_-1148662313095098724_13014{blockUC
State=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaU
nderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[1
0.2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|R
BW], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.105:21100 is added to
blk_-1148662313095098724_13014{blockUC
State=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaU
nderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[1
0.2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|R
BW], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.3:21100 is added to
blk_-1148662313095098724_13014{blockUCSt
ate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaUnd
erConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[10.
2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW
], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.7:21100 is added to
blk_-1148662313095098724_13014{blockUCSt
ate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaUnd
erConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[10.
2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW
], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.109:21100 is added to
blk_-1148662313095098724_13014{blockUC
State=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaU
nderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[1
0.2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|R
BW], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.8:21100 is added to
blk_-1148662313095098724_13014{blockUCSt
ate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.8:21100|RBW],
ReplicaUnderConstruction[10.2.20.109:21100|RBW], ReplicaUnd
erConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.20.3:21100|RBW],
ReplicaUnderConstruction[10.2.20.105:21100|RBW], ReplicaUnderConstruction[10.
2.20.103:21100|RBW], ReplicaUnderConstruction[10.2.20.9:21100|RBW],
ReplicaUnderConstruction[10.2.19.110:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW
], ReplicaUnderConstruction[10.2.20.2:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: DIR* NameSystem.completeFile: file
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.split
is c
losed by DFSClient_-540240165
10/09/13 23:17:45 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=create
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_20100913
2304_0001/job.splitmetainfo dst=null
perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:45 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=setPermission
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_
201009132304_0001/job.splitmetainfo dst=null
perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.allocateBlock:
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.splitmetainf
o. blk_3663919217952367661_13015{blockUCState=UNDER_CONSTRUCTION,
primaryNodeIndex=-1, replicas=[ReplicaUnderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConst
ruction[10.2.19.7:21100|RBW], ReplicaUnderConstruction[10.2.19.9:21100|RBW]]}
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.9:21100 is added to
blk_3663919217952367661_13015{blockUCSta
te=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.19.7:21100|RBW], ReplicaUnderC
onstruction[10.2.19.9:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.7:21100 is added to
blk_3663919217952367661_13015{blockUCSta
te=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.19.7:21100|RBW], ReplicaUnderC
onstruction[10.2.19.9:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.7:21100 is added to
blk_3663919217952367661_13015{blockUCSta
te=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.20.7:21100|RBW],
ReplicaUnderConstruction[10.2.19.7:21100|RBW], ReplicaUnderC
onstruction[10.2.19.9:21100|RBW]]} size 0
10/09/13 23:17:45 INFO hdfs.StateChange: DIR* NameSystem.completeFile: file
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.splitmetai
nfo is closed by DFSClient_-540240165
10/09/13 23:17:45 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=create
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_20100913
2304_0001/job.xml dst=null perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:45 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=setPermission
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_
201009132304_0001/job.xml dst=null
perm=dadkins:supergroup:rw-r--r--
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* ask 10.2.19.105:21100 to
replicate blk_441168967748892583_13013 to datanode(s) 10.2.20.4:21100
10.2.20.5:21100 10
.2.19.107:21100 10.2.20.102:21100 10.2.20.6:21100 10.2.20.8:21100
10.2.20.101:21100
10/09/13 23:17:45 INFO hdfs.StateChange: BLOCK* NameSystem.allocateBlock:
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.xml.
blk_261
8516530812979395_13016{blockUCState=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.107:21100|RBW],
ReplicaUnderConstruction[
10.2.19.5:21100|RBW], ReplicaUnderConstruction[10.2.20.5:21100|RBW]]}
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.20.5:21100 is added to
blk_2618516530812979395_13016{blockUCSta
te=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.107:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW], ReplicaUnde
rConstruction[10.2.20.5:21100|RBW]]} size 0
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.5:21100 is added to
blk_2618516530812979395_13016{blockUCSta
te=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.107:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW], ReplicaUnde
rConstruction[10.2.20.5:21100|RBW]]} size 0
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addStoredBlock:
blockMap updated: 10.2.19.107:21100 is added to
blk_2618516530812979395_13016{blockUCS
tate=UNDER_CONSTRUCTION, primaryNodeIndex=-1,
replicas=[ReplicaUnderConstruction[10.2.19.107:21100|RBW],
ReplicaUnderConstruction[10.2.19.5:21100|RBW], ReplicaUn
derConstruction[10.2.20.5:21100|RBW]]} size 0
10/09/13 23:17:46 INFO hdfs.StateChange: DIR* NameSystem.completeFile: file
/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_0001/job.xml
is clo
sed by DFSClient_-540240165
10/09/13 23:17:46 INFO FSNamesystem.audit: ugi=dadkins ip=/10.2.12.100
cmd=open
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009132304_000
1/job.xml dst=null perm=null
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.19.1:21100 10.2.19.10:21100
10.2.19.105:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_-1148662313095098724_13014 to 10.2.19.8:21100 10.2.20.109:21100
10.2.20.7:21100
10.2.20.3:21100 10.2.20.105:21100 10.2.20.103:21100 10.2.20.9:21100
10.2.19.110:21100 10.2.19.5:21100 10.2.20.2:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_3663919217952367661_13015 to 10.2.20.7:21100 10.2.19.7:21100
10.2.19.9:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_2618516530812979395_13016 to 10.2.19.107:21100 10.2.19.5:21100
10.2.20.5:21100
10/09/13 23:17:46 INFO FSNamesystem.audit: ugi=dadkins ip=/10.128.130.145
cmd=delete
src=/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_20100913
2304_0001 dst=null perm=null
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addBlock: block
blk_441168967748892583_13013 on 10.2.20.4:21100 size 252058 does not belong to
any fil
e.
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.20.4:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addBlock: block
blk_441168967748892583_13013 on 10.2.20.5:21100 size 252058 does not belong to
any fil
e.
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.20.5:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addBlock: block
blk_441168967748892583_13013 on 10.2.19.107:21100 size 252058 does not belong
to any f
ile.
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.19.107:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addBlock: block
blk_441168967748892583_13013 on 10.2.20.102:21100 size 252058 does not belong
to any f
ile.
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.20.102:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addBlock: block
blk_441168967748892583_13013 on 10.2.20.6:21100 size 252058 does not belong to
any fil
e.
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.20.6:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addBlock: block
blk_441168967748892583_13013 on 10.2.20.8:21100 size 252058 does not belong to
any fil
e.
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.20.8:21100
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addBlock: block
blk_441168967748892583_13013 on 10.2.20.101:21100 size 252058 does not belong
to any f
ile.
10/09/13 23:17:46 INFO hdfs.StateChange: BLOCK* NameSystem.addToInvalidates:
blk_441168967748892583_13013 to 10.2.20.101:21100
> IOException: Filesystem closed on submitJob
> -------------------------------------------
>
> Key: MAPREDUCE-2060
> URL: https://issues.apache.org/jira/browse/MAPREDUCE-2060
> Project: Hadoop Map/Reduce
> Issue Type: Bug
> Components: jobtracker
> Affects Versions: 0.22.0
> Environment:
> https://svn.apache.org/repos/asf/hadoop/mapreduce/tr...@994941
> https://svn.apache.org/repos/asf/hadoop/hdfs/tr...@993542
> Reporter: Dan Adkins
>
> I get the following strange error on the jobtracker when attempting to submit
> a job:
> 10/09/09 20:31:35 INFO ipc.Server: IPC Server handler 7 on 31000, call
> submitJob(job_201009092028_0001,
> hdfs://hns4.sea1.qc:21000/tmp/hadoop-mr20/mapred/staging/dadkins/.staging/job_201009092028_0001,
> org.apache.hadoop.security.credenti...@20c87621) from 10.128.130.145:49253:
> error: java.io.IOException: Filesystem closed
> java.io.IOException: Filesystem closed
> at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:307)
> at org.apache.hadoop.hdfs.DFSClient.mkdirs(DFSClient.java:1212)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.mkdirs(DistributedFileSystem.java:494)
> at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:1491)
> at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:395)
> at org.apache.hadoop.mapred.JobTracker.submitJob(JobTracker.java:3078)
> at org.apache.hadoop.mapred.JobTracker.submitJob(JobTracker.java:3014)
> at org.apache.hadoop.mapred.JobTracker.submitJob(JobTracker.java:2996)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at
> org.apache.hadoop.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:349)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1380)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1376)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1105)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1374)
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
