[
https://issues.apache.org/jira/browse/HADOOP-16021?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
asin updated HADOOP-16021:
--------------------------
Description:
I want append the data in a file , when i use SequenceFile.appendIfExists , it
throw NullPointerException at at
org.apache.hadoop.io.SequenceFile$Writer.(SequenceFile.java:1119)
when i remove the 'appendIfExists', it works, but it will cover old file.
when i try use CompressionType.RECORD or CompressionType.BLOCK throw "not
support" exception
{code:java}
// my code
SequenceFile.Writer writer = null;
writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(path),
SequenceFile.Writer.keyClass(Text.class),
SequenceFile.Writer.valueClass(Text.class),
SequenceFile.Writer.appendIfExists(true) );
{code}
{code:java}
// all my code
public class Writer1 implements VoidFunction<Iterator<Tuple2<String, String>>> {
private static Configuration conf = new Configuration();
private int MAX_LINE = 3; // little num,for test
@Override
public void call(Iterator<Tuple2<String, String>> iterator) throws
Exception {
int partitionId = TaskContext.get().partitionId();
int count = 0;
SequenceFile.Writer writer = null;
while (iterator.hasNext()) {
Tuple2<String, String> tp = iterator.next();
Path path = new Path("D:/tmp-doc/logs/logs.txt");
if (writer == null)
writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(path),
SequenceFile.Writer.keyClass(Text.class),
SequenceFile.Writer.valueClass(Text.class),
SequenceFile.Writer.appendIfExists(true)
);
writer.append(new Text(tp._1), new Text(tp._2));
count++;
if (count > MAX_LINE) {
IOUtils.closeStream(writer);
count = 0;
writer = SequenceFile.createWriter(... // same as above
}
}
if (count > 0) {
IOUtils.closeStream(writer);
}
IOUtils.closeStream(writer);
}
}
{code}
{{more info
see:[https://stackoverflow.com/questions/53943978/hadoop-sequencefile-createwriter-appendifexists-codec-cause-nullpointerexception]}}
was:
I want append the data in a file , when i use SequenceFile.appendIfExists , it
throw NullPointerException at at
org.apache.hadoop.io.SequenceFile$Writer.(SequenceFile.java:1119)
when i remove the 'appendIfExists', it works, but it will cover old file.
when i try use CompressionType.RECORD or CompressionType.BLOCK throw "not
support" exception
{code:java}
// my code
SequenceFile.Writer writer = null;
writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(path),
SequenceFile.Writer.keyClass(Text.class),
SequenceFile.Writer.valueClass(Text.class),
SequenceFile.Writer.appendIfExists(true) );
{code}
{{more info
see:[https://stackoverflow.com/questions/53943978/hadoop-sequencefile-createwriter-appendifexists-codec-cause-nullpointerexception]}}
> SequenceFile.createWriter appendIfExists codec cause NullPointerException
> -------------------------------------------------------------------------
>
> Key: HADOOP-16021
> URL: https://issues.apache.org/jira/browse/HADOOP-16021
> Project: Hadoop Common
> Issue Type: Bug
> Components: common
> Affects Versions: 2.7.3
> Environment: windows10, hadoop2.7.3, jdk8
> Reporter: asin
> Priority: Major
> Labels: bug
> Attachments: 57.png
>
>
>
> I want append the data in a file , when i use SequenceFile.appendIfExists ,
> it throw NullPointerException at at
> org.apache.hadoop.io.SequenceFile$Writer.(SequenceFile.java:1119)
> when i remove the 'appendIfExists', it works, but it will cover old file.
>
> when i try use CompressionType.RECORD or CompressionType.BLOCK throw "not
> support" exception
>
> {code:java}
> // my code
> SequenceFile.Writer writer = null;
> writer = SequenceFile.createWriter(conf,
> SequenceFile.Writer.file(path),
> SequenceFile.Writer.keyClass(Text.class),
> SequenceFile.Writer.valueClass(Text.class),
> SequenceFile.Writer.appendIfExists(true) );
> {code}
>
> {code:java}
> // all my code
> public class Writer1 implements VoidFunction<Iterator<Tuple2<String,
> String>>> {
> private static Configuration conf = new Configuration();
> private int MAX_LINE = 3; // little num,for test
> @Override
> public void call(Iterator<Tuple2<String, String>> iterator) throws
> Exception {
> int partitionId = TaskContext.get().partitionId();
> int count = 0;
> SequenceFile.Writer writer = null;
> while (iterator.hasNext()) {
> Tuple2<String, String> tp = iterator.next();
> Path path = new Path("D:/tmp-doc/logs/logs.txt");
> if (writer == null)
> writer = SequenceFile.createWriter(conf,
> SequenceFile.Writer.file(path),
> SequenceFile.Writer.keyClass(Text.class),
> SequenceFile.Writer.valueClass(Text.class),
> SequenceFile.Writer.appendIfExists(true)
> );
> writer.append(new Text(tp._1), new Text(tp._2));
> count++;
> if (count > MAX_LINE) {
> IOUtils.closeStream(writer);
> count = 0;
> writer = SequenceFile.createWriter(... // same as above
> }
> }
> if (count > 0) {
> IOUtils.closeStream(writer);
> }
> IOUtils.closeStream(writer);
> }
> }
> {code}
>
> {{more info
> see:[https://stackoverflow.com/questions/53943978/hadoop-sequencefile-createwriter-appendifexists-codec-cause-nullpointerexception]}}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
