Another way to do that is to decorate your *ObjectMapper*, this I do to
compress with LZ4 and Gzip, here are some examples of Json text and Smile
binary formats,
basically your *ObjectMapper* will give an already compressed/descompressed
Input/Output stream, please note that for LZ4 I compress/decompress twice,
in case you wonder why the double Input/Output stream:
public static final Charset UTF8=Charset.forName("UTF-8");
public static final int LZ4_BLOCK_SIZE=32 * 1024;
public static final ObjectMapper LZ4_JSON_MAPPER=new ObjectMapper(new
JsonFactory()
.setInputDecorator(new InputDecorator()
{
@Override
public InputStream decorate(IOContext context,InputStream
inputStream)
{
return new LZ4BlockInputStream(new
LZ4BlockInputStream(inputStream));
}
@Override
public InputStream decorate(IOContext context,byte[] bytes,int
offset,int length)
{
return new LZ4BlockInputStream(new LZ4BlockInputStream(new
ByteArrayInputStream(bytes,offset,length)));
}
@Override
public Reader decorate(IOContext context,Reader reader)
{
return new InputStreamReader(new LZ4BlockInputStream(new
LZ4BlockInputStream(new ReaderInputStream(reader,UTF8))),UTF8);
}
})
.setOutputDecorator(new OutputDecorator()
{
@Override
public OutputStream decorate(IOContext context,OutputStream
outputStream)
{
return new LZ4BlockOutputStream(new
LZ4BlockOutputStream(outputStream,
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor()),
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor());
}
@Override
public Writer decorate(IOContext context,Writer writer)
{
return new OutputStreamWriter(new LZ4BlockOutputStream(new
LZ4BlockOutputStream(new WriterOutputStream(writer,UTF8),
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor()),
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor())
);
}
}))
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
.setSerializationInclusion(JsonInclude.Include.NON_NULL);
public static final ObjectMapper GZIP_JSON_MAPPER=new ObjectMapper(new
JsonFactory()
.setInputDecorator(new InputDecorator()
{
@Override
public InputStream decorate(IOContext context,InputStream
inputStream) throws IOException
{
return new GZIPInputStream(inputStream);
}
@Override
public InputStream decorate(IOContext context,byte[] bytes,int
offset,int length) throws IOException
{
return new GZIPInputStream(new
ByteArrayInputStream(bytes,offset,length));
}
@Override
public Reader decorate(IOContext context,Reader reader) throws
IOException
{
return new InputStreamReader(new GZIPInputStream(new
ReaderInputStream(reader,UTF8)),UTF8);
}
})
.setOutputDecorator(new OutputDecorator()
{
@Override
public OutputStream decorate(IOContext context,OutputStream
outputStream) throws IOException
{
return new GZIPOutputStream(outputStream);
}
@Override
public Writer decorate(IOContext context,Writer writer) throws
IOException
{
return new OutputStreamWriter(new GZIPOutputStream(new
WriterOutputStream(writer,UTF8)));
}
}))
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
.setSerializationInclusion(JsonInclude.Include.NON_NULL);
public static final ObjectMapper LZ4_SMILE_MAPPER=new ObjectMapper(new
SmileFactory().disable(ENCODE_BINARY_AS_7BIT)
.setInputDecorator(new InputDecorator()
{
@Override
public InputStream decorate(IOContext context,InputStream
inputStream)
{
return new LZ4BlockInputStream(new
LZ4BlockInputStream(inputStream));
}
@Override
public InputStream decorate(IOContext context,byte[] bytes,int
offset,int length)
{
return new LZ4BlockInputStream(new LZ4BlockInputStream(new
ByteArrayInputStream(bytes,offset,length)));
}
@Override
public Reader decorate(IOContext context,Reader reader)
{
return new InputStreamReader(new LZ4BlockInputStream(new
LZ4BlockInputStream(new ReaderInputStream(reader,UTF8))),UTF8);
}
})
.setOutputDecorator(new OutputDecorator()
{
@Override
public OutputStream decorate(IOContext context,OutputStream
outputStream)
{
return new LZ4BlockOutputStream(new
LZ4BlockOutputStream(outputStream,
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor()),
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor());
}
@Override
public Writer decorate(IOContext context,Writer writer)
{
return new OutputStreamWriter(new LZ4BlockOutputStream(new
LZ4BlockOutputStream(new WriterOutputStream(writer,UTF8),
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor()),
LZ4_BLOCK_SIZE,LZ4Factory.fastestInstance().fastCompressor())
);
}
}))
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
.setSerializationInclusion(JsonInclude.Include.NON_NULL);
public static final ObjectMapper GZIP_SMILE_MAPPER=new ObjectMapper(new
SmileFactory().disable(ENCODE_BINARY_AS_7BIT)
.setInputDecorator(new InputDecorator()
{
@Override
public InputStream decorate(IOContext context,InputStream
inputStream) throws IOException
{
return new GZIPInputStream(inputStream);
}
@Override
public InputStream decorate(IOContext context,byte[] bytes,int
offset,int length) throws IOException
{
return new GZIPInputStream(new
ByteArrayInputStream(bytes,offset,length));
}
@Override
public Reader decorate(IOContext context,Reader reader) throws
IOException
{
return new InputStreamReader(new GZIPInputStream(new
ReaderInputStream(reader,UTF8)),UTF8);
}
})
.setOutputDecorator(new OutputDecorator()
{
@Override
public OutputStream decorate(IOContext context,OutputStream
outputStream) throws IOException
{
return new GZIPOutputStream(outputStream);
}
@Override
public Writer decorate(IOContext context,Writer writer) throws
IOException
{
return new OutputStreamWriter(new GZIPOutputStream(new
WriterOutputStream(writer,UTF8)));
}
}))
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
.setSerializationInclusion(JsonInclude.Include.NON_NULL);
On Wednesday, June 19, 2019 at 2:00:27 AM UTC+1, Rob Sargent wrote:
>
> I'm using java 1.8.0_211 on ubuntu18.04
> in my build.gradle the deps for jackson are
> jooqdbCompile "com.fasterxml.jackson.core:jackson-databind:2.9.+"
> jooqdbCompile "com.fasterxml.jackson.core:jackson-core:2.9.+"
> jooqdbCompile "com.fasterxml.jackson.core:jackson-annotations:2.9.+"
> (I did just notice today that a 2.10 exists)
>
> From this
>
> public static void writeAsString(AbstractPayload payload, String outdirName) {
> File ofile = new File(outdirName, payload.getRunTag().toString()+
> ".json.gz");
> try (FileOutputStream fos = new FileOutputStream(ofile);
> GZIPOutputStream zos = new GZIPOutputStream(fos,true) ) {
> logger.error("Writing to file: {}", ofile.getCanonicalPath());
> payload.asJson(zos);
> }
> catch (Exception e) {
> logger.error ("Complete write failure of payload {}",
> payload.getRunTag(), e);
> e.printStackTrace();
> }
> }
>
> to the "asJson" call here
>
> public void asJson(OutputStream stream) {
> try {
> getObjectMapper().writeValue(stream, this);
> }
> catch (Exception e) {
> e.printStackTrace();
> logger.error("{}: could not serialize to stream", getRunTag());
> }
> }
>
> and the "getObjectMapper() call is
>
> private ObjectMapper getObjectMapper() {
> if (objectMapper == null) {
> objectMapper = new ObjectMapper();
> objectMapper.configure(SerializationFeature.INDENT_OUTPUT, false);
> objectMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET,
> false);
>
> objectMapper.setDefaultPropertyInclusion(JsonInclude.Value.construct(Include.NON_EMPTY,
> Include.ALWAYS));
> }
> return objectMapper;
> }
>
>
> I see plenty of flush()ing jackson-core so I don't think that's the problem,
>
> but putting the GZIPOutputStream in a try/with appeared to help. Prior to
> that,
>
> small "payloads" (typically under 2K bytes) where getting written as 10-byte
> files.
>
>
> gzip -d just complains and does nothing to the source json.gz file
>
>
> I can zcat the json.gz files successfully, but do get the gzip error messages:
>
>
> zcat test/crashpad/chaser/db182faa-cafd-47e9-9798-bc645486ffe1.json.gz |
> python -m json.tool >/tmp/c.json
>
> gzip: test/crashpad/chaser/db182faa-cafd-47e9-9798-bc645486ffe1.json.gz:
> unexpected end of file
> tail -f /tmp/c.json
> [
> null,
> "174616,9746805,9815786,9815795,9815801,9815813",
> null,
> null
> ]
> ]
> }
> }
> }
>
>
>
>
--
You received this message because you are subscribed to the Google Groups
"jackson-user" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/jackson-user/756ee4a2-d15f-46fe-a758-20f700082560%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
