yjhjstz commented on code in PR #1887:
URL: https://github.com/apache/cloudberry/pull/1887#discussion_r3779832456
##########
src/backend/utils/misc/fstream/gfile.c:
##########
@@ -253,7 +253,629 @@ static int bz_file_open(gfile_t *fd)
fd->u.bz->s.next_in = fd->u.bz->in;
fd->read = bz_file_read;
fd->close = bz_file_close;
-
+
+ return 0;
+}
+#endif
+
+#ifdef USE_LZO
+/*
+ * LZO-compressed file support (standard lzop container format).
+ *
+ * Uses in-process liblzo2 decompression, following the same pattern as
+ * .gz (zlib) / .bz2 (bzlib) / .zst (zstd). The file format is auto-detected
+ * in lzo_file_open() by probing the first 9 bytes:
+ *
+ * - standard lzop container format: 9-byte magic + full header + block
+ * checksums
+ * - Hadoop Raw LZO format: no magic/header/checksum, plain LZO block
+ * stream (handled as a bonus path; only the standard lzop format is
+ * advertised via the .lzo extension)
+ *
+ * lzop header layout:
+ * magic 9 bytes fixed magic \x89LZO\x00\x0d\x0a\x1a\x0a
+ * version 2 bytes version (big endian)
+ * lib_version 2 bytes library version
+ * ver_needed 2 bytes minimum version required to decompress
+ * method 1 byte compression algorithm
+ * level 1 byte compression level
+ * flags 4 bytes flags (control checksum types etc.)
+ * mode 4 bytes file mode
+ * mtime_low 4 bytes mtime low 32 bits
+ * mtime_high 4 bytes mtime high 32 bits
+ * [extra_ver] 1 byte if F_H_EXTRA_FIELD(0x40) set
+ * [filter] 4 bytes if F_H_FILTER(0x800) set
+ * name_len 1 byte original file name length
+ * name N bytes original file name
+ * [path_len] 4 bytes if F_H_PATH(0x2000) set
+ * [path] N bytes if F_H_PATH(0x2000) set
+ * checksum 4 bytes header checksum
+ *
+ * Data block layout (common to both formats):
+ * uncomp_len 4 bytes decompressed size, big endian (0 = EOF marker)
+ * comp_len 4 bytes compressed size, big endian
+ * [d_adler32] 4 bytes adler32 of decompressed data (F_ADLER32_D=0x01)
+ * [d_crc32] 4 bytes crc32 of decompressed data (F_CRC32_D=0x100)
+ * [c_adler32] 4 bytes adler32 of compressed data (F_ADLER32_C=0x02)
+ * [c_crc32] 4 bytes crc32 of compressed data (F_CRC32_C=0x200)
+ * data comp_len bytes LZO compressed data (or raw data if
+ * incompressible)
+ *
+ * The only difference between the two formats is whether the magic/header
+ * and checksum fields are present. The decompression path is driven by
+ * flags: Raw LZO format has flags=0 so all checksum logic is naturally
+ * skipped. The 9 probe bytes (which belong to the first data block in Raw
+ * LZO) are cached in peek_buf and consumed first by lzo_read_peek /
+ * lzo_read_uint32_peek to keep byte alignment.
+ */
+#include <lzo/lzo1x.h>
+
+/* LZO block buffer size: standard lzop default block size is 256KB */
+#define LZO_BUFFER_SIZE (1<<20)
+
+/*
+ * LZO decompression state structure (complete definition; gfile.h only
+ * declares the pointer). Same heap-allocated double buffer design as
+ * zlib_stuff / bzlib_stuff.
+ */
+struct lzo_stuff
+{
+ int out_size;
+ int out_pos;
+ int eof;
+ unsigned int flags; /* flags from lzop header; 0 for raw
LZO */
+ bool_t has_lzop_header; /* TRUE=standard lzop, FALSE=raw LZO */
+ int peek_size; /* valid bytes in peek_buf */
Review Comment:
how to support file >=2GB ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
