[
https://issues.apache.org/jira/browse/ORC-420?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16706502#comment-16706502
]
ASF GitHub Bot commented on ORC-420:
------------------------------------
majetideepak closed pull request #326: ORC-420: [C++] Implement string
dictionary encoding for C++ writer
URL: https://github.com/apache/orc/pull/326
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/c++/include/orc/Writer.hh b/c++/include/orc/Writer.hh
index 1284c65d87..01e0128638 100644
--- a/c++/include/orc/Writer.hh
+++ b/c++/include/orc/Writer.hh
@@ -185,6 +185,12 @@ namespace orc {
* @return if not set, the default is false
*/
bool getEnableIndex() const;
+
+ /**
+ * Get whether or not to enable dictionary encoding
+ * @return if not set, the default is false
+ */
+ bool getEnableDictionary() const;
};
class Writer {
diff --git a/c++/src/ColumnWriter.cc b/c++/src/ColumnWriter.cc
index 9b128e3940..c6c30f3833 100644
--- a/c++/src/ColumnWriter.cc
+++ b/c++/src/ColumnWriter.cc
@@ -204,6 +204,10 @@ namespace orc {
}
}
+ void ColumnWriter::writeDictionary() {
+ // PASS
+ }
+
class StructColumnWriter : public ColumnWriter {
public:
StructColumnWriter(
@@ -237,6 +241,8 @@ namespace orc {
virtual void writeIndex(
std::vector<proto::Stream> &streams) const override;
+ virtual void writeDictionary() override;
+
virtual void reset() override;
private:
@@ -382,6 +388,12 @@ namespace orc {
}
}
+ void StructColumnWriter::writeDictionary() {
+ for (uint32_t i = 0; i < children.size(); ++i) {
+ children[i]->writeDictionary();
+ }
+ }
+
class IntegerColumnWriter : public ColumnWriter {
public:
IntegerColumnWriter(
@@ -831,6 +843,137 @@ namespace orc {
dataStream->recordPosition(rowIndexPosition.get());
}
+ /**
+ * Implementation of increasing sorted string dictionary
+ */
+ class StringDictionary {
+ public:
+ struct DictEntry {
+ DictEntry(const char * str, size_t len):data(str),length(len) {}
+ const char * data;
+ size_t length;
+ };
+
+ StringDictionary():totalLength(0) {}
+
+ // insert a new string into dictionary, return its insertion order
+ size_t insert(const char * data, size_t len);
+
+ // write dictionary data & length to output buffer
+ void flush(AppendOnlyBufferedStream * dataStream,
+ RleEncoder * lengthEncoder) const;
+
+ // reorder input index buffer from insertion order to dictionary order
+ void reorder(std::vector<int64_t>& idxBuffer) const;
+
+ // get dict entries in insertion order
+ void getEntriesInInsertionOrder(std::vector<const DictEntry *>&) const;
+
+ // return count of entries
+ size_t size() const;
+
+ // return total length of strings in the dictioanry
+ uint64_t length() const;
+
+ void clear();
+
+ private:
+ struct LessThan {
+ bool operator()(const DictEntry& left, const DictEntry& right) const {
+ int ret = memcmp(left.data, right.data, std::min(left.length,
right.length));
+ if (ret != 0) {
+ return ret < 0;
+ }
+ return left.length < right.length;
+ }
+ };
+
+ std::map<DictEntry, size_t, LessThan> dict;
+ std::vector<std::vector<char>> data;
+ uint64_t totalLength;
+
+ // use friend class here to avoid being bothered by const function calls
+ friend class StringColumnWriter;
+ friend class CharColumnWriter;
+ friend class VarCharColumnWriter;
+ // store indexes of insertion order in the dictionary for not-null rows
+ std::vector<int64_t> idxInDictBuffer;
+ };
+
+ // insert a new string into dictionary, return its insertion order
+ size_t StringDictionary::insert(const char * str, size_t len) {
+ auto ret = dict.insert({DictEntry(str, len), dict.size()});
+ if (ret.second) {
+ // make a copy to internal storage
+ data.push_back(std::vector<char>(len));
+ memcpy(data.back().data(), str, len);
+ // update dictionary entry to link pointer to internal storage
+ DictEntry * entry = const_cast<DictEntry *>(&(ret.first->first));
+ entry->data = data.back().data();
+ totalLength += len;
+ }
+ return ret.first->second;
+ }
+
+ // write dictionary data & length to output buffer
+ void StringDictionary::flush(AppendOnlyBufferedStream * dataStream,
+ RleEncoder * lengthEncoder) const {
+ for (auto it = dict.cbegin(); it != dict.cend(); ++it) {
+ dataStream->write(it->first.data, it->first.length);
+ lengthEncoder->write(static_cast<int64_t>(it->first.length));
+ }
+ }
+
+ /**
+ * Reorder input index buffer from insertion order to dictionary order
+ *
+ * We require this function because string values are buffered by indexes
+ * in their insertion order. Until the entire dictionary is complete can
+ * we get their sorted indexes in the dictionary in that ORC specification
+ * demands dictionary should be ordered. Therefore this function transforms
+ * the indexes from insertion order to dictionary value order for final
+ * output.
+ */
+ void StringDictionary::reorder(std::vector<int64_t>& idxBuffer) const {
+ // iterate the dictionary to get mapping from insertion order to value
order
+ std::vector<size_t> mapping(dict.size());
+ size_t dictIdx = 0;
+ for (auto it = dict.cbegin(); it != dict.cend(); ++it) {
+ mapping[it->second] = dictIdx++;
+ }
+
+ // do the transformation
+ for (size_t i = 0; i != idxBuffer.size(); ++i) {
+ idxBuffer[i] = static_cast<int64_t>(
+ mapping[static_cast<size_t>(idxBuffer[i])]);
+ }
+ }
+
+ // get dict entries in insertion order
+ void StringDictionary::getEntriesInInsertionOrder(
+ std::vector<const DictEntry *>& entries) const {
+ entries.resize(dict.size());
+ for (auto it = dict.cbegin(); it != dict.cend(); ++it) {
+ entries[it->second] = &(it->first);
+ }
+ }
+
+ // return count of entries
+ size_t StringDictionary::size() const {
+ return dict.size();
+ }
+
+ // return total length of strings in the dictioanry
+ uint64_t StringDictionary::length() const {
+ return totalLength;
+ }
+
+ void StringDictionary::clear() {
+ totalLength = 0;
+ data.clear();
+ dict.clear();
+ }
+
class StringColumnWriter : public ColumnWriter {
public:
StringColumnWriter(const Type& type,
@@ -850,10 +993,50 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void createRowIndexEntry() override;
+
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
+ private:
+ /**
+ * dictionary related functions
+ */
+ bool checkDictionaryKeyRatio();
+ void createDirectStreams();
+ void createDictStreams();
+ void deleteDictStreams();
+ void fallbackToDirectEncoding();
+
protected:
- std::unique_ptr<RleEncoder> lengthEncoder;
- std::unique_ptr<AppendOnlyBufferedStream> dataStream;
RleVersion rleVersion;
+ bool useCompression;
+ const StreamsFactory& streamsFactory;
+ bool alignedBitPacking;
+
+ // direct encoding streams
+ std::unique_ptr<RleEncoder> directLengthEncoder;
+ std::unique_ptr<AppendOnlyBufferedStream> directDataStream;
+
+ // dictionary encoding streams
+ std::unique_ptr<RleEncoder> dictDataEncoder;
+ std::unique_ptr<RleEncoder> dictLengthEncoder;
+ std::unique_ptr<AppendOnlyBufferedStream> dictStream;
+
+ /**
+ * dictionary related variables
+ */
+ StringDictionary dictionary;
+ // whether or not dictionary checking is done
+ bool doneDictionaryCheck;
+ // whether or not it should be used
+ bool useDictionary;
+ // keys in the dictionary should not exceed this ratio
+ double dictSizeThreshold;
+
+ // record start row of each row group; null rows are skipped
+ mutable std::vector<size_t> startOfRowGroups;
};
StringColumnWriter::StringColumnWriter(
@@ -861,16 +1044,24 @@ namespace orc {
const StreamsFactory& factory,
const WriterOptions& options) :
ColumnWriter(type, factory, options),
- rleVersion(options.getRleVersion()) {
- std::unique_ptr<BufferedOutputStream> lengthStream =
- factory.createStream(proto::Stream_Kind_LENGTH);
- lengthEncoder = createRleEncoder(std::move(lengthStream),
- false,
- rleVersion,
- memPool,
- options.getAlignedBitpacking());
- dataStream.reset(new AppendOnlyBufferedStream(
- factory.createStream(proto::Stream_Kind_DATA)));
+ rleVersion(options.getRleVersion()),
+ useCompression(options.getCompression() !=
CompressionKind_NONE),
+ streamsFactory(factory),
+
alignedBitPacking(options.getAlignedBitpacking()),
+ doneDictionaryCheck(false),
+ useDictionary(options.getEnableDictionary()),
+
dictSizeThreshold(options.getDictionaryKeySizeThreshold()){
+ if (type.getKind() == TypeKind::BINARY) {
+ useDictionary = false;
+ doneDictionaryCheck = true;
+ }
+
+ if (useDictionary) {
+ createDictStreams();
+ } else {
+ doneDictionaryCheck = true;
+ createDirectStreams();
+ }
if (enableIndex) {
recordPosition();
@@ -892,7 +1083,9 @@ namespace orc {
const char* notNull = stringBatch->hasNulls ?
stringBatch->notNull.data() + offset : nullptr;
- lengthEncoder->add(length, numValues, notNull);
+ if (!useDictionary){
+ directLengthEncoder->add(length, numValues, notNull);
+ }
StringColumnStatisticsImpl* strStats =
dynamic_cast<StringColumnStatisticsImpl*>(colIndexStatistics.get());
@@ -902,7 +1095,12 @@ namespace orc {
bool hasNull = false;
for (uint64_t i = 0; i < numValues; ++i) {
if (!notNull || notNull[i]) {
- dataStream->write(data[i], static_cast<size_t>(length[i]));
+ if (useDictionary) {
+ size_t index = dictionary.insert(data[i],
static_cast<size_t>(length[i]));
+ dictionary.idxInDictBuffer.push_back(static_cast<int64_t>(index));
+ } else {
+ directDataStream->write(data[i], static_cast<size_t>(length[i]));
+ }
strStats->update(data[i], static_cast<size_t>(length[i]));
strStats->increase(1);
} else if (!hasNull) {
@@ -917,38 +1115,225 @@ namespace orc {
void StringColumnWriter::flush(std::vector<proto::Stream>& streams) {
ColumnWriter::flush(streams);
- proto::Stream length;
- length.set_kind(proto::Stream_Kind_LENGTH);
- length.set_column(static_cast<uint32_t>(columnId));
- length.set_length(lengthEncoder->flush());
- streams.push_back(length);
+ if (useDictionary) {
+ proto::Stream data;
+ data.set_kind(proto::Stream_Kind_DATA);
+ data.set_column(static_cast<uint32_t>(columnId));
+ data.set_length(dictDataEncoder->flush());
+ streams.push_back(data);
+
+ proto::Stream dict;
+ dict.set_kind(proto::Stream_Kind_DICTIONARY_DATA);
+ dict.set_column(static_cast<uint32_t>(columnId));
+ dict.set_length(dictStream->flush());
+ streams.push_back(dict);
+
+ proto::Stream length;
+ length.set_kind(proto::Stream_Kind_LENGTH);
+ length.set_column(static_cast<uint32_t>(columnId));
+ length.set_length(dictLengthEncoder->flush());
+ streams.push_back(length);
+ } else {
+ proto::Stream length;
+ length.set_kind(proto::Stream_Kind_LENGTH);
+ length.set_column(static_cast<uint32_t>(columnId));
+ length.set_length(directLengthEncoder->flush());
+ streams.push_back(length);
- proto::Stream data;
- data.set_kind(proto::Stream_Kind_DATA);
- data.set_column(static_cast<uint32_t>(columnId));
- data.set_length(dataStream->flush());
- streams.push_back(data);
+ proto::Stream data;
+ data.set_kind(proto::Stream_Kind_DATA);
+ data.set_column(static_cast<uint32_t>(columnId));
+ data.set_length(directDataStream->flush());
+ streams.push_back(data);
+ }
}
uint64_t StringColumnWriter::getEstimatedSize() const {
uint64_t size = ColumnWriter::getEstimatedSize();
- size += lengthEncoder->getBufferSize();
- size += dataStream->getSize();
+ if (!useDictionary) {
+ size += directLengthEncoder->getBufferSize();
+ size += directDataStream->getSize();
+ } else {
+ size += dictionary.length();
+ size += dictionary.size() * sizeof(int32_t);
+ size += dictionary.idxInDictBuffer.size() * sizeof(int32_t);
+ if (useCompression) {
+ size /= 3; // estimated ratio is 3:1
+ }
+ }
return size;
}
void StringColumnWriter::getColumnEncoding(
std::vector<proto::ColumnEncoding>& encodings) const {
proto::ColumnEncoding encoding;
- encoding.set_kind(RleVersionMapper(rleVersion));
- encoding.set_dictionarysize(0);
+ if (!useDictionary) {
+ encoding.set_kind(rleVersion == RleVersion_1 ?
+ proto::ColumnEncoding_Kind_DIRECT :
+ proto::ColumnEncoding_Kind_DIRECT_V2);
+ } else {
+ encoding.set_kind(rleVersion == RleVersion_1 ?
+ proto::ColumnEncoding_Kind_DICTIONARY :
+ proto::ColumnEncoding_Kind_DICTIONARY_V2);
+ }
+ encoding.set_dictionarysize(static_cast<uint32_t>(dictionary.size()));
encodings.push_back(encoding);
}
void StringColumnWriter::recordPosition() const {
ColumnWriter::recordPosition();
- dataStream->recordPosition(rowIndexPosition.get());
- lengthEncoder->recordPosition(rowIndexPosition.get());
+ if (!useDictionary) {
+ directDataStream->recordPosition(rowIndexPosition.get());
+ directLengthEncoder->recordPosition(rowIndexPosition.get());
+ } else {
+ if (enableIndex) {
+ startOfRowGroups.push_back(dictionary.idxInDictBuffer.size());
+ }
+ }
+ }
+
+ bool StringColumnWriter::checkDictionaryKeyRatio() {
+ if (!doneDictionaryCheck) {
+ useDictionary = dictionary.size() <= static_cast<size_t>(
+ static_cast<double>(dictionary.idxInDictBuffer.size()) *
dictSizeThreshold);
+ doneDictionaryCheck = true;
+ }
+
+ return useDictionary;
+ }
+
+ void StringColumnWriter::createRowIndexEntry() {
+ if (useDictionary && !doneDictionaryCheck) {
+ if (!checkDictionaryKeyRatio()) {
+ fallbackToDirectEncoding();
+ }
+ }
+ ColumnWriter::createRowIndexEntry();
+ }
+
+ void StringColumnWriter::reset() {
+ ColumnWriter::reset();
+
+ dictionary.clear();
+ dictionary.idxInDictBuffer.resize(0);
+ startOfRowGroups.clear();
+ startOfRowGroups.push_back(0);
+ }
+
+ void StringColumnWriter::createDirectStreams() {
+ std::unique_ptr<BufferedOutputStream> directLengthStream =
+ streamsFactory.createStream(proto::Stream_Kind_LENGTH);
+ directLengthEncoder = createRleEncoder(std::move(directLengthStream),
+ false,
+ rleVersion,
+ memPool,
+ alignedBitPacking);
+ directDataStream.reset(new AppendOnlyBufferedStream(
+ streamsFactory.createStream(proto::Stream_Kind_DATA)));
+ }
+
+ void StringColumnWriter::createDictStreams() {
+ std::unique_ptr<BufferedOutputStream> dictDataStream =
+ streamsFactory.createStream(proto::Stream_Kind_DATA);
+ dictDataEncoder = createRleEncoder(std::move(dictDataStream),
+ false,
+ rleVersion,
+ memPool,
+ alignedBitPacking);
+ std::unique_ptr<BufferedOutputStream> dictLengthStream =
+ streamsFactory.createStream(proto::Stream_Kind_LENGTH);
+ dictLengthEncoder = createRleEncoder(std::move(dictLengthStream),
+ false,
+ rleVersion,
+ memPool,
+ alignedBitPacking);
+ dictStream.reset(new AppendOnlyBufferedStream(
+ streamsFactory.createStream(proto::Stream_Kind_DICTIONARY_DATA)));
+ }
+
+ void StringColumnWriter::deleteDictStreams() {
+ dictDataEncoder.reset(nullptr);
+ dictLengthEncoder.reset(nullptr);
+ dictStream.reset(nullptr);
+
+ dictionary.clear();
+ dictionary.idxInDictBuffer.clear();
+ startOfRowGroups.clear();
+ }
+
+ void StringColumnWriter::writeDictionary() {
+ if (useDictionary && !doneDictionaryCheck) {
+ // when index is disabled, dictionary check happens while writing 1st
stripe
+ if (!checkDictionaryKeyRatio()) {
+ fallbackToDirectEncoding();
+ return;
+ }
+ }
+
+ if (useDictionary) {
+ // flush dictionary data & length streams
+ dictionary.flush(dictStream.get(), dictLengthEncoder.get());
+
+ // convert index from insertion order to dictionary order
+ dictionary.reorder(dictionary.idxInDictBuffer);
+
+ // write data sequences
+ int64_t * data = dictionary.idxInDictBuffer.data();
+ if (enableIndex) {
+ size_t prevOffset = 0;
+ for (size_t i = 0; i < startOfRowGroups.size(); ++i) {
+ // write sequences in batch for a row group stride
+ size_t offset = startOfRowGroups[i];
+ dictDataEncoder->add(data + prevOffset, offset - prevOffset,
nullptr);
+
+ // update index positions
+ int rowGroupId = static_cast<int>(i);
+ proto::RowIndexEntry* indexEntry =
+ (rowGroupId < rowIndex->entry_size()) ?
+ rowIndex->mutable_entry(rowGroupId) : rowIndexEntry.get();
+
+ // add positions for direct streams
+ RowIndexPositionRecorder recorder(*indexEntry);
+ dictDataEncoder->recordPosition(&recorder);
+
+ prevOffset = offset;
+ }
+
+ dictDataEncoder->add(data + prevOffset,
+ dictionary.idxInDictBuffer.size() - prevOffset,
+ nullptr);
+ } else {
+ dictDataEncoder->add(data, dictionary.idxInDictBuffer.size(), nullptr);
+ }
+ }
+ }
+
+ void StringColumnWriter::fallbackToDirectEncoding() {
+ createDirectStreams();
+
+ if (enableIndex) {
+ // fallback happens at the 1st row group;
+ // simply complete positions for direct streams
+ proto::RowIndexEntry * indexEntry = rowIndexEntry.get();
+ RowIndexPositionRecorder recorder(*indexEntry);
+ directDataStream->recordPosition(&recorder);
+ directLengthEncoder->recordPosition(&recorder);
+ }
+
+ // get dictionary entries in insertion order
+ std::vector<const StringDictionary::DictEntry *> entries;
+ dictionary.getEntriesInInsertionOrder(entries);
+
+ // store each length of the data into a vector
+ const StringDictionary::DictEntry * dictEntry = nullptr;
+ for (uint64_t i = 0; i != dictionary.idxInDictBuffer.size(); ++i) {
+ // write one row data in direct encoding
+ dictEntry = entries[static_cast<size_t>(dictionary.idxInDictBuffer[i])];
+ directDataStream->write(dictEntry->data, dictEntry->length);
+ directLengthEncoder->write(static_cast<int64_t>(dictEntry->length));
+ }
+
+ deleteDictStreams();
}
struct Utf8Utils {
@@ -1083,14 +1468,25 @@ namespace orc {
' ',
static_cast<size_t>(length[i]) - originLength);
}
- dataStream->write(charData, static_cast<size_t>(length[i]));
+
+ if (useDictionary) {
+ size_t index = dictionary.insert(charData,
static_cast<size_t>(length[i]));
+ dictionary.idxInDictBuffer.push_back(static_cast<int64_t>(index));
+ } else {
+ directDataStream->write(charData, static_cast<size_t>(length[i]));
+ }
+
strStats->update(charData, static_cast<size_t>(length[i]));
strStats->increase(1);
} else if (!hasNull) {
hasNull = true;
}
}
- lengthEncoder->add(length, numValues, notNull);
+
+ if (!useDictionary) {
+ directLengthEncoder->add(length, numValues, notNull);
+ }
+
if (hasNull) {
strStats->setHasNull(true);
}
@@ -1139,9 +1535,14 @@ namespace orc {
if (!notNull || notNull[i]) {
uint64_t itemLength = Utf8Utils::truncateBytesTo(
maxLength, data[i], static_cast<uint64_t>(length[i]));
-
length[i] = static_cast<int64_t>(itemLength);
- dataStream->write(data[i], static_cast<size_t>(length[i]));
+
+ if (useDictionary) {
+ size_t index = dictionary.insert(data[i],
static_cast<size_t>(length[i]));
+ dictionary.idxInDictBuffer.push_back(static_cast<int64_t>(index));
+ } else {
+ directDataStream->write(data[i], static_cast<size_t>(length[i]));
+ }
strStats->update(data[i], static_cast<size_t>(length[i]));
strStats->increase(1);
@@ -1149,7 +1550,11 @@ namespace orc {
hasNull = true;
}
}
- lengthEncoder->add(length, numValues, notNull);
+
+ if (!useDictionary) {
+ directLengthEncoder->add(length, numValues, notNull);
+ }
+
if (hasNull) {
strStats->setHasNull(true);
}
@@ -1193,7 +1598,7 @@ namespace orc {
for (uint64_t i = 0; i < numValues; ++i) {
uint64_t unsignedLength = static_cast<uint64_t>(length[i]);
if (!notNull || notNull[i]) {
- dataStream->write(data[i], unsignedLength);
+ directDataStream->write(data[i], unsignedLength);
binStats->update(unsignedLength);
binStats->increase(1);
@@ -1201,7 +1606,7 @@ namespace orc {
hasNull = true;
}
}
- lengthEncoder->add(length, numValues, notNull);
+ directLengthEncoder->add(length, numValues, notNull);
if (hasNull) {
binStats->setHasNull(true);
}
@@ -1680,6 +2085,10 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
private:
std::unique_ptr<RleEncoder> lengthEncoder;
RleVersion rleVersion;
@@ -1843,6 +2252,19 @@ namespace orc {
lengthEncoder->recordPosition(rowIndexPosition.get());
}
+ void ListColumnWriter::reset() {
+ ColumnWriter::reset();
+ if (child) {
+ child->reset();
+ }
+ }
+
+ void ListColumnWriter::writeDictionary() {
+ if (child) {
+ child->writeDictionary();
+ }
+ }
+
class MapColumnWriter : public ColumnWriter {
public:
MapColumnWriter(const Type& type,
@@ -1878,6 +2300,10 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
private:
std::unique_ptr<ColumnWriter> keyWriter;
std::unique_ptr<ColumnWriter> elemWriter;
@@ -2077,6 +2503,25 @@ namespace orc {
lengthEncoder->recordPosition(rowIndexPosition.get());
}
+ void MapColumnWriter::reset() {
+ ColumnWriter::reset();
+ if (keyWriter) {
+ keyWriter->reset();
+ }
+ if (elemWriter) {
+ elemWriter->reset();
+ }
+ }
+
+ void MapColumnWriter::writeDictionary() {
+ if (keyWriter) {
+ keyWriter->writeDictionary();
+ }
+ if (elemWriter) {
+ elemWriter->writeDictionary();
+ }
+ }
+
class UnionColumnWriter : public ColumnWriter {
public:
UnionColumnWriter(const Type& type,
@@ -2112,6 +2557,10 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
private:
std::unique_ptr<ByteRleEncoder> rleEncoder;
std::vector<ColumnWriter*> children;
@@ -2281,6 +2730,19 @@ namespace orc {
rleEncoder->recordPosition(rowIndexPosition.get());
}
+ void UnionColumnWriter::reset() {
+ ColumnWriter::reset();
+ for (uint32_t i = 0; i < children.size(); ++i) {
+ children[i]->reset();
+ }
+ }
+
+ void UnionColumnWriter::writeDictionary() {
+ for (uint32_t i = 0; i < children.size(); ++i) {
+ children[i]->writeDictionary();
+ }
+ }
+
std::unique_ptr<ColumnWriter> buildWriter(
const Type& type,
const StreamsFactory& factory,
diff --git a/c++/src/ColumnWriter.hh b/c++/src/ColumnWriter.hh
index 14fc54a295..a1419d7b88 100644
--- a/c++/src/ColumnWriter.hh
+++ b/c++/src/ColumnWriter.hh
@@ -165,6 +165,11 @@ namespace orc {
*/
virtual void reset();
+ /**
+ * Write dictionary to streams for string columns
+ */
+ virtual void writeDictionary();
+
protected:
/**
* Utility function to translate ColumnStatistics into protobuf form and
diff --git a/c++/src/RLE.hh b/c++/src/RLE.hh
index 3eb3489177..6822bd812e 100644
--- a/c++/src/RLE.hh
+++ b/c++/src/RLE.hh
@@ -79,6 +79,8 @@ namespace orc {
*/
virtual void recordPosition(PositionRecorder* recorder) const;
+ virtual void write(int64_t val) = 0;
+
protected:
std::unique_ptr<BufferedOutputStream> outputStream;
size_t bufferPosition;
@@ -88,8 +90,6 @@ namespace orc {
bool isSigned;
char* buffer;
- virtual void write(int64_t val) = 0;
-
virtual void writeByte(char c);
virtual void writeVulong(int64_t val);
diff --git a/c++/src/RLEv1.hh b/c++/src/RLEv1.hh
index b83c51cede..4dd9e5da58 100644
--- a/c++/src/RLEv1.hh
+++ b/c++/src/RLEv1.hh
@@ -37,12 +37,13 @@ public:
*/
uint64_t flush() override;
+ void write(int64_t val) override;
+
private:
int64_t delta;
bool repeat;
int tailRunLength;
- void write(int64_t val) override;
void writeValues();
};
diff --git a/c++/src/RLEv2.hh b/c++/src/RLEv2.hh
index a51b276cba..f85dabd9e6 100644
--- a/c++/src/RLEv2.hh
+++ b/c++/src/RLEv2.hh
@@ -75,6 +75,8 @@ public:
*/
uint64_t flush() override;
+ void write(int64_t val) override;
+
private:
const bool alignedBitPacking;
@@ -95,7 +97,6 @@ private:
void computeZigZagLiterals(EncodingOption& option);
void preparePatchedBlob(EncodingOption& option);
- void write(int64_t val) override ;
void writeInts(int64_t* input, uint32_t offset, size_t len, uint32_t
bitSize);
void initializeLiterals(int64_t val);
void writeValues(EncodingOption& option);
diff --git a/c++/src/Writer.cc b/c++/src/Writer.cc
index 621876927b..f22bf283b9 100644
--- a/c++/src/Writer.cc
+++ b/c++/src/Writer.cc
@@ -196,6 +196,10 @@ namespace orc {
return privateBits->enableIndex;
}
+ bool WriterOptions::getEnableDictionary() const {
+ return privateBits->dictionaryKeySizeThreshold > 0.0;
+ }
+
Writer::~Writer() {
// PASS
}
@@ -381,6 +385,9 @@ namespace orc {
columnWriter->mergeRowGroupStatsIntoStripeStats();
}
+ // dictionary should be written before any stream is flushed
+ columnWriter->writeDictionary();
+
std::vector<proto::Stream> streams;
// write ROW_INDEX streams
if (options.getEnableIndex()) {
diff --git a/c++/test/CMakeLists.txt b/c++/test/CMakeLists.txt
index 015814ac45..6e3f3b3b13 100644
--- a/c++/test/CMakeLists.txt
+++ b/c++/test/CMakeLists.txt
@@ -34,6 +34,7 @@ add_executable (orc-test
TestCompression.cc
TestDecompression.cc
TestDecimal.cc
+ TestDictionaryEncoding.cc
TestDriver.cc
TestInt128.cc
TestReader.cc
diff --git a/c++/test/TestDictionaryEncoding.cc
b/c++/test/TestDictionaryEncoding.cc
new file mode 100644
index 0000000000..d4952a9cb0
--- /dev/null
+++ b/c++/test/TestDictionaryEncoding.cc
@@ -0,0 +1,456 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "orc/OrcFile.hh"
+
+#include "MemoryInputStream.hh"
+#include "MemoryOutputStream.hh"
+
+#include "wrap/gmock.h"
+#include "wrap/gtest-wrapper.h"
+
+#include <cmath>
+#include <sstream>
+
+namespace orc {
+
+ const int DEFAULT_MEM_STREAM_SIZE = 10 * 1024 * 1024; // 10M
+ const int DICTIONARY_SIZE_1K = 1000;
+ const double DICT_THRESHOLD = 0.2; // make sure dictionary is used
+ const double FALLBACK_THRESHOLD = 0.0; // make sure fallback happens
+
+ static std::unique_ptr<Reader> createReader(MemoryPool * memoryPool,
+ std::unique_ptr<InputStream>
stream) {
+ ReaderOptions options;
+ options.setMemoryPool(*memoryPool);
+ return createReader(std::move(stream), options);
+ }
+
+ static std::unique_ptr<RowReader> createRowReader(Reader* reader) {
+ RowReaderOptions rowReaderOpts;
+ return reader->createRowReader(rowReaderOpts);
+ }
+
+ void testStringDictionary(bool enableIndex, double threshold) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:string>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 65535;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * strBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ strBatch->data[i] = dataBuffer + offset;
+ strBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+ structBatch->numElements = rowCount;
+ strBatch->numElements = rowCount;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+ std::string str(
+ strBatch->data[i],
+ static_cast<size_t>(strBatch->length[i]));
+ EXPECT_EQ(i % dictionarySize, static_cast<uint64_t>(atoi(str.c_str())));
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testVarcharDictionary(bool enableIndex, double threshold) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:varchar(2)>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 65535;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * varcharBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ varcharBatch->data[i] = dataBuffer + offset;
+ varcharBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+ structBatch->numElements = rowCount;
+ varcharBatch->numElements = rowCount;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ varcharBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ EXPECT_FALSE(varcharBatch->length[i] > 2);
+ std::string varcharRead(
+ varcharBatch->data[i],
+ static_cast<size_t>(varcharBatch->length[i]));
+ std::string varcharExpected = os.str().substr(0, 2);
+ EXPECT_EQ(varcharRead, varcharExpected);
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testCharDictionary(bool enableIndex, double threshold) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:char(3)>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 10000;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * charBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+ charBatch->data[i] = dataBuffer + offset;
+ charBatch->length[i] = static_cast<int64_t>(os.str().size());
+ offset += os.str().size();
+ }
+ structBatch->numElements = rowCount;
+ charBatch->numElements = rowCount;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ charBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ EXPECT_EQ(3, charBatch->length[i]);
+ std::string charsRead(
+ charBatch->data[i],
+ static_cast<size_t>(charBatch->length[i]));
+
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ std::string charsExpected = os.str().substr(0, 3);
+ while (charsExpected.length() < 3) {
+ charsExpected += ' ';
+ }
+ EXPECT_EQ(charsExpected, charsRead);
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testStringDictionaryWithNull(double threshold, bool enableIndex) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:string>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 65535;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * strBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ if (i % 2 == 0) {
+ strBatch->notNull[i] = false;
+ } else {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ strBatch->notNull[i] = true;
+ strBatch->data[i] = dataBuffer + offset;
+ strBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+ }
+
+ structBatch->numElements = rowCount;
+ strBatch->numElements = rowCount;
+ strBatch->hasNulls = true;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ if (i % 2 == 0) {
+ EXPECT_FALSE(strBatch->notNull[i]);
+ } else {
+ EXPECT_TRUE(strBatch->notNull[i]);
+ std::string str(strBatch->data[i],
+ static_cast<size_t>(strBatch->length[i]));
+ EXPECT_EQ(i % dictionarySize,
static_cast<uint64_t>(atoi(str.c_str())));
+ }
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testDictionaryMultipleStripes(double threshold, bool enableIndex) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:string>"));
+
+ WriterOptions options;
+ options.setStripeSize(1);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[800000];
+ uint64_t offset = 0, rowCount = 65535, stripeCount = 3;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * strBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+
+ for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ strBatch->data[i] = dataBuffer + offset;
+ strBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+
+ structBatch->numElements = rowCount;
+ strBatch->numElements = rowCount;
+
+ writer->add(*batch);
+ }
+
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(new MemoryInputStream(
+ memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+
+ // make sure there are multiple stripes
+ EXPECT_EQ(rowCount * stripeCount, reader->getNumberOfRows());
+ EXPECT_EQ(stripeCount, reader->getNumberOfStripes());
+
+ // test reading sequentially for data correctness
+ batch = rowReader->createRowBatch(rowCount);
+ for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
+ EXPECT_EQ(true, rowReader->next(*batch));
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+ std::string str(strBatch->data[i],
+ static_cast<size_t>(strBatch->length[i]));
+ EXPECT_EQ(i % dictionarySize,
static_cast<uint64_t>(atoi(str.c_str())));
+ }
+ }
+ EXPECT_EQ(false, rowReader->next(*batch));
+
+ // test seeking to check positions
+ batch = rowReader->createRowBatch(1);
+
+ for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
+ for (uint64_t i = 0; i < rowCount; i += 10000 / 2) {
+ rowReader->seekToRow(stripe * rowCount + i);
+ EXPECT_EQ(true, rowReader->next(*batch));
+
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+ std::string str(strBatch->data[0],
+ static_cast<size_t>(strBatch->length[0]));
+ EXPECT_EQ(i % dictionarySize,
static_cast<uint64_t>(atoi(str.c_str())));
+ }
+ }
+ }
+
+ // test dictionary encoding with index disabled
+ // the decision of using dictionary if made at the end of 1st stripe
+ TEST(DictionaryEncoding, writeStringDictionaryEncodingWithoutIndex) {
+ testStringDictionary(false, DICT_THRESHOLD);
+ testStringDictionary(false, FALLBACK_THRESHOLD);
+ }
+
+ // test dictionary encoding with index enabled
+ // the decision of using dictionary if made at the end of 1st row group
+ TEST(DictionaryEncoding, writeStringDictionaryEncodingWithIndex) {
+ testStringDictionary(true, DICT_THRESHOLD);
+ testStringDictionary(true, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeVarcharDictionaryEncodingWithoutIndex) {
+ testVarcharDictionary(false, DICT_THRESHOLD);
+ testVarcharDictionary(false, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeVarcharDictionaryEncodingWithIndex) {
+ testVarcharDictionary(true, DICT_THRESHOLD);
+ testVarcharDictionary(true, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeCharDictionaryEncodingWithoutIndex) {
+ testCharDictionary(false, DICT_THRESHOLD);
+ testCharDictionary(false, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeCharDictionaryEncodingWithIndex) {
+ testCharDictionary(true, DICT_THRESHOLD);
+ testCharDictionary(true, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, stringDictionaryWithNullWithIndex) {
+ testStringDictionaryWithNull(DICT_THRESHOLD, true);
+ testStringDictionaryWithNull(FALLBACK_THRESHOLD, true);
+ }
+
+ TEST(DictionaryEncoding, stringDictionaryWithNullWithoutIndex) {
+ testStringDictionaryWithNull(DICT_THRESHOLD, false);
+ testStringDictionaryWithNull(FALLBACK_THRESHOLD, false);
+ }
+
+ TEST(DictionaryEncoding, multipleStripesWithIndex) {
+ testDictionaryMultipleStripes(DICT_THRESHOLD, true);
+ testDictionaryMultipleStripes(FALLBACK_THRESHOLD, true);
+ }
+
+ TEST(DictionaryEncoding, multipleStripesWithoutIndex) {
+ testDictionaryMultipleStripes(DICT_THRESHOLD, false);
+ testDictionaryMultipleStripes(FALLBACK_THRESHOLD, false);
+ }
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [C++] Implement string dictionary encoding for C++ writer
> ---------------------------------------------------------
>
> Key: ORC-420
> URL: https://issues.apache.org/jira/browse/ORC-420
> Project: ORC
> Issue Type: New Feature
> Components: C++
> Reporter: Gang Wu
> Assignee: Gang Wu
> Priority: Major
>
> The scope of this Jira is to add string dictionary encoding support to C++
> writer.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
