This is an automated email from the ASF dual-hosted git repository.
mdeepak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/master by this push:
new e4fdb94 ORC-420: [C++] Implement string dictionary encoding for C++
writer
e4fdb94 is described below
commit e4fdb9473f31b4f1a9ee571258cbcbdff012c7b7
Author: Gang Wu <[email protected]>
AuthorDate: Fri Oct 19 15:59:38 2018 -0700
ORC-420: [C++] Implement string dictionary encoding for C++ writer
Fixes #326
Signed-off-by: Deepak Majeti <[email protected]>
---
c++/include/orc/Writer.hh | 6 +
c++/src/ColumnWriter.cc | 536 ++++++++++++++++++++++++++++++++++---
c++/src/ColumnWriter.hh | 5 +
c++/src/RLE.hh | 4 +-
c++/src/RLEv1.hh | 3 +-
c++/src/RLEv2.hh | 3 +-
c++/src/Writer.cc | 7 +
c++/test/CMakeLists.txt | 1 +
c++/test/TestDictionaryEncoding.cc | 456 +++++++++++++++++++++++++++++++
9 files changed, 980 insertions(+), 41 deletions(-)
diff --git a/c++/include/orc/Writer.hh b/c++/include/orc/Writer.hh
index 7efb12b..a333a23 100644
--- a/c++/include/orc/Writer.hh
+++ b/c++/include/orc/Writer.hh
@@ -185,6 +185,12 @@ namespace orc {
* @return if not set, the default is false
*/
bool getEnableIndex() const;
+
+ /**
+ * Get whether or not to enable dictionary encoding
+ * @return if not set, the default is false
+ */
+ bool getEnableDictionary() const;
};
class Writer {
diff --git a/c++/src/ColumnWriter.cc b/c++/src/ColumnWriter.cc
index 9b128e3..c6c30f3 100644
--- a/c++/src/ColumnWriter.cc
+++ b/c++/src/ColumnWriter.cc
@@ -204,6 +204,10 @@ namespace orc {
}
}
+ void ColumnWriter::writeDictionary() {
+ // PASS
+ }
+
class StructColumnWriter : public ColumnWriter {
public:
StructColumnWriter(
@@ -237,6 +241,8 @@ namespace orc {
virtual void writeIndex(
std::vector<proto::Stream> &streams) const override;
+ virtual void writeDictionary() override;
+
virtual void reset() override;
private:
@@ -382,6 +388,12 @@ namespace orc {
}
}
+ void StructColumnWriter::writeDictionary() {
+ for (uint32_t i = 0; i < children.size(); ++i) {
+ children[i]->writeDictionary();
+ }
+ }
+
class IntegerColumnWriter : public ColumnWriter {
public:
IntegerColumnWriter(
@@ -831,6 +843,137 @@ namespace orc {
dataStream->recordPosition(rowIndexPosition.get());
}
+ /**
+ * Implementation of increasing sorted string dictionary
+ */
+ class StringDictionary {
+ public:
+ struct DictEntry {
+ DictEntry(const char * str, size_t len):data(str),length(len) {}
+ const char * data;
+ size_t length;
+ };
+
+ StringDictionary():totalLength(0) {}
+
+ // insert a new string into dictionary, return its insertion order
+ size_t insert(const char * data, size_t len);
+
+ // write dictionary data & length to output buffer
+ void flush(AppendOnlyBufferedStream * dataStream,
+ RleEncoder * lengthEncoder) const;
+
+ // reorder input index buffer from insertion order to dictionary order
+ void reorder(std::vector<int64_t>& idxBuffer) const;
+
+ // get dict entries in insertion order
+ void getEntriesInInsertionOrder(std::vector<const DictEntry *>&) const;
+
+ // return count of entries
+ size_t size() const;
+
+ // return total length of strings in the dictioanry
+ uint64_t length() const;
+
+ void clear();
+
+ private:
+ struct LessThan {
+ bool operator()(const DictEntry& left, const DictEntry& right) const {
+ int ret = memcmp(left.data, right.data, std::min(left.length,
right.length));
+ if (ret != 0) {
+ return ret < 0;
+ }
+ return left.length < right.length;
+ }
+ };
+
+ std::map<DictEntry, size_t, LessThan> dict;
+ std::vector<std::vector<char>> data;
+ uint64_t totalLength;
+
+ // use friend class here to avoid being bothered by const function calls
+ friend class StringColumnWriter;
+ friend class CharColumnWriter;
+ friend class VarCharColumnWriter;
+ // store indexes of insertion order in the dictionary for not-null rows
+ std::vector<int64_t> idxInDictBuffer;
+ };
+
+ // insert a new string into dictionary, return its insertion order
+ size_t StringDictionary::insert(const char * str, size_t len) {
+ auto ret = dict.insert({DictEntry(str, len), dict.size()});
+ if (ret.second) {
+ // make a copy to internal storage
+ data.push_back(std::vector<char>(len));
+ memcpy(data.back().data(), str, len);
+ // update dictionary entry to link pointer to internal storage
+ DictEntry * entry = const_cast<DictEntry *>(&(ret.first->first));
+ entry->data = data.back().data();
+ totalLength += len;
+ }
+ return ret.first->second;
+ }
+
+ // write dictionary data & length to output buffer
+ void StringDictionary::flush(AppendOnlyBufferedStream * dataStream,
+ RleEncoder * lengthEncoder) const {
+ for (auto it = dict.cbegin(); it != dict.cend(); ++it) {
+ dataStream->write(it->first.data, it->first.length);
+ lengthEncoder->write(static_cast<int64_t>(it->first.length));
+ }
+ }
+
+ /**
+ * Reorder input index buffer from insertion order to dictionary order
+ *
+ * We require this function because string values are buffered by indexes
+ * in their insertion order. Until the entire dictionary is complete can
+ * we get their sorted indexes in the dictionary in that ORC specification
+ * demands dictionary should be ordered. Therefore this function transforms
+ * the indexes from insertion order to dictionary value order for final
+ * output.
+ */
+ void StringDictionary::reorder(std::vector<int64_t>& idxBuffer) const {
+ // iterate the dictionary to get mapping from insertion order to value
order
+ std::vector<size_t> mapping(dict.size());
+ size_t dictIdx = 0;
+ for (auto it = dict.cbegin(); it != dict.cend(); ++it) {
+ mapping[it->second] = dictIdx++;
+ }
+
+ // do the transformation
+ for (size_t i = 0; i != idxBuffer.size(); ++i) {
+ idxBuffer[i] = static_cast<int64_t>(
+ mapping[static_cast<size_t>(idxBuffer[i])]);
+ }
+ }
+
+ // get dict entries in insertion order
+ void StringDictionary::getEntriesInInsertionOrder(
+ std::vector<const DictEntry *>& entries) const {
+ entries.resize(dict.size());
+ for (auto it = dict.cbegin(); it != dict.cend(); ++it) {
+ entries[it->second] = &(it->first);
+ }
+ }
+
+ // return count of entries
+ size_t StringDictionary::size() const {
+ return dict.size();
+ }
+
+ // return total length of strings in the dictioanry
+ uint64_t StringDictionary::length() const {
+ return totalLength;
+ }
+
+ void StringDictionary::clear() {
+ totalLength = 0;
+ data.clear();
+ dict.clear();
+ }
+
class StringColumnWriter : public ColumnWriter {
public:
StringColumnWriter(const Type& type,
@@ -850,10 +993,50 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void createRowIndexEntry() override;
+
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
+ private:
+ /**
+ * dictionary related functions
+ */
+ bool checkDictionaryKeyRatio();
+ void createDirectStreams();
+ void createDictStreams();
+ void deleteDictStreams();
+ void fallbackToDirectEncoding();
+
protected:
- std::unique_ptr<RleEncoder> lengthEncoder;
- std::unique_ptr<AppendOnlyBufferedStream> dataStream;
RleVersion rleVersion;
+ bool useCompression;
+ const StreamsFactory& streamsFactory;
+ bool alignedBitPacking;
+
+ // direct encoding streams
+ std::unique_ptr<RleEncoder> directLengthEncoder;
+ std::unique_ptr<AppendOnlyBufferedStream> directDataStream;
+
+ // dictionary encoding streams
+ std::unique_ptr<RleEncoder> dictDataEncoder;
+ std::unique_ptr<RleEncoder> dictLengthEncoder;
+ std::unique_ptr<AppendOnlyBufferedStream> dictStream;
+
+ /**
+ * dictionary related variables
+ */
+ StringDictionary dictionary;
+ // whether or not dictionary checking is done
+ bool doneDictionaryCheck;
+ // whether or not it should be used
+ bool useDictionary;
+ // keys in the dictionary should not exceed this ratio
+ double dictSizeThreshold;
+
+ // record start row of each row group; null rows are skipped
+ mutable std::vector<size_t> startOfRowGroups;
};
StringColumnWriter::StringColumnWriter(
@@ -861,16 +1044,24 @@ namespace orc {
const StreamsFactory& factory,
const WriterOptions& options) :
ColumnWriter(type, factory, options),
- rleVersion(options.getRleVersion()) {
- std::unique_ptr<BufferedOutputStream> lengthStream =
- factory.createStream(proto::Stream_Kind_LENGTH);
- lengthEncoder = createRleEncoder(std::move(lengthStream),
- false,
- rleVersion,
- memPool,
- options.getAlignedBitpacking());
- dataStream.reset(new AppendOnlyBufferedStream(
- factory.createStream(proto::Stream_Kind_DATA)));
+ rleVersion(options.getRleVersion()),
+ useCompression(options.getCompression() !=
CompressionKind_NONE),
+ streamsFactory(factory),
+
alignedBitPacking(options.getAlignedBitpacking()),
+ doneDictionaryCheck(false),
+ useDictionary(options.getEnableDictionary()),
+
dictSizeThreshold(options.getDictionaryKeySizeThreshold()){
+ if (type.getKind() == TypeKind::BINARY) {
+ useDictionary = false;
+ doneDictionaryCheck = true;
+ }
+
+ if (useDictionary) {
+ createDictStreams();
+ } else {
+ doneDictionaryCheck = true;
+ createDirectStreams();
+ }
if (enableIndex) {
recordPosition();
@@ -892,7 +1083,9 @@ namespace orc {
const char* notNull = stringBatch->hasNulls ?
stringBatch->notNull.data() + offset : nullptr;
- lengthEncoder->add(length, numValues, notNull);
+ if (!useDictionary){
+ directLengthEncoder->add(length, numValues, notNull);
+ }
StringColumnStatisticsImpl* strStats =
dynamic_cast<StringColumnStatisticsImpl*>(colIndexStatistics.get());
@@ -902,7 +1095,12 @@ namespace orc {
bool hasNull = false;
for (uint64_t i = 0; i < numValues; ++i) {
if (!notNull || notNull[i]) {
- dataStream->write(data[i], static_cast<size_t>(length[i]));
+ if (useDictionary) {
+ size_t index = dictionary.insert(data[i],
static_cast<size_t>(length[i]));
+ dictionary.idxInDictBuffer.push_back(static_cast<int64_t>(index));
+ } else {
+ directDataStream->write(data[i], static_cast<size_t>(length[i]));
+ }
strStats->update(data[i], static_cast<size_t>(length[i]));
strStats->increase(1);
} else if (!hasNull) {
@@ -917,38 +1115,225 @@ namespace orc {
void StringColumnWriter::flush(std::vector<proto::Stream>& streams) {
ColumnWriter::flush(streams);
- proto::Stream length;
- length.set_kind(proto::Stream_Kind_LENGTH);
- length.set_column(static_cast<uint32_t>(columnId));
- length.set_length(lengthEncoder->flush());
- streams.push_back(length);
+ if (useDictionary) {
+ proto::Stream data;
+ data.set_kind(proto::Stream_Kind_DATA);
+ data.set_column(static_cast<uint32_t>(columnId));
+ data.set_length(dictDataEncoder->flush());
+ streams.push_back(data);
+
+ proto::Stream dict;
+ dict.set_kind(proto::Stream_Kind_DICTIONARY_DATA);
+ dict.set_column(static_cast<uint32_t>(columnId));
+ dict.set_length(dictStream->flush());
+ streams.push_back(dict);
+
+ proto::Stream length;
+ length.set_kind(proto::Stream_Kind_LENGTH);
+ length.set_column(static_cast<uint32_t>(columnId));
+ length.set_length(dictLengthEncoder->flush());
+ streams.push_back(length);
+ } else {
+ proto::Stream length;
+ length.set_kind(proto::Stream_Kind_LENGTH);
+ length.set_column(static_cast<uint32_t>(columnId));
+ length.set_length(directLengthEncoder->flush());
+ streams.push_back(length);
- proto::Stream data;
- data.set_kind(proto::Stream_Kind_DATA);
- data.set_column(static_cast<uint32_t>(columnId));
- data.set_length(dataStream->flush());
- streams.push_back(data);
+ proto::Stream data;
+ data.set_kind(proto::Stream_Kind_DATA);
+ data.set_column(static_cast<uint32_t>(columnId));
+ data.set_length(directDataStream->flush());
+ streams.push_back(data);
+ }
}
uint64_t StringColumnWriter::getEstimatedSize() const {
uint64_t size = ColumnWriter::getEstimatedSize();
- size += lengthEncoder->getBufferSize();
- size += dataStream->getSize();
+ if (!useDictionary) {
+ size += directLengthEncoder->getBufferSize();
+ size += directDataStream->getSize();
+ } else {
+ size += dictionary.length();
+ size += dictionary.size() * sizeof(int32_t);
+ size += dictionary.idxInDictBuffer.size() * sizeof(int32_t);
+ if (useCompression) {
+ size /= 3; // estimated ratio is 3:1
+ }
+ }
return size;
}
void StringColumnWriter::getColumnEncoding(
std::vector<proto::ColumnEncoding>& encodings) const {
proto::ColumnEncoding encoding;
- encoding.set_kind(RleVersionMapper(rleVersion));
- encoding.set_dictionarysize(0);
+ if (!useDictionary) {
+ encoding.set_kind(rleVersion == RleVersion_1 ?
+ proto::ColumnEncoding_Kind_DIRECT :
+ proto::ColumnEncoding_Kind_DIRECT_V2);
+ } else {
+ encoding.set_kind(rleVersion == RleVersion_1 ?
+ proto::ColumnEncoding_Kind_DICTIONARY :
+ proto::ColumnEncoding_Kind_DICTIONARY_V2);
+ }
+ encoding.set_dictionarysize(static_cast<uint32_t>(dictionary.size()));
encodings.push_back(encoding);
}
void StringColumnWriter::recordPosition() const {
ColumnWriter::recordPosition();
- dataStream->recordPosition(rowIndexPosition.get());
- lengthEncoder->recordPosition(rowIndexPosition.get());
+ if (!useDictionary) {
+ directDataStream->recordPosition(rowIndexPosition.get());
+ directLengthEncoder->recordPosition(rowIndexPosition.get());
+ } else {
+ if (enableIndex) {
+ startOfRowGroups.push_back(dictionary.idxInDictBuffer.size());
+ }
+ }
+ }
+
+ bool StringColumnWriter::checkDictionaryKeyRatio() {
+ if (!doneDictionaryCheck) {
+ useDictionary = dictionary.size() <= static_cast<size_t>(
+ static_cast<double>(dictionary.idxInDictBuffer.size()) *
dictSizeThreshold);
+ doneDictionaryCheck = true;
+ }
+
+ return useDictionary;
+ }
+
+ void StringColumnWriter::createRowIndexEntry() {
+ if (useDictionary && !doneDictionaryCheck) {
+ if (!checkDictionaryKeyRatio()) {
+ fallbackToDirectEncoding();
+ }
+ }
+ ColumnWriter::createRowIndexEntry();
+ }
+
+ void StringColumnWriter::reset() {
+ ColumnWriter::reset();
+
+ dictionary.clear();
+ dictionary.idxInDictBuffer.resize(0);
+ startOfRowGroups.clear();
+ startOfRowGroups.push_back(0);
+ }
+
+ void StringColumnWriter::createDirectStreams() {
+ std::unique_ptr<BufferedOutputStream> directLengthStream =
+ streamsFactory.createStream(proto::Stream_Kind_LENGTH);
+ directLengthEncoder = createRleEncoder(std::move(directLengthStream),
+ false,
+ rleVersion,
+ memPool,
+ alignedBitPacking);
+ directDataStream.reset(new AppendOnlyBufferedStream(
+ streamsFactory.createStream(proto::Stream_Kind_DATA)));
+ }
+
+ void StringColumnWriter::createDictStreams() {
+ std::unique_ptr<BufferedOutputStream> dictDataStream =
+ streamsFactory.createStream(proto::Stream_Kind_DATA);
+ dictDataEncoder = createRleEncoder(std::move(dictDataStream),
+ false,
+ rleVersion,
+ memPool,
+ alignedBitPacking);
+ std::unique_ptr<BufferedOutputStream> dictLengthStream =
+ streamsFactory.createStream(proto::Stream_Kind_LENGTH);
+ dictLengthEncoder = createRleEncoder(std::move(dictLengthStream),
+ false,
+ rleVersion,
+ memPool,
+ alignedBitPacking);
+ dictStream.reset(new AppendOnlyBufferedStream(
+ streamsFactory.createStream(proto::Stream_Kind_DICTIONARY_DATA)));
+ }
+
+ void StringColumnWriter::deleteDictStreams() {
+ dictDataEncoder.reset(nullptr);
+ dictLengthEncoder.reset(nullptr);
+ dictStream.reset(nullptr);
+
+ dictionary.clear();
+ dictionary.idxInDictBuffer.clear();
+ startOfRowGroups.clear();
+ }
+
+ void StringColumnWriter::writeDictionary() {
+ if (useDictionary && !doneDictionaryCheck) {
+ // when index is disabled, dictionary check happens while writing 1st
stripe
+ if (!checkDictionaryKeyRatio()) {
+ fallbackToDirectEncoding();
+ return;
+ }
+ }
+
+ if (useDictionary) {
+ // flush dictionary data & length streams
+ dictionary.flush(dictStream.get(), dictLengthEncoder.get());
+
+ // convert index from insertion order to dictionary order
+ dictionary.reorder(dictionary.idxInDictBuffer);
+
+ // write data sequences
+ int64_t * data = dictionary.idxInDictBuffer.data();
+ if (enableIndex) {
+ size_t prevOffset = 0;
+ for (size_t i = 0; i < startOfRowGroups.size(); ++i) {
+ // write sequences in batch for a row group stride
+ size_t offset = startOfRowGroups[i];
+ dictDataEncoder->add(data + prevOffset, offset - prevOffset,
nullptr);
+
+ // update index positions
+ int rowGroupId = static_cast<int>(i);
+ proto::RowIndexEntry* indexEntry =
+ (rowGroupId < rowIndex->entry_size()) ?
+ rowIndex->mutable_entry(rowGroupId) : rowIndexEntry.get();
+
+ // add positions for direct streams
+ RowIndexPositionRecorder recorder(*indexEntry);
+ dictDataEncoder->recordPosition(&recorder);
+
+ prevOffset = offset;
+ }
+
+ dictDataEncoder->add(data + prevOffset,
+ dictionary.idxInDictBuffer.size() - prevOffset,
+ nullptr);
+ } else {
+ dictDataEncoder->add(data, dictionary.idxInDictBuffer.size(), nullptr);
+ }
+ }
+ }
+
+ void StringColumnWriter::fallbackToDirectEncoding() {
+ createDirectStreams();
+
+ if (enableIndex) {
+ // fallback happens at the 1st row group;
+ // simply complete positions for direct streams
+ proto::RowIndexEntry * indexEntry = rowIndexEntry.get();
+ RowIndexPositionRecorder recorder(*indexEntry);
+ directDataStream->recordPosition(&recorder);
+ directLengthEncoder->recordPosition(&recorder);
+ }
+
+ // get dictionary entries in insertion order
+ std::vector<const StringDictionary::DictEntry *> entries;
+ dictionary.getEntriesInInsertionOrder(entries);
+
+ // store each length of the data into a vector
+ const StringDictionary::DictEntry * dictEntry = nullptr;
+ for (uint64_t i = 0; i != dictionary.idxInDictBuffer.size(); ++i) {
+ // write one row data in direct encoding
+ dictEntry = entries[static_cast<size_t>(dictionary.idxInDictBuffer[i])];
+ directDataStream->write(dictEntry->data, dictEntry->length);
+ directLengthEncoder->write(static_cast<int64_t>(dictEntry->length));
+ }
+
+ deleteDictStreams();
}
struct Utf8Utils {
@@ -1083,14 +1468,25 @@ namespace orc {
' ',
static_cast<size_t>(length[i]) - originLength);
}
- dataStream->write(charData, static_cast<size_t>(length[i]));
+
+ if (useDictionary) {
+ size_t index = dictionary.insert(charData,
static_cast<size_t>(length[i]));
+ dictionary.idxInDictBuffer.push_back(static_cast<int64_t>(index));
+ } else {
+ directDataStream->write(charData, static_cast<size_t>(length[i]));
+ }
+
strStats->update(charData, static_cast<size_t>(length[i]));
strStats->increase(1);
} else if (!hasNull) {
hasNull = true;
}
}
- lengthEncoder->add(length, numValues, notNull);
+
+ if (!useDictionary) {
+ directLengthEncoder->add(length, numValues, notNull);
+ }
+
if (hasNull) {
strStats->setHasNull(true);
}
@@ -1139,9 +1535,14 @@ namespace orc {
if (!notNull || notNull[i]) {
uint64_t itemLength = Utf8Utils::truncateBytesTo(
maxLength, data[i], static_cast<uint64_t>(length[i]));
-
length[i] = static_cast<int64_t>(itemLength);
- dataStream->write(data[i], static_cast<size_t>(length[i]));
+
+ if (useDictionary) {
+ size_t index = dictionary.insert(data[i],
static_cast<size_t>(length[i]));
+ dictionary.idxInDictBuffer.push_back(static_cast<int64_t>(index));
+ } else {
+ directDataStream->write(data[i], static_cast<size_t>(length[i]));
+ }
strStats->update(data[i], static_cast<size_t>(length[i]));
strStats->increase(1);
@@ -1149,7 +1550,11 @@ namespace orc {
hasNull = true;
}
}
- lengthEncoder->add(length, numValues, notNull);
+
+ if (!useDictionary) {
+ directLengthEncoder->add(length, numValues, notNull);
+ }
+
if (hasNull) {
strStats->setHasNull(true);
}
@@ -1193,7 +1598,7 @@ namespace orc {
for (uint64_t i = 0; i < numValues; ++i) {
uint64_t unsignedLength = static_cast<uint64_t>(length[i]);
if (!notNull || notNull[i]) {
- dataStream->write(data[i], unsignedLength);
+ directDataStream->write(data[i], unsignedLength);
binStats->update(unsignedLength);
binStats->increase(1);
@@ -1201,7 +1606,7 @@ namespace orc {
hasNull = true;
}
}
- lengthEncoder->add(length, numValues, notNull);
+ directLengthEncoder->add(length, numValues, notNull);
if (hasNull) {
binStats->setHasNull(true);
}
@@ -1680,6 +2085,10 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
private:
std::unique_ptr<RleEncoder> lengthEncoder;
RleVersion rleVersion;
@@ -1843,6 +2252,19 @@ namespace orc {
lengthEncoder->recordPosition(rowIndexPosition.get());
}
+ void ListColumnWriter::reset() {
+ ColumnWriter::reset();
+ if (child) {
+ child->reset();
+ }
+ }
+
+ void ListColumnWriter::writeDictionary() {
+ if (child) {
+ child->writeDictionary();
+ }
+ }
+
class MapColumnWriter : public ColumnWriter {
public:
MapColumnWriter(const Type& type,
@@ -1878,6 +2300,10 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
private:
std::unique_ptr<ColumnWriter> keyWriter;
std::unique_ptr<ColumnWriter> elemWriter;
@@ -2077,6 +2503,25 @@ namespace orc {
lengthEncoder->recordPosition(rowIndexPosition.get());
}
+ void MapColumnWriter::reset() {
+ ColumnWriter::reset();
+ if (keyWriter) {
+ keyWriter->reset();
+ }
+ if (elemWriter) {
+ elemWriter->reset();
+ }
+ }
+
+ void MapColumnWriter::writeDictionary() {
+ if (keyWriter) {
+ keyWriter->writeDictionary();
+ }
+ if (elemWriter) {
+ elemWriter->writeDictionary();
+ }
+ }
+
class UnionColumnWriter : public ColumnWriter {
public:
UnionColumnWriter(const Type& type,
@@ -2112,6 +2557,10 @@ namespace orc {
virtual void recordPosition() const override;
+ virtual void writeDictionary() override;
+
+ virtual void reset() override;
+
private:
std::unique_ptr<ByteRleEncoder> rleEncoder;
std::vector<ColumnWriter*> children;
@@ -2281,6 +2730,19 @@ namespace orc {
rleEncoder->recordPosition(rowIndexPosition.get());
}
+ void UnionColumnWriter::reset() {
+ ColumnWriter::reset();
+ for (uint32_t i = 0; i < children.size(); ++i) {
+ children[i]->reset();
+ }
+ }
+
+ void UnionColumnWriter::writeDictionary() {
+ for (uint32_t i = 0; i < children.size(); ++i) {
+ children[i]->writeDictionary();
+ }
+ }
+
std::unique_ptr<ColumnWriter> buildWriter(
const Type& type,
const StreamsFactory& factory,
diff --git a/c++/src/ColumnWriter.hh b/c++/src/ColumnWriter.hh
index 14fc54a..a1419d7 100644
--- a/c++/src/ColumnWriter.hh
+++ b/c++/src/ColumnWriter.hh
@@ -165,6 +165,11 @@ namespace orc {
*/
virtual void reset();
+ /**
+ * Write dictionary to streams for string columns
+ */
+ virtual void writeDictionary();
+
protected:
/**
* Utility function to translate ColumnStatistics into protobuf form and
diff --git a/c++/src/RLE.hh b/c++/src/RLE.hh
index 3eb3489..6822bd8 100644
--- a/c++/src/RLE.hh
+++ b/c++/src/RLE.hh
@@ -79,6 +79,8 @@ namespace orc {
*/
virtual void recordPosition(PositionRecorder* recorder) const;
+ virtual void write(int64_t val) = 0;
+
protected:
std::unique_ptr<BufferedOutputStream> outputStream;
size_t bufferPosition;
@@ -88,8 +90,6 @@ namespace orc {
bool isSigned;
char* buffer;
- virtual void write(int64_t val) = 0;
-
virtual void writeByte(char c);
virtual void writeVulong(int64_t val);
diff --git a/c++/src/RLEv1.hh b/c++/src/RLEv1.hh
index 26a180e..8e31d70 100644
--- a/c++/src/RLEv1.hh
+++ b/c++/src/RLEv1.hh
@@ -37,12 +37,13 @@ public:
*/
uint64_t flush() override;
+ void write(int64_t val) override;
+
private:
int64_t delta;
bool repeat;
uint64_t tailRunLength;
- void write(int64_t val) override;
void writeValues();
};
diff --git a/c++/src/RLEv2.hh b/c++/src/RLEv2.hh
index a51b276..f85dabd 100644
--- a/c++/src/RLEv2.hh
+++ b/c++/src/RLEv2.hh
@@ -75,6 +75,8 @@ public:
*/
uint64_t flush() override;
+ void write(int64_t val) override;
+
private:
const bool alignedBitPacking;
@@ -95,7 +97,6 @@ private:
void computeZigZagLiterals(EncodingOption& option);
void preparePatchedBlob(EncodingOption& option);
- void write(int64_t val) override ;
void writeInts(int64_t* input, uint32_t offset, size_t len, uint32_t
bitSize);
void initializeLiterals(int64_t val);
void writeValues(EncodingOption& option);
diff --git a/c++/src/Writer.cc b/c++/src/Writer.cc
index 9a7a777..6af1043 100644
--- a/c++/src/Writer.cc
+++ b/c++/src/Writer.cc
@@ -196,6 +196,10 @@ namespace orc {
return privateBits->enableIndex;
}
+ bool WriterOptions::getEnableDictionary() const {
+ return privateBits->dictionaryKeySizeThreshold > 0.0;
+ }
+
Writer::~Writer() {
// PASS
}
@@ -381,6 +385,9 @@ namespace orc {
columnWriter->mergeRowGroupStatsIntoStripeStats();
}
+ // dictionary should be written before any stream is flushed
+ columnWriter->writeDictionary();
+
std::vector<proto::Stream> streams;
// write ROW_INDEX streams
if (options.getEnableIndex()) {
diff --git a/c++/test/CMakeLists.txt b/c++/test/CMakeLists.txt
index 015814a..6e3f3b3 100644
--- a/c++/test/CMakeLists.txt
+++ b/c++/test/CMakeLists.txt
@@ -34,6 +34,7 @@ add_executable (orc-test
TestCompression.cc
TestDecompression.cc
TestDecimal.cc
+ TestDictionaryEncoding.cc
TestDriver.cc
TestInt128.cc
TestReader.cc
diff --git a/c++/test/TestDictionaryEncoding.cc
b/c++/test/TestDictionaryEncoding.cc
new file mode 100644
index 0000000..d4952a9
--- /dev/null
+++ b/c++/test/TestDictionaryEncoding.cc
@@ -0,0 +1,456 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "orc/OrcFile.hh"
+
+#include "MemoryInputStream.hh"
+#include "MemoryOutputStream.hh"
+
+#include "wrap/gmock.h"
+#include "wrap/gtest-wrapper.h"
+
+#include <cmath>
+#include <sstream>
+
+namespace orc {
+
+ const int DEFAULT_MEM_STREAM_SIZE = 10 * 1024 * 1024; // 10M
+ const int DICTIONARY_SIZE_1K = 1000;
+ const double DICT_THRESHOLD = 0.2; // make sure dictionary is used
+ const double FALLBACK_THRESHOLD = 0.0; // make sure fallback happens
+
+ static std::unique_ptr<Reader> createReader(MemoryPool * memoryPool,
+ std::unique_ptr<InputStream>
stream) {
+ ReaderOptions options;
+ options.setMemoryPool(*memoryPool);
+ return createReader(std::move(stream), options);
+ }
+
+ static std::unique_ptr<RowReader> createRowReader(Reader* reader) {
+ RowReaderOptions rowReaderOpts;
+ return reader->createRowReader(rowReaderOpts);
+ }
+
+ void testStringDictionary(bool enableIndex, double threshold) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:string>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 65535;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * strBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ strBatch->data[i] = dataBuffer + offset;
+ strBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+ structBatch->numElements = rowCount;
+ strBatch->numElements = rowCount;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+ std::string str(
+ strBatch->data[i],
+ static_cast<size_t>(strBatch->length[i]));
+ EXPECT_EQ(i % dictionarySize, static_cast<uint64_t>(atoi(str.c_str())));
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testVarcharDictionary(bool enableIndex, double threshold) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:varchar(2)>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 65535;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * varcharBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ varcharBatch->data[i] = dataBuffer + offset;
+ varcharBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+ structBatch->numElements = rowCount;
+ varcharBatch->numElements = rowCount;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ varcharBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ EXPECT_FALSE(varcharBatch->length[i] > 2);
+ std::string varcharRead(
+ varcharBatch->data[i],
+ static_cast<size_t>(varcharBatch->length[i]));
+ std::string varcharExpected = os.str().substr(0, 2);
+ EXPECT_EQ(varcharRead, varcharExpected);
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testCharDictionary(bool enableIndex, double threshold) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:char(3)>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 10000;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * charBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+ charBatch->data[i] = dataBuffer + offset;
+ charBatch->length[i] = static_cast<int64_t>(os.str().size());
+ offset += os.str().size();
+ }
+ structBatch->numElements = rowCount;
+ charBatch->numElements = rowCount;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ charBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ EXPECT_EQ(3, charBatch->length[i]);
+ std::string charsRead(
+ charBatch->data[i],
+ static_cast<size_t>(charBatch->length[i]));
+
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ std::string charsExpected = os.str().substr(0, 3);
+ while (charsExpected.length() < 3) {
+ charsExpected += ' ';
+ }
+ EXPECT_EQ(charsExpected, charsRead);
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testStringDictionaryWithNull(double threshold, bool enableIndex) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:string>"));
+
+ WriterOptions options;
+ options.setStripeSize(1024);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[327675];
+ uint64_t offset = 0, rowCount = 65535;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * strBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ if (i % 2 == 0) {
+ strBatch->notNull[i] = false;
+ } else {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ strBatch->notNull[i] = true;
+ strBatch->data[i] = dataBuffer + offset;
+ strBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+ }
+
+ structBatch->numElements = rowCount;
+ strBatch->numElements = rowCount;
+ strBatch->hasNulls = true;
+ writer->add(*batch);
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(
+ new MemoryInputStream (memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+ EXPECT_EQ(rowCount, reader->getNumberOfRows());
+
+ batch = rowReader->createRowBatch(rowCount);
+ EXPECT_EQ(true, rowReader->next(*batch));
+ EXPECT_EQ(rowCount, batch->numElements);
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ if (i % 2 == 0) {
+ EXPECT_FALSE(strBatch->notNull[i]);
+ } else {
+ EXPECT_TRUE(strBatch->notNull[i]);
+ std::string str(strBatch->data[i],
+ static_cast<size_t>(strBatch->length[i]));
+ EXPECT_EQ(i % dictionarySize,
static_cast<uint64_t>(atoi(str.c_str())));
+ }
+ }
+
+ EXPECT_EQ(false, rowReader->next(*batch));
+ }
+
+ void testDictionaryMultipleStripes(double threshold, bool enableIndex) {
+ MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE);
+ MemoryPool * pool = getDefaultPool();
+ std::unique_ptr<Type>
type(Type::buildTypeFromString("struct<col1:string>"));
+
+ WriterOptions options;
+ options.setStripeSize(1);
+ options.setCompressionBlockSize(1024);
+ options.setCompression(CompressionKind_ZLIB);
+ options.setMemoryPool(pool);
+ options.setDictionaryKeySizeThreshold(threshold);
+ options.setRowIndexStride(enableIndex ? 10000 : 0);
+ std::unique_ptr<Writer> writer = createWriter(*type, &memStream, options);
+
+ char dataBuffer[800000];
+ uint64_t offset = 0, rowCount = 65535, stripeCount = 3;
+ std::unique_ptr<ColumnVectorBatch> batch =
+ writer->createRowBatch(rowCount);
+ StructVectorBatch * structBatch =
+ dynamic_cast<StructVectorBatch *>(batch.get());
+ StringVectorBatch * strBatch =
+ dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+
+ uint64_t dictionarySize = DICTIONARY_SIZE_1K;
+
+ for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ std::ostringstream os;
+ os << (i % dictionarySize);
+ memcpy(dataBuffer + offset, os.str().c_str(), os.str().size());
+
+ strBatch->data[i] = dataBuffer + offset;
+ strBatch->length[i] = static_cast<int64_t>(os.str().size());
+
+ offset += os.str().size();
+ }
+
+ structBatch->numElements = rowCount;
+ strBatch->numElements = rowCount;
+
+ writer->add(*batch);
+ }
+
+ writer->close();
+
+ std::unique_ptr<InputStream> inStream(new MemoryInputStream(
+ memStream.getData(), memStream.getLength()));
+ std::unique_ptr<Reader> reader = createReader(pool, std::move(inStream));
+ std::unique_ptr<RowReader> rowReader = createRowReader(reader.get());
+
+ // make sure there are multiple stripes
+ EXPECT_EQ(rowCount * stripeCount, reader->getNumberOfRows());
+ EXPECT_EQ(stripeCount, reader->getNumberOfStripes());
+
+ // test reading sequentially for data correctness
+ batch = rowReader->createRowBatch(rowCount);
+ for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
+ EXPECT_EQ(true, rowReader->next(*batch));
+
+ for (uint64_t i = 0; i < rowCount; ++i) {
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+ std::string str(strBatch->data[i],
+ static_cast<size_t>(strBatch->length[i]));
+ EXPECT_EQ(i % dictionarySize,
static_cast<uint64_t>(atoi(str.c_str())));
+ }
+ }
+ EXPECT_EQ(false, rowReader->next(*batch));
+
+ // test seeking to check positions
+ batch = rowReader->createRowBatch(1);
+
+ for (uint64_t stripe = 0; stripe != stripeCount; ++stripe) {
+ for (uint64_t i = 0; i < rowCount; i += 10000 / 2) {
+ rowReader->seekToRow(stripe * rowCount + i);
+ EXPECT_EQ(true, rowReader->next(*batch));
+
+ structBatch = dynamic_cast<StructVectorBatch *>(batch.get());
+ strBatch = dynamic_cast<StringVectorBatch *>(structBatch->fields[0]);
+ std::string str(strBatch->data[0],
+ static_cast<size_t>(strBatch->length[0]));
+ EXPECT_EQ(i % dictionarySize,
static_cast<uint64_t>(atoi(str.c_str())));
+ }
+ }
+ }
+
+ // test dictionary encoding with index disabled
+ // the decision of using dictionary if made at the end of 1st stripe
+ TEST(DictionaryEncoding, writeStringDictionaryEncodingWithoutIndex) {
+ testStringDictionary(false, DICT_THRESHOLD);
+ testStringDictionary(false, FALLBACK_THRESHOLD);
+ }
+
+ // test dictionary encoding with index enabled
+ // the decision of using dictionary if made at the end of 1st row group
+ TEST(DictionaryEncoding, writeStringDictionaryEncodingWithIndex) {
+ testStringDictionary(true, DICT_THRESHOLD);
+ testStringDictionary(true, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeVarcharDictionaryEncodingWithoutIndex) {
+ testVarcharDictionary(false, DICT_THRESHOLD);
+ testVarcharDictionary(false, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeVarcharDictionaryEncodingWithIndex) {
+ testVarcharDictionary(true, DICT_THRESHOLD);
+ testVarcharDictionary(true, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeCharDictionaryEncodingWithoutIndex) {
+ testCharDictionary(false, DICT_THRESHOLD);
+ testCharDictionary(false, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, writeCharDictionaryEncodingWithIndex) {
+ testCharDictionary(true, DICT_THRESHOLD);
+ testCharDictionary(true, FALLBACK_THRESHOLD);
+ }
+
+ TEST(DictionaryEncoding, stringDictionaryWithNullWithIndex) {
+ testStringDictionaryWithNull(DICT_THRESHOLD, true);
+ testStringDictionaryWithNull(FALLBACK_THRESHOLD, true);
+ }
+
+ TEST(DictionaryEncoding, stringDictionaryWithNullWithoutIndex) {
+ testStringDictionaryWithNull(DICT_THRESHOLD, false);
+ testStringDictionaryWithNull(FALLBACK_THRESHOLD, false);
+ }
+
+ TEST(DictionaryEncoding, multipleStripesWithIndex) {
+ testDictionaryMultipleStripes(DICT_THRESHOLD, true);
+ testDictionaryMultipleStripes(FALLBACK_THRESHOLD, true);
+ }
+
+ TEST(DictionaryEncoding, multipleStripesWithoutIndex) {
+ testDictionaryMultipleStripes(DICT_THRESHOLD, false);
+ testDictionaryMultipleStripes(FALLBACK_THRESHOLD, false);
+ }
+}
