Repository: cassandra Updated Branches: refs/heads/cassandra-3.11 ebdf77194 -> 5f54d4210 refs/heads/trunk 393226de6 -> 691189616
Add charset to Analyser input stream Patch by Alex Petrov; reviewed by Jay Zhuang for CASSANDRA-13151 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5f54d421 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5f54d421 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5f54d421 Branch: refs/heads/cassandra-3.11 Commit: 5f54d42107b0e0ece0ebb94a285cd7957b608523 Parents: ebdf771 Author: Alex Petrov <[email protected]> Authored: Fri Apr 7 11:26:31 2017 +0200 Committer: Alex Petrov <[email protected]> Committed: Fri Apr 7 11:26:31 2017 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/index/sasi/analyzer/StandardAnalyzer.java | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/5f54d421/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 0761ca5..7b03b6c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.11.0 + * Add charset to Analyser input stream (CASSANDRA-13151) * Fix testLimitSSTables flake caused by concurrent flush (CASSANDRA-12820) * cdc column addition strikes again (CASSANDRA-13382) * Fix static column indexes (CASSANDRA-13277) http://git-wip-us.apache.org/repos/asf/cassandra/blob/5f54d421/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java index 069164c..3b58bf9 100644 --- a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java +++ b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Map; import org.apache.cassandra.index.sasi.analyzer.filter.*; @@ -152,7 +153,7 @@ public class StandardAnalyzer extends AbstractAnalyzer this.options = tokenizerOptions; this.filterPipeline = getFilterPipeline(); - Reader reader = new InputStreamReader(new DataInputBuffer(ByteBufferUtil.EMPTY_BYTE_BUFFER, false)); + Reader reader = new InputStreamReader(new DataInputBuffer(ByteBufferUtil.EMPTY_BYTE_BUFFER, false), StandardCharsets.UTF_8); this.scanner = new StandardTokenizerImpl(reader); this.inputReader = reader; } @@ -179,15 +180,16 @@ public class StandardAnalyzer extends AbstractAnalyzer public void reset(ByteBuffer input) { this.next = null; - Reader reader = new InputStreamReader(new DataInputBuffer(input, false)); + Reader reader = new InputStreamReader(new DataInputBuffer(input, false), StandardCharsets.UTF_8); scanner.yyreset(reader); this.inputReader = reader; } + @VisibleForTesting public void reset(InputStream input) { this.next = null; - Reader reader = new InputStreamReader(input); + Reader reader = new InputStreamReader(input, StandardCharsets.UTF_8); scanner.yyreset(reader); this.inputReader = reader; }
