[ 
https://issues.apache.org/jira/browse/ARROW-2226?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16380894#comment-16380894
 ] 

ASF GitHub Bot commented on ARROW-2226:
---------------------------------------

TheNeuralBit closed pull request #1671: ARROW-2226, ARROW-2233: [JS] Dictionary 
bugfixes
URL: https://github.com/apache/arrow/pull/1671
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/js/src/data.ts b/js/src/data.ts
index cdd9f29a7..3bfb3209b 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -152,10 +152,9 @@ export class DictionaryData<T extends DataType> extends 
BaseData<Dictionary<T>>
     public get indices() { return this._indices; }
     public get dictionary() { return this._dictionary; }
     constructor(type: Dictionary<T>, dictionary: Vector<T>, indices: 
Data<Int<any>>) {
-        super(type, indices.length, (indices as any)._nullCount);
+        super(type, indices.length, indices.offset, (indices as 
any)._nullCount);
         this._indices = indices;
         this._dictionary = dictionary;
-        this.length = this._indices.length;
     }
     public get nullCount() { return this._indices.nullCount; }
     public get nullBitmap() { return this._indices.nullBitmap; }
diff --git a/js/src/vector.ts b/js/src/vector.ts
index f36c691e1..6c2bbbb86 100644
--- a/js/src/vector.ts
+++ b/js/src/vector.ts
@@ -399,6 +399,9 @@ export class DictionaryVector<T extends DataType = 
DataType> extends Vector<Dict
     public readonly dictionary: Vector<T>;
     constructor(data: Data<Dictionary<T>>, view: View<Dictionary<T>> = new 
DictionaryView<T>(data.dictionary, new IntVector(data.indices))) {
         super(data as Data<any>, view);
+        if (view instanceof ValidityView) {
+            view = (view as any).view;
+        }
         if (data instanceof DictionaryData && view instanceof DictionaryView) {
             this.indices = view.indices;
             this.dictionary = data.dictionary;
diff --git a/js/test/unit/vector-tests.ts b/js/test/unit/vector-tests.ts
index e2be22983..3eb3fbe01 100644
--- a/js/test/unit/vector-tests.ts
+++ b/js/test/unit/vector-tests.ts
@@ -17,14 +17,15 @@
 
 import { TextEncoder } from 'text-encoding-utf-8';
 import Arrow from '../Arrow';
-import { type, TypedArray, TypedArrayConstructor } from '../../src/Arrow';
+import { type, TypedArray, TypedArrayConstructor, Vector } from 
'../../src/Arrow';
+import { packBools } from '../../src/util/bit'
 
 const utf8Encoder = new TextEncoder('utf-8');
 
-const { BoolData, FlatData, FlatListData } = Arrow.data;
-const { IntVector, FloatVector, BoolVector, Utf8Vector } = Arrow.vector;
+const { BoolData, FlatData, FlatListData, DictionaryData } = Arrow.data;
+const { IntVector, FloatVector, BoolVector, Utf8Vector, DictionaryVector } = 
Arrow.vector;
 const {
-    Utf8, Bool,
+    Dictionary, Utf8, Bool,
     Float16, Float32, Float64,
     Int8, Int16, Int32, Int64,
     Uint8, Uint16, Uint32, Uint64,
@@ -310,6 +311,54 @@ describe(`Utf8Vector`, () => {
     let offset = 0;
     const offsets = Uint32Array.of(0, ...values.map((d) => { offset += 
d.length; return offset; }));
     const vector = new Utf8Vector(new FlatListData(new Utf8(), n, null, 
offsets, utf8Encoder.encode(values.join(''))));
+    basicVectorTests(vector, values, ['abc', '123']);
+    describe(`sliced`, () => {
+        basicVectorTests(vector.slice(1,3), values.slice(1,3), ['foo', 'abc']);
+    });
+});
+
+describe(`DictionaryVector`, () => {
+    const dictionary = ['foo', 'bar', 'baz'];
+    const extras = ['abc', '123']; // values to search for that should NOT be 
found
+    let offset = 0;
+    const offsets = Uint32Array.of(0, ...dictionary.map((d) => { offset += 
d.length; return offset; }));
+    const dictionary_vec = new Utf8Vector(new FlatListData(new Utf8(), 
dictionary.length, null, offsets, utf8Encoder.encode(dictionary.join(''))));
+
+    const indices = Array.from({length: 50}, () => Math.random() * 3 | 0);
+
+    describe(`index with nullCount == 0`, () => {
+        const indices_data = new FlatData(new Int32(), indices.length, new 
Uint8Array(0), indices);
+
+        const values = Array.from(indices).map((d) => dictionary[d]);
+        const vector = new DictionaryVector(new DictionaryData(new 
Dictionary(dictionary_vec.type, indices_data.type), dictionary_vec, 
indices_data));
+
+        basicVectorTests(vector, values, extras);
+
+        describe(`sliced`, () => {
+            basicVectorTests(vector.slice(10, 20), values.slice(10,20), 
extras);
+        })
+    });
+
+    describe(`index with nullCount > 0`, () => {
+        const validity = Array.from({length: indices.length}, () => 
Math.random() > 0.2 ? true : false);
+        const indices_data = new FlatData(new Int32(), indices.length, 
packBools(validity), indices, 0, validity.reduce((acc, d) => acc + (d ? 0 : 1), 
0));
+        const values = Array.from(indices).map((d, i) => validity[i] ? 
dictionary[d] : null);
+        const vector = new DictionaryVector(new DictionaryData(new 
Dictionary(dictionary_vec.type, indices_data.type), dictionary_vec, 
indices_data));
+
+        basicVectorTests(vector, values, ['abc', '123']);
+        describe(`sliced`, () => {
+            basicVectorTests(vector.slice(10, 20), values.slice(10,20), 
extras);
+        });
+    });
+});
+
+// Creates some basic tests for the given vector.
+// Verifies that:
+// - `get` and the native iterator return the same data as `values`
+// - `indexOf` returns the same indices as `values`
+function basicVectorTests(vector: Vector, values: any[], extras: any[]) {
+    const n = values.length;
+
     test(`gets expected values`, () => {
         let i = -1;
         while (++i < n) {
@@ -325,14 +374,14 @@ describe(`Utf8Vector`, () => {
         }
     });
     test(`indexOf returns expected values`, () => {
-        let testValues = values.concat(['abc', '12345']);
+        let testValues = values.concat(extras);
 
         for (const value of testValues) {
             const expected = values.indexOf(value);
             expect(vector.indexOf(value)).toEqual(expected);
         }
     });
-});
+}
 
 function toMap<T>(entries: Record<string, T>, keys: string[]) {
     return keys.reduce((map, key) => {


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[email protected]


> [JS] DictionaryData should use indices' offset in constructor
> -------------------------------------------------------------
>
>                 Key: ARROW-2226
>                 URL: https://issues.apache.org/jira/browse/ARROW-2226
>             Project: Apache Arrow
>          Issue Type: Bug
>          Components: JavaScript
>    Affects Versions: JS-0.3.0
>            Reporter: Paul Taylor
>            Assignee: Paul Taylor
>            Priority: Major
>              Labels: pull-request-available
>             Fix For: JS-0.4.0
>
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to