vibhatha commented on code in PR #41646:
URL: https://github.com/apache/arrow/pull/41646#discussion_r1623040761
##########
java/dataset/src/test/java/org/apache/arrow/dataset/substrait/TestAceroSubstraitConsumer.java:
##########
@@ -457,4 +465,42 @@ private static ByteBuffer getByteBuffer(String
base64EncodedSubstrait) {
substraitExpression.put(decodedSubstrait);
return substraitExpression;
}
+
+ @Test
+ public void testCsvConvertOptions() throws Exception {
+ final Schema schema = new Schema(Arrays.asList(
+ Field.nullable("Id", new ArrowType.Int(32, true)),
+ Field.nullable("Name", new ArrowType.Utf8()),
+ Field.nullable("Language", new ArrowType.Utf8())
+ ), null);
+ String path = "file://" + getClass().getResource("/").getPath() +
"/data/student.csv";
+ BufferAllocator allocator = rootAllocator();
+ try (ArrowSchema cSchema = ArrowSchema.allocateNew(allocator);
+ CDataDictionaryProvider provider = new CDataDictionaryProvider()) {
+ Data.exportSchema(allocator, schema, provider, cSchema);
+ CsvConvertOptions convertOptions = new
CsvConvertOptions(ImmutableMap.of("delimiter", ";"));
+ convertOptions.setArrowSchema(cSchema);
+ CsvFragmentScanOptions fragmentScanOptions = new CsvFragmentScanOptions(
+ convertOptions, ImmutableMap.of(), ImmutableMap.of());
+ ScanOptions options = new ScanOptions.Builder(/*batchSize*/ 32768)
+ .columns(Optional.empty())
+ .fragmentScanOptions(fragmentScanOptions)
+ .build();
+ try (
+ DatasetFactory datasetFactory = new
FileSystemDatasetFactory(allocator, NativeMemoryPool.getDefault(),
+ FileFormat.CSV, path);
+ Dataset dataset = datasetFactory.finish();
+ Scanner scanner = dataset.newScan(options);
+ ArrowReader reader = scanner.scanBatches()
+ ) {
+ assertEquals(schema.getFields(),
reader.getVectorSchemaRoot().getSchema().getFields());
+ int rowCount = 0;
+ while (reader.loadNextBatch()) {
+ assertEquals("[1, 2, 3]",
reader.getVectorSchemaRoot().getVector("Id").toString());
Review Comment:
nit: should we check all columns?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
