Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228010100
--- Diff:
store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---
@@ -1522,4 +1522,204 @@ public boolean accept(File dir, String name) {
e.printStackTrace();
}
}
+
+ @Test
+ public void testReadNextRowWithRowUtil() {
+ String path = "./carbondata";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[12];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("intField", DataTypes.INT);
+ fields[3] = new Field("longField", DataTypes.LONG);
+ fields[4] = new Field("doubleField", DataTypes.DOUBLE);
+ fields[5] = new Field("boolField", DataTypes.BOOLEAN);
+ fields[6] = new Field("dateField", DataTypes.DATE);
+ fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[8] = new Field("decimalField", DataTypes.createDecimalType(8,
2));
+ fields[9] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[10] = new Field("arrayField",
DataTypes.createArrayType(DataTypes.STRING));
+ fields[11] = new Field("floatField", DataTypes.FLOAT);
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields)).build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ String.valueOf(i),
+ String.valueOf(Long.MAX_VALUE - i),
+ String.valueOf((double) i / 2),
+ String.valueOf(true),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "12.345",
+ "varchar",
+ "Hello#World#From#Carbon",
+ "1.23"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbonindex");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon index file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchemaInIndexFile(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ int count = 0;
+ for (int i = 0; i < schema.getFields().length; i++) {
+ if (!((schema.getFields())[i].getFieldName().contains("."))) {
+ count++;
+ }
+ }
+ String[] strings = new String[count];
+ int index = 0;
+ for (int i = 0; i < schema.getFields().length; i++) {
+ if (!((schema.getFields())[i].getFieldName().contains("."))) {
+ strings[index] = (schema.getFields())[i].getFieldName();
+ index++;
+ }
+ }
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .projection(strings)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] data = (Object[]) reader.readNextRow();
+
+ assert (RowUtil.getString(data, 0).equals("robot" + i));
+ assertEquals(RowUtil.getShort(data, 1), i);
+ assertEquals(RowUtil.getInt(data, 2), i);
+ assertEquals(RowUtil.getLong(data, 3), Long.MAX_VALUE - i);
+ assertEquals(RowUtil.getDouble(data, 4), ((double) i) / 2);
+ assert (RowUtil.getBoolean(data, 5));
+ assertEquals(RowUtil.getInt(data, 6), 17957);
+ assert (RowUtil.getDecimal(data, 8).equals("12.35"));
+ assert (RowUtil.getVarchar(data, 9).equals("varchar"));
+
+ Object[] arr = RowUtil.getArray(data, 10);
+ assert (arr[0].equals("Hello"));
+ assert (arr[1].equals("World"));
+ assert (arr[2].equals("From"));
+ assert (arr[3].equals("Carbon"));
+
+ assertEquals(RowUtil.getFloat(data, 11), (float) 1.23);
+ i++;
+ }
+ reader.close();
+ } catch (Throwable e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Test
+ public void testReadNextRowWithProjectionAndRowUtil() {
+ String path = "./carbondata";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[12];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("intField", DataTypes.INT);
+ fields[3] = new Field("longField", DataTypes.LONG);
+ fields[4] = new Field("doubleField", DataTypes.DOUBLE);
+ fields[5] = new Field("boolField", DataTypes.BOOLEAN);
+ fields[6] = new Field("dateField", DataTypes.DATE);
+ fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[8] = new Field("decimalField", DataTypes.createDecimalType(8,
2));
+ fields[9] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[10] = new Field("arrayField",
DataTypes.createArrayType(DataTypes.STRING));
+ fields[11] = new Field("floatField", DataTypes.FLOAT);
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields)).build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ String.valueOf(i),
+ String.valueOf(Long.MAX_VALUE - i),
+ String.valueOf((double) i / 2),
+ String.valueOf(true),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "12.345",
+ "varchar",
+ "Hello#World#From#Carbon",
+ "1.23"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] data = (Object[]) reader.readNextRow();
+
+ assert (RowUtil.getString(data, 0).equals("robot" + i));
+ assertEquals(RowUtil.getInt(data, 1), 17957);
+ assert (RowUtil.getVarchar(data, 3).equals("varchar"));
+ Object[] arr = RowUtil.getArray(data, 4);
+ assert (arr[0].equals("Hello"));
+ assert (arr[1].equals("World"));
+ assert (arr[2].equals("From"));
+ assert (arr[3].equals("Carbon"));
+ assertEquals(RowUtil.getShort(data, 5), i);
+ assertEquals(RowUtil.getInt(data, 6), i);
+ assertEquals(RowUtil.getLong(data, 7), Long.MAX_VALUE - i);
+ assertEquals(RowUtil.getDouble(data, 8), ((double) i) / 2);
+ assert (RowUtil.getBoolean(data, 9));
+ assert (RowUtil.getDecimal(data, 10).equals("12.35"));
+ assertEquals(RowUtil.getFloat(data, 11), (float) 1.23);
+ i++;
+ }
+ reader.close();
+ } catch (Throwable e) {
+ e.printStackTrace();
--- End diff --
ok, done
---
