yanghua commented on a change in pull request #3888: URL: https://github.com/apache/hudi/pull/3888#discussion_r751875479
########## File path: hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestNonIndex.scala ########## @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi + +import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings +import org.apache.hudi.config.{HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.index.HoodieIndex +import org.apache.hudi.testutils.{DataSourceTestUtils, HoodieClientTestBase} +import org.junit.jupiter.api.Test + +import scala.collection.JavaConversions._ +import org.apache.hadoop.fs.Path +import org.apache.hudi.common.fs.FSUtils +import org.apache.hudi.common.model.{HoodieRecord, WriteOperationType} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.keygen.constant.KeyGeneratorOptions +import org.apache.spark.sql.{Dataset, Row, SaveMode} + +import scala.collection.JavaConverters + +class TestNonIndex extends HoodieClientTestBase { + val commonOpts = Map( + "hoodie.insert.shuffle.parallelism" -> "4", + "hoodie.upsert.shuffle.parallelism" -> "4", + KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key() -> "partition", + HoodieIndexConfig.INDEX_TYPE.key() -> HoodieIndex.IndexType.NON_INDEX.name(), + HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key() -> "org.apache.hudi.keygen.EmptyKeyGenerator" + ) + + @Test + def testNonIndexMORInsert(): Unit = { + val spark = sqlContext.sparkSession + + val records1 = recordsToStrings(dataGen.generateInserts("001", 100)).toList + // first insert, parquet files + val inputDF1: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records1, 2)) + inputDF1.write.format("org.apache.hudi") + .options(commonOpts) + .option(DataSourceWriteOptions.OPERATION.key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) + .option(DataSourceWriteOptions.TABLE_TYPE.key(), DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL) + .mode(SaveMode.Overwrite) + .save(basePath) + + // second insert, log files + val records2 = recordsToStrings(dataGen.generateInserts("002", 100)).toList + // first insert, parquet files Review comment: ? ########## File path: hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestEmptyKeyGenerator.java ########## @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.keygen; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.hudi.testutils.KeyGeneratorTestUtilities; +import org.apache.spark.sql.Row; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestEmptyKeyGenerator extends KeyGeneratorTestUtilities { + + private TypedProperties getCommonProps() { + TypedProperties properties = new TypedProperties(); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); + return properties; + } + + private TypedProperties getPropertiesWithoutPartitionPathProp() { + return getCommonProps(); + } + + private TypedProperties getPropertiesWithoutRecordKeyProp() { + TypedProperties properties = getCommonProps(); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); + return properties; + } + + private TypedProperties getWrongPartitionPathFieldProps() { + TypedProperties properties = new TypedProperties(); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "_wrong_partition_path"); + return properties; + } + + private TypedProperties getRecordKeyFieldProps() { + TypedProperties properties = getCommonProps(); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + return properties; + } + + @Test + public void testNullPartitionPathFields() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new EmptyKeyGenerator(getPropertiesWithoutPartitionPathProp())); + } + + @Test + public void testRecordKeyFields() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new EmptyKeyGenerator(getRecordKeyFieldProps())); + } + + @Test + public void testWrongPartitionPathField() { + EmptyKeyGenerator keyGenerator = + new EmptyKeyGenerator(getWrongPartitionPathFieldProps()); + GenericRecord record = getRecord(); + Assertions.assertEquals(KeyGenUtils.HUDI_DEFAULT_PARTITION_PATH, + keyGenerator.getPartitionPath(record)); + Assertions.assertEquals(KeyGenUtils.HUDI_DEFAULT_PARTITION_PATH, + keyGenerator.getPartitionPath(KeyGeneratorTestUtilities.getRow(record))); + } + + @Test + public void testMultiplePartitionKeyGenerator() { Review comment: testMultiplePartitionKeys? ########## File path: hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/EmptyAvroKeyGenerator.java ########## @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.keygen; + +import org.apache.avro.generic.GenericRecord; + +import org.apache.hudi.common.model.HoodieKey; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.keygen.constant.KeyGeneratorOptions; + +import java.util.Collections; +import java.util.List; +import java.util.Arrays; +import java.util.stream.Collectors; + +/** + * Avro key generator for non record key Hudi tables. Review comment: `non` -> `empty`? ########## File path: hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestEmptyKeyGenerator.java ########## @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.keygen; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.hudi.testutils.KeyGeneratorTestUtilities; +import org.apache.spark.sql.Row; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestEmptyKeyGenerator extends KeyGeneratorTestUtilities { + + private TypedProperties getCommonProps() { + TypedProperties properties = new TypedProperties(); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); + return properties; + } + + private TypedProperties getPropertiesWithoutPartitionPathProp() { + return getCommonProps(); + } + + private TypedProperties getPropertiesWithoutRecordKeyProp() { + TypedProperties properties = getCommonProps(); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); + return properties; + } + + private TypedProperties getWrongPartitionPathFieldProps() { + TypedProperties properties = new TypedProperties(); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "_wrong_partition_path"); + return properties; + } + + private TypedProperties getRecordKeyFieldProps() { + TypedProperties properties = getCommonProps(); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + return properties; + } + + @Test + public void testNullPartitionPathFields() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new EmptyKeyGenerator(getPropertiesWithoutPartitionPathProp())); + } + + @Test + public void testRecordKeyFields() { Review comment: testNonEmptyRecordKeyFields? ########## File path: hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BucketInfo.java ########## @@ -48,6 +48,10 @@ public String getPartitionPath() { return partitionPath; } + public void setBucketType(BucketType bucketType) { Review comment: Does the original design make the `BucketInfo` immutable? But now, it has a setter. Can we inject this field from the constructor? ########## File path: hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java ########## @@ -129,6 +129,6 @@ public void close() { } public enum IndexType { - HBASE, INMEMORY, BLOOM, GLOBAL_BLOOM, SIMPLE, GLOBAL_SIMPLE + HBASE, INMEMORY, BLOOM, GLOBAL_BLOOM, SIMPLE, GLOBAL_SIMPLE, NON_INDEX Review comment: seems `NO_INDEX` looks better? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
