nsivabalan commented on a change in pull request #1433: URL: https://github.com/apache/incubator-hudi/pull/1433#discussion_r428954499
########## File path: hudi-spark/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java ########## @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.keygen; + +import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.exception.HoodieKeyException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestSimpleKeyGenerator extends TestKeyGeneratorUtilities { + + private TypedProperties getCommonProps() { + TypedProperties properties = new TypedProperties(); + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_row_key"); + properties.put(DataSourceWriteOptions.HIVE_STYLE_PARTITIONING_OPT_KEY(), "true"); + return properties; + } + + private TypedProperties getPropertiesWithoutPartitionPathProp() { + return getCommonProps(); + } + + private TypedProperties getPropertiesWithoutRecordKeyProp() { + TypedProperties properties = new TypedProperties(); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp"); + return properties; + } + + private TypedProperties getWrongRecordKeyFieldProps() { + TypedProperties properties = new TypedProperties(); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp"); + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_wrong_key"); + return properties; + } + + private TypedProperties getComplexRecordKeyProp() { + TypedProperties properties = new TypedProperties(); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp"); + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_row_key,pii_col"); + return properties; + } + + private TypedProperties getProps() { + TypedProperties properties = getCommonProps(); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp"); + return properties; + } + + @Test + public void testNullPartitionPathFields() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new SimpleKeyGenerator(getPropertiesWithoutPartitionPathProp())); + } + + @Test + public void testNullRecordKeyFields() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new SimpleKeyGenerator(getPropertiesWithoutRecordKeyProp())); + } + + @Test + public void testWrongRecordKeyField() { + SimpleKeyGenerator keyGenerator = new SimpleKeyGenerator(getWrongRecordKeyFieldProps()); + Assertions.assertThrows(HoodieKeyException.class, () -> keyGenerator.getRecordKey(getRecord())); + } + + @Test + public void testComplexRecordKeyField() { + SimpleKeyGenerator keyGenerator = new SimpleKeyGenerator(getComplexRecordKeyProp()); + Assertions.assertThrows(HoodieKeyException.class, () -> keyGenerator.getRecordKey(getRecord())); + } + + @Test + public void testHappyFlow() { + SimpleKeyGenerator keyGenerator = new SimpleKeyGenerator(getProps()); + HoodieKey key = keyGenerator.getKey(getRecord()); + Assertions.assertEquals(key.getRecordKey(), "key1"); + Assertions.assertEquals(key.getPartitionPath(), "timestamp=4357686"); + } +} Review comment: here as well ########## File path: hudi-spark/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java ########## @@ -154,4 +153,4 @@ private long convertLongTimeToMillis(Long partitionVal) { return MILLISECONDS.convert(partitionVal, timeUnit); } -} +} Review comment: need a new line ########## File path: hudi-spark/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java ########## @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.keygen; + +import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.config.TypedProperties; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCustomKeyGenerator extends TestKeyGeneratorUtilities { + + private TypedProperties getCommonProps(boolean getComplexRecordKey) { + TypedProperties properties = new TypedProperties(); + if (getComplexRecordKey) { + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_row_key, pii_col"); + } else { + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_row_key"); + } + properties.put(DataSourceWriteOptions.HIVE_STYLE_PARTITIONING_OPT_KEY(), "true"); + return properties; + } + + private TypedProperties getPropertiesForSimpleKeyGen() { + TypedProperties properties = getCommonProps(false); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp:simple"); + return properties; + } + + private TypedProperties getImproperPartitionFieldFormatProp() { + TypedProperties properties = getCommonProps(false); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp"); + return properties; + } + + private TypedProperties getInvalidPartitionKeyTypeProps() { + TypedProperties properties = getCommonProps(false); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp:dummy"); + return properties; + } + + private TypedProperties getComplexRecordKeyWithSimplePartitionProps() { + TypedProperties properties = getCommonProps(true); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp:simple"); + return properties; + } + + private TypedProperties getComplexRecordKeyAndPartitionPathProps() { + TypedProperties properties = getCommonProps(true); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp:simple,ts_ms:timestamp"); + populateNecessaryPropsForTimestampBasedKeyGen(properties); + return properties; + } + + private TypedProperties getPropsWithoutRecordKeyFieldProps() { + TypedProperties properties = new TypedProperties(); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp:simple"); + return properties; + } + + private void populateNecessaryPropsForTimestampBasedKeyGen(TypedProperties properties) { + properties.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING"); + properties.put("hoodie.deltastreamer.keygen.timebased.input.dateformat", "yyyy-MM-dd"); + properties.put("hoodie.deltastreamer.keygen.timebased.output.dateformat", "yyyyMMdd"); + } + + private TypedProperties getPropertiesForTimestampBasedKeyGen() { + TypedProperties properties = getCommonProps(false); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "ts_ms:timestamp"); + populateNecessaryPropsForTimestampBasedKeyGen(properties); + return properties; + } + + private TypedProperties getPropertiesForNonPartitionedKeyGen() { + TypedProperties properties = getCommonProps(false); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), ""); + return properties; + } + + @Test + public void testSimpleKeyGenerator() { + KeyGenerator keyGenerator = new CustomKeyGenerator(getPropertiesForSimpleKeyGen()); + HoodieKey key = keyGenerator.getKey(getRecord()); + Assertions.assertEquals(key.getRecordKey(), "key1"); + Assertions.assertEquals(key.getPartitionPath(), "timestamp=4357686"); + } + + @Test + public void testTimestampBasedKeyGenerator() { + KeyGenerator keyGenerator = new CustomKeyGenerator(getPropertiesForTimestampBasedKeyGen()); + HoodieKey key = keyGenerator.getKey(getRecord()); + Assertions.assertEquals(key.getRecordKey(), "key1"); + Assertions.assertEquals(key.getPartitionPath(), "ts_ms=20200321"); + } + + @Test + public void testNonPartitionedKeyGenerator() { + KeyGenerator keyGenerator = new CustomKeyGenerator(getPropertiesForNonPartitionedKeyGen()); + HoodieKey key = keyGenerator.getKey(getRecord()); + Assertions.assertEquals(key.getRecordKey(), "key1"); + Assertions.assertTrue(key.getPartitionPath().isEmpty()); + } + + @Test + public void testInvalidPartitionKeyType() { + try { + KeyGenerator keyGenerator = new CustomKeyGenerator(getInvalidPartitionKeyTypeProps()); + keyGenerator.getKey(getRecord()); + Assertions.fail("should fail when invalid PartitionKeyType is provided!"); + } catch (Exception e) { + Assertions.assertTrue(e.getMessage().contains("No enum constant org.apache.hudi.keygen.CustomKeyGenerator.PartitionKeyType.DUMMY")); + } + } + + @Test + public void testNoRecordKeyFieldProp() { + try { + KeyGenerator keyGenerator = new CustomKeyGenerator(getPropsWithoutRecordKeyFieldProps()); + keyGenerator.getKey(getRecord()); + Assertions.fail("should fail when record key field is not provided!"); + } catch (Exception e) { + Assertions.assertTrue(e.getMessage().contains("Property hoodie.datasource.write.recordkey.field not found")); + } + } + + @Test + public void testPartitionFieldsInImproperFormat() { + try { + KeyGenerator keyGenerator = new CustomKeyGenerator(getImproperPartitionFieldFormatProp()); + keyGenerator.getKey(getRecord()); + Assertions.fail("should fail when partition key field is provided in improper format!"); + } catch (Exception e) { + Assertions.assertTrue(e.getMessage().contains("Unable to find field names for partition path in proper format")); + } + } + + @Test + public void testComplexRecordKeyWithSimplePartitionPath() { + KeyGenerator keyGenerator = new CustomKeyGenerator(getComplexRecordKeyWithSimplePartitionProps()); + HoodieKey key = keyGenerator.getKey(getRecord()); + Assertions.assertEquals(key.getRecordKey(), "_row_key:key1,pii_col:pi"); + Assertions.assertEquals(key.getPartitionPath(), "timestamp=4357686"); + } + + @Test + public void testComplexRecordKeysWithComplexPartitionPath() { + KeyGenerator keyGenerator = new CustomKeyGenerator(getComplexRecordKeyAndPartitionPathProps()); + HoodieKey key = keyGenerator.getKey(getRecord()); + Assertions.assertEquals(key.getRecordKey(), "_row_key:key1,pii_col:pi"); + Assertions.assertEquals(key.getPartitionPath(), "timestamp=4357686/ts_ms=20200321"); + } +} Review comment: same here ########## File path: hudi-spark/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java ########## @@ -95,4 +95,4 @@ public void testScalar() { HoodieKey hk5 = new TimestampBasedKeyGenerator(properties).getKey(baseRecord); assertEquals(hk5.getPartitionPath(), "2024-10-04 12"); } -} +} Review comment: sam here. ########## File path: hudi-spark/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.keygen; + +import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.exception.HoodieKeyException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestComplexKeyGenerator extends TestKeyGeneratorUtilities { + + private TypedProperties getCommonProps(boolean getComplexRecordKey) { + TypedProperties properties = new TypedProperties(); + if (getComplexRecordKey) { + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_row_key, pii_col"); + } else { + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_row_key"); + } + properties.put(DataSourceWriteOptions.HIVE_STYLE_PARTITIONING_OPT_KEY(), "true"); + return properties; + } + + private TypedProperties getPropertiesWithoutPartitionPathProp() { + return getCommonProps(false); + } + + private TypedProperties getPropertiesWithoutRecordKeyProp() { + TypedProperties properties = new TypedProperties(); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp"); + return properties; + } + + private TypedProperties getWrongRecordKeyFieldProps() { + TypedProperties properties = new TypedProperties(); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp"); + properties.put(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_wrong_key"); + return properties; + } + + private TypedProperties getProps() { + TypedProperties properties = getCommonProps(true); + properties.put(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY(), "timestamp,ts_ms"); + return properties; + } + + @Test + public void testNullPartitionPathFields() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new ComplexKeyGenerator(getPropertiesWithoutPartitionPathProp())); + } + + @Test + public void testNullRecordKeyFields() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new ComplexKeyGenerator(getPropertiesWithoutRecordKeyProp())); + } + + @Test + public void testWrongRecordKeyField() { + ComplexKeyGenerator keyGenerator = new ComplexKeyGenerator(getWrongRecordKeyFieldProps()); + Assertions.assertThrows(HoodieKeyException.class, () -> keyGenerator.getRecordKey(getRecord())); + } + + @Test + public void testHappyFlow() { + ComplexKeyGenerator keyGenerator = new ComplexKeyGenerator(getProps()); + HoodieKey key = keyGenerator.getKey(getRecord()); + Assertions.assertEquals(key.getRecordKey(), "_row_key:key1,pii_col:pi"); + Assertions.assertEquals(key.getPartitionPath(), "timestamp=4357686/ts_ms=2020-03-21"); + } +} Review comment: same here ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
