nsivabalan commented on a change in pull request #2128:
URL: https://github.com/apache/hudi/pull/2128#discussion_r499276941
##########
File path:
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadGenerator.java
##########
@@ -333,23 +312,37 @@ private int getSize(Schema elementSchema) {
* @param elementSchema
* @return Number of entries to add
*/
- private int numEntriesToAdd(Schema elementSchema) {
- // Find the size of the primitive data type in bytes
- int primitiveDataTypeSize = getSize(elementSchema);
- int numEntriesToAdd = numberOfBytesToAdd / primitiveDataTypeSize;
- // If more than 10 entries are being added for this same complex field and
there are still more complex fields to
- // be visited in the schema, reduce the number of entries to add by a
factor of 10 to allow for other complex
- // fields to pack some entries
- if (numEntriesToAdd % 10 > 0 && this.numberOfComplexFields > 1) {
- numEntriesToAdd = numEntriesToAdd / 10;
- numberOfBytesToAdd -= numEntriesToAdd * primitiveDataTypeSize;
- this.shouldAddMore = true;
- } else {
- this.numberOfBytesToAdd = 0;
- this.shouldAddMore = false;
+ private void determineExtraEntriesRequired(int numberOfComplexFields, int
numberOfBytesToAdd) {
+ for (Schema.Field f : baseSchema.getFields()) {
+ Schema elementSchema = f.schema();
+ // Find the size of the primitive data type in bytes
+ int primitiveDataTypeSize = 0;
+ if (elementSchema.getType() == Type.ARRAY &&
isPrimitive(elementSchema.getElementType())) {
+ primitiveDataTypeSize = getSize(elementSchema.getElementType());
+ } else if (elementSchema.getType() == Type.MAP &&
isPrimitive(elementSchema.getValueType())) {
+ primitiveDataTypeSize = getSize(elementSchema.getValueType());
+ } else {
+ continue;
+ }
+
+ int numEntriesToAdd = numberOfBytesToAdd / primitiveDataTypeSize;
+ // If more than 10 entries are being added for this same complex field
and there are still more complex fields to
+ // be visited in the schema, reduce the number of entries to add by a
factor of 10 to allow for other complex
+ // fields to pack some entries
+ if (numEntriesToAdd > 10 && numberOfComplexFields > 1) {
Review comment:
what happens if there is only one complexField and numEntriesToAdd is >
10 ?
##########
File path:
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadGenerator.java
##########
@@ -205,45 +193,36 @@ private Object typeConvert(Schema schema) {
case LONG:
return getNextConstrainedLong();
case STRING:
- return UUID.randomUUID().toString();
+ return UUID.randomUUID().toString();
case ENUM:
- List<String> enumSymbols = localSchema.getEnumSymbols();
- return new GenericData.EnumSymbol(localSchema,
enumSymbols.get(random.nextInt(enumSymbols.size() - 1)));
+ List<String> enumSymbols = fieldSchema.getEnumSymbols();
+ return new GenericData.EnumSymbol(fieldSchema,
enumSymbols.get(random.nextInt(enumSymbols.size() - 1)));
case RECORD:
- return convert(localSchema);
+ return getNewPayload(fieldSchema);
case ARRAY:
- Schema elementSchema = localSchema.getElementType();
+ Schema.Field elementField = new Schema.Field(field.name(),
fieldSchema.getElementType(), "", null);
List listRes = new ArrayList();
- if (isPrimitive(elementSchema) && this.shouldAddMore) {
- int numEntriesToAdd = numEntriesToAdd(elementSchema);
- while (numEntriesToAdd > 0) {
- listRes.add(typeConvert(elementSchema));
- numEntriesToAdd--;
- }
- } else {
- listRes.add(typeConvert(elementSchema));
+ int numEntriesToAdd = extraEntriesMap.getOrDefault(field.name(), 1);
+ while (numEntriesToAdd > 0) {
Review comment:
minor. we might as well do while (numEntriesToAdd-- > 0) {
and remove line 208.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
