likyh commented on code in PR #3513:
URL:
https://github.com/apache/incubator-devlake/pull/3513#discussion_r1142983670
##########
backend/helpers/e2ehelper/data_flow_tester.go:
##########
@@ -509,3 +523,54 @@ func (t *DataFlowTester) VerifyTableWithOptions(dst
schema.Tabler, opts TableOpt
assert.Equal(t.T, expectedTotal, actualTotal, fmt.Sprintf(`%s count not
match count,[expected:%d][actual:%d]`, dst.TableName(), expectedTotal,
actualTotal))
}
+
+func checkDiversity(rows *[]map[string]interface{}, minUniqueValues int,
fieldNames ...string) error {
+ // Check if the input rows is a non-nil pointer to a slice.
+ if rows == nil || reflect.ValueOf(rows).Kind() != reflect.Ptr ||
reflect.ValueOf(rows).Elem().Kind() != reflect.Slice {
+ return errors.Default.New("expected a non-nil pointer to a
slice")
+ }
+
+ // Dereference the pointer to the slice of maps.
+ records := *rows
+
+ // Initialize a map to store unique values for each field.
+ fieldMaps := make(map[string]map[interface{}]bool)
+
+ // Iterate over the records.
+ for _, record := range records {
+ // Iterate over the fields of each record.
+ for field, value := range record {
+ // If specific field names are provided, check if the
current field is in the list.
+ if len(fieldNames) > 0 && !contains(fieldNames, field) {
+ continue
+ }
+
+ // If the map for the current field doesn't exist,
create one.
+ if fieldMaps[field] == nil {
+ fieldMaps[field] = make(map[interface{}]bool)
+ }
+ // Add the current field value to the map.
+ fieldMaps[field][value] = true
+ }
+ }
+
+ // Iterate over the maps for each field and check if the unique value
count meets the requirement.
+ for _, fieldMap := range fieldMaps {
+ if len(fieldMap) < minUniqueValues {
+ return errors.Default.New("data diversity check failed")
Review Comment:
add more detail messages in this error. Such as which column, existing
values...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
