warren830 commented on code in PR #3513:
URL:
https://github.com/apache/incubator-devlake/pull/3513#discussion_r1143289528
##########
backend/helpers/e2ehelper/data_flow_tester.go:
##########
@@ -506,6 +518,65 @@ func (t *DataFlowTester) VerifyTableWithOptions(dst
schema.Tabler, opts TableOpt
assert.Equal(t.T, expected[field],
formatDbValue(actual[field], opts.Nullable), fmt.Sprintf(`%s.%s not match (with
params from csv %s)`, dst.TableName(), field, pkValues))
}
}
+ err = checkDiversity(dbRows, 3, opts.IgnoreDataDiversityCheckFields...)
+ if err != nil {
+ panic(err)
+ }
assert.Equal(t.T, expectedTotal, actualTotal, fmt.Sprintf(`%s count not
match count,[expected:%d][actual:%d]`, dst.TableName(), expectedTotal,
actualTotal))
}
+
+func checkDiversity(rows *[]map[string]interface{}, minUniqueValues int,
ignoreFieldNames ...string) error {
+ // Check if the input rows is a non-nil pointer to a slice.
+ if rows == nil || reflect.ValueOf(rows).Kind() != reflect.Ptr ||
reflect.ValueOf(rows).Elem().Kind() != reflect.Slice {
+ return errors.Default.New("expected a non-nil pointer to a
slice")
+ }
+
+ // Dereference the pointer to the slice of maps.
+ records := *rows
+
+ // Initialize a map to store unique values for each field.
+ fieldMaps := make(map[string]map[interface{}]bool)
+
+ // Iterate over the records.
+ for _, record := range records {
+ // Iterate over the fields of each record.
+ for field, value := range record {
+ // If ignoreFieldNames are provided, check if the
current field is in the list, if true, skip it.
+ if ignoreFieldNames != nil && len(ignoreFieldNames) > 0
&& contains(ignoreFieldNames, field) {
+ continue
+ }
+
+ // If the map for the current field doesn't exist,
create one.
+ if fieldMaps[field] == nil {
+ fieldMaps[field] = make(map[interface{}]bool)
+ }
+ // Add the current field value to the map.
+ fieldMaps[field][value] = true
+ }
+ }
+
+ // Iterate over the maps for each field and check if the unique value
count meets the requirement.
+ for filedName, fieldMap := range fieldMaps {
+ if len(fieldMap) < minUniqueValues {
+ valueList := make([]interface{}, len(fieldMap))
Review Comment:
Great! Thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
