corgy-w commented on code in PR #9996:
URL: https://github.com/apache/seatunnel/pull/9996#discussion_r2667027311
##########
seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/EmbeddingTransform.java:
##########
@@ -197,29 +197,33 @@ public void open() {
}
private void initOutputFields(SeaTunnelRowType inputRowType,
ReadonlyConfig config) {
- Map<Integer, FieldSpec> fieldSpecMap = new HashMap<>();
- List<String> fieldNames = new ArrayList<>();
Map<String, Object> fieldsConfig =
config.get(EmbeddingTransformConfig.VECTORIZATION_FIELDS);
if (fieldsConfig == null || fieldsConfig.isEmpty()) {
throw new IllegalArgumentException("vectorization_fields
configuration is required");
}
- for (Map.Entry<String, Object> field : fieldsConfig.entrySet()) {
- FieldSpec fieldSpec = new FieldSpec(field);
- log.info("Field spec: {}", fieldSpec.toString());
- String srcField = fieldSpec.getFieldName();
- int srcFieldIndex;
- try {
- srcFieldIndex = inputRowType.indexOf(srcField);
- } catch (IllegalArgumentException e) {
- throw
TransformCommonError.cannotFindInputFieldError(getPluginName(), srcField);
- }
- if (fieldSpec.isMultimodalField()) {
- isMultimodalFields = true;
+ List<String> fieldNames = new ArrayList<>();
+ Map<VectorFieldSpec, List<Integer>> fieldSpecMap = new HashMap<>();
+ for (Map.Entry<String, Object> fieldConfig : fieldsConfig.entrySet()) {
+ VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(fieldConfig);
+ log.info("Vector field spec: {}", vectorFieldSpec);
+ List<String> srcFieldNames =
+ vectorFieldSpec.getSrcFieldSpecs().stream()
+ .map(SrcFieldSpec::getFieldName)
+ .collect(Collectors.toList());
+ List<Integer> srcFieldIndexes = new ArrayList<>();
+ for (String srcFieldName : srcFieldNames) {
+ try {
+ srcFieldIndexes.add(inputRowType.indexOf(srcFieldName));
+ } catch (IllegalArgumentException e) {
+ throw TransformCommonError.cannotFindInputFieldsError(
+ getPluginName(), srcFieldNames);
+ }
}
- fieldSpecMap.put(srcFieldIndex, fieldSpec);
- fieldNames.add(field.getKey());
+ isMultimodalFields = vectorFieldSpec.isMultimodalField();
+ fieldSpecMap.put(vectorFieldSpec, srcFieldIndexes);
+ fieldNames.add(vectorFieldSpec.getFieldName());
Review Comment:
@loupipalien
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
