This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 5f53f13363 Validate BigQuery field names with Locale.ROOT (#9271)
5f53f13363 is described below
commit 5f53f133636336ba5f57e4a6d25de913bd136f66
Author: Sharang Gupta <[email protected]>
AuthorDate: Wed Nov 4 19:28:15 2026 +0000
Validate BigQuery field names with Locale.ROOT (#9271)
Under a Turkish default locale toLowerCase() turns "ID" into "ıd", so
validateBQFieldName rejected valid names and sanitizeJobName replaced
every I with a dash. The test's needsException helper also swallowed
its own HopException, so its negative checks could not fail.
---
.../beam/core/transform/BeamBQOutputTransform.java | 7 ++++---
.../HopPipelineMetaToBeamPipelineConverter.java | 3 ++-
.../core/transform/BeamBQOutputTransformTest.java | 23 +++++++++++++++++-----
.../hop/beam/pipeline/BeamSourceConverterTest.java | 14 +++++++++++++
4 files changed, 38 insertions(+), 9 deletions(-)
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/BeamBQOutputTransform.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/BeamBQOutputTransform.java
index cf7b3677a4..f03876e04b 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/BeamBQOutputTransform.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/BeamBQOutputTransform.java
@@ -23,6 +23,7 @@ import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
import java.util.ArrayList;
import java.util.List;
+import java.util.Locale;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
import org.apache.beam.sdk.metrics.Counter;
import org.apache.beam.sdk.metrics.Metrics;
@@ -201,7 +202,7 @@ public class BeamBQOutputTransform extends
PTransform<PCollection<HopRow>, PDone
throw new HopException(
"A BigQuery field name can not be longer than 300 characters: '" +
name + "'");
}
- String lowerName = name.toLowerCase();
+ String lowerName = name.toLowerCase(Locale.ROOT);
char first = lowerName.charAt(0);
// Starting with a letter or an underscore
if (!((first >= 'a' && first <= 'z') || (first == '_'))) {
@@ -213,7 +214,7 @@ public class BeamBQOutputTransform extends
PTransform<PCollection<HopRow>, PDone
if (lowerName.startsWith(prefix)) {
throw new HopException(
"A BigQuery field name can not start with : "
- + (prefix.toUpperCase())
+ + (prefix.toUpperCase(Locale.ROOT))
+ " for name: '"
+ name
+ "'");
@@ -221,7 +222,7 @@ public class BeamBQOutputTransform extends
PTransform<PCollection<HopRow>, PDone
}
if (name.length() > 1) {
// Validate the other characters as well.
- for (char c : name.substring(1).toLowerCase().toCharArray()) {
+ for (char c : name.substring(1).toLowerCase(Locale.ROOT).toCharArray()) {
// A letter, a digit or an underscore
if (!((c >= 'a' && c <= 'z') || (c == '_') || (c >= '0' && c <= '9')))
{
throw new HopException(
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
index 79d7b784fe..31c2a31f53 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.apache.beam.runners.dataflow.DataflowRunner;
@@ -397,7 +398,7 @@ public class HopPipelineMetaToBeamPipelineConverter {
* @return The sanitezed name without spaces and other special characters
*/
private String sanitizeJobName(String name) {
- String newName = name.toLowerCase();
+ String newName = name.toLowerCase(Locale.ROOT);
if (name.matches("^[0-9].*")) {
newName = "hop-" + newName;
}
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/core/transform/BeamBQOutputTransformTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/core/transform/BeamBQOutputTransformTest.java
index 1f6f6f9681..f6b5091a13 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/core/transform/BeamBQOutputTransformTest.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/core/transform/BeamBQOutputTransformTest.java
@@ -18,7 +18,10 @@
package org.apache.hop.beam.core.transform;
import static
org.apache.hop.beam.core.transform.BeamBQOutputTransform.validateBQFieldName;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.util.Locale;
import org.apache.hop.core.exception.HopException;
import org.junit.jupiter.api.Test;
@@ -53,12 +56,22 @@ class BeamBQOutputTransformTest {
"no Exception thrown: BQ field name only contain letters, digits or
underscore");
}
- public void needsException(String fieldName, String exceptionDescription) {
+ @Test
+ void testValidateBQFieldNameIgnoresDefaultLocale() throws Exception {
+ Locale defaultLocale = Locale.getDefault();
+ Locale.setDefault(Locale.forLanguageTag("tr-TR"));
try {
- validateBQFieldName(fieldName);
- throw new HopException(exceptionDescription);
- } catch (Exception e) {
- // OK!
+ validateBQFieldName("ID");
+ validateBQFieldName("CUSTOMER_ID");
+ HopException prefixError =
+ assertThrows(HopException.class, () ->
validateBQFieldName("_FILE_ID"));
+ assertTrue(prefixError.getMessage().contains("can not start with :
_FILE_"));
+ } finally {
+ Locale.setDefault(defaultLocale);
}
}
+
+ public void needsException(String fieldName, String exceptionDescription) {
+ assertThrows(HopException.class, () -> validateBQFieldName(fieldName),
exceptionDescription);
+ }
}
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/BeamSourceConverterTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/BeamSourceConverterTest.java
index 4152814843..8df4e950c6 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/BeamSourceConverterTest.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/BeamSourceConverterTest.java
@@ -17,6 +17,7 @@
package org.apache.hop.beam.pipeline;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -24,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.values.PCollection;
@@ -130,6 +132,18 @@ class BeamSourceConverterTest {
}
}
+ @Test
+ void jobNameIsLowerCasedIndependentlyOfTheDefaultLocale() throws Exception {
+ Locale defaultLocale = Locale.getDefault();
+ Locale.setDefault(Locale.forLanguageTag("tr-TR"));
+ try {
+ Pipeline graph = convert(pipeline("ID Job", mqttInput("read")));
+ assertEquals("id-job", graph.getOptions().getJobName());
+ } finally {
+ Locale.setDefault(defaultLocale);
+ }
+ }
+
@Test
void informationalHopIsNotARowInput() throws Exception {
PipelineMeta pipeline = new PipelineMeta();