This is an automated email from the ASF dual-hosted git repository.
xuzifu666 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new 843f468405 [CALCITE-7733] Validate identifiers and escape LOAD path
when generating Pig Latin scripts
843f468405 is described below
commit 843f4684059f3878995cfb3b7705841c95d971b9
Author: Yu Xu <[email protected]>
AuthorDate: Wed Oct 7 23:28:50 2026 +0800
[CALCITE-7733] Validate identifiers and escape LOAD path when generating
Pig Latin scripts
---
.../apache/calcite/adapter/pig/PigAggregate.java | 13 +++-
.../org/apache/calcite/adapter/pig/PigFilter.java | 2 +-
.../org/apache/calcite/adapter/pig/PigRel.java | 12 ++-
.../apache/calcite/adapter/pig/PigTableScan.java | 7 +-
.../org/apache/calcite/adapter/pig/PigUtils.java | 88 +++++++++++++++++++++
.../apache/calcite/adapter/pig/PigUtilsTest.java | 90 ++++++++++++++++++++++
.../org/apache/calcite/test/AbstractPigTest.java | 14 ++++
.../org/apache/calcite/test/PigAdapterTest.java | 62 ++++++++++++---
pig/src/test/resources/data'quote.txt | 2 +
pig/src/test/resources/model.json | 27 +++++++
10 files changed, 299 insertions(+), 18 deletions(-)
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
index 742c82816b..8250ae32cd 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigAggregate.java
@@ -22,7 +22,6 @@
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.pig.scripting.Pig;
@@ -96,14 +95,13 @@ private String getPigAggregateStatement(Implementor
implementor) {
*/
private String getPigGroupBy(Implementor implementor) {
final String relAlias = implementor.getPigRelationAlias(this);
- final List<RelDataTypeField> allFields =
getInput().getRowType().getFieldList();
final List<Integer> groupedFieldIndexes = groupSet.asList();
if (groupedFieldIndexes.size() < 1) {
return relAlias + " = GROUP " + relAlias + " ALL;";
} else {
final List<String> groupedFieldNames = new
ArrayList<>(groupedFieldIndexes.size());
for (int fieldIndex : groupedFieldIndexes) {
- groupedFieldNames.add(allFields.get(fieldIndex).getName());
+ groupedFieldNames.add(getInputFieldName(fieldIndex));
}
return relAlias + " = GROUP " + relAlias + " BY ("
+ String.join(", ", groupedFieldNames) + ");";
@@ -156,6 +154,9 @@ private List<String> getPigAggregateCalls(Implementor
implementor) {
private String getPigAggregateCall(String relAlias, AggregateCall aggCall) {
final PigAggFunction aggFunc = toPigAggFunc(aggCall);
final String alias = aggCall.getName();
+ // The alias is used after AS, where it cannot be quoted or escaped in
+ // Pig Latin
+ PigUtils.checkValidIdentifier(alias);
final String fields = String.join(", ", getArgNames(relAlias, aggCall));
return aggFunc.name() + "(" + fields + ") AS " + alias;
}
@@ -212,6 +213,10 @@ private List<String> getDistinctCalls(Implementor
implementor) {
}
private String getInputFieldName(int fieldIndex) {
- return getInput().getRowType().getFieldList().get(fieldIndex).getName();
+ final String fieldName =
+ getInput().getRowType().getFieldList().get(fieldIndex).getName();
+ // Field names cannot be quoted or escaped in Pig Latin
+ PigUtils.checkValidIdentifier(fieldName);
+ return fieldName;
}
}
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
index cc6a8a8c0f..c57ae3f5ff 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigFilter.java
@@ -139,7 +139,7 @@ private static String getLiteralAsString(RexLiteral
literal) {
// Escape before wrapping
return '\''
+ (raw != null
- ? raw.replace("\\", "\\\\").replace("'", "\\'")
+ ? PigUtils.escapeStringLiteral(raw)
: null)
+ '\'';
}
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
index 6776b47a70..3941cf29e3 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigRel.java
@@ -56,7 +56,11 @@ class Implementor {
public String getTableName(RelNode input) {
final List<String> qualifiedName = input.getTable().getQualifiedName();
- return qualifiedName.get(qualifiedName.size() - 1);
+ final String tableName = qualifiedName.get(qualifiedName.size() - 1);
+ // The table name is used as a relation alias, which cannot be quoted
+ // or escaped in Pig Latin
+ PigUtils.checkValidIdentifier(tableName);
+ return tableName;
}
public String getPigRelationAlias(RelNode input) {
@@ -64,7 +68,11 @@ public String getPigRelationAlias(RelNode input) {
}
public String getFieldName(RelNode input, int index) {
- return input.getRowType().getFieldList().get(index).getName();
+ final String fieldName =
+ input.getRowType().getFieldList().get(index).getName();
+ // Field names cannot be quoted or escaped in Pig Latin
+ PigUtils.checkValidIdentifier(fieldName);
+ return fieldName;
}
public void addStatement(String statement) {
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableScan.java
b/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableScan.java
index d68a424307..6ae89c9b0f 100644
--- a/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableScan.java
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigTableScan.java
@@ -51,8 +51,11 @@ public PigTableScan(RelOptCluster cluster, RelTraitSet
traitSet, RelOptTable tab
final String alias = implementor.getPigRelationAlias(this);
final String schema = '(' + getSchemaForPigStatement(implementor)
+ ')';
- final String statement = alias + " = LOAD '" + pigTable.getFilePath()
- + "' USING PigStorage() AS " + schema + ';';
+ // Escape the file path so that quotes or backslashes in it (e.g. in a
+ // Windows path) cannot break out of the string literal
+ final String statement = alias + " = LOAD "
+ + PigUtils.quoteStringLiteral(pigTable.getFilePath())
+ + " USING PigStorage() AS " + schema + ';';
implementor.addStatement(statement);
}
diff --git a/pig/src/main/java/org/apache/calcite/adapter/pig/PigUtils.java
b/pig/src/main/java/org/apache/calcite/adapter/pig/PigUtils.java
new file mode 100644
index 0000000000..8a6e6ab0fe
--- /dev/null
+++ b/pig/src/main/java/org/apache/calcite/adapter/pig/PigUtils.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.pig;
+
+import java.util.regex.Pattern;
+
+/**
+ * Utility methods for generating Pig Latin scripts.
+ *
+ * <p>Unlike SQL, Pig Latin has no mechanism to quote identifiers, so names
+ * that do not match its identifier syntax cannot be represented in a script
+ * and must be rejected; see
+ * <a href="https://pig.apache.org/docs/r0.18.0/basic.html#identifiers";>
+ * Pig Latin identifiers</a>.
+ */
+public final class PigUtils {
+
+ /** Private constructor to prevent instantiation. */
+ private PigUtils() {
+ }
+
+ /**
+ * Matches a valid Pig Latin identifier: an ASCII letter followed by any
+ * number of letters, digits or underscores.
+ */
+ private static final Pattern IDENTIFIER =
+ Pattern.compile("[A-Za-z][A-Za-z0-9_]*");
+
+ /**
+ * Validates that the given name is a valid Pig Latin identifier, throwing
+ * if it is not.
+ *
+ * <p>Identifiers are used in generated scripts as relation aliases, field
+ * names and {@code AS} aliases, where they cannot be quoted or escaped;
+ * an invalid name would produce a script that Pig cannot parse.
+ *
+ * @param name Name to validate
+ * @throws IllegalArgumentException if {@code name} is not a valid Pig Latin
+ * identifier
+ */
+ public static void checkValidIdentifier(String name) {
+ if (!IDENTIFIER.matcher(name).matches()) {
+ throw new IllegalArgumentException(
+ "Invalid Pig Latin identifier: '" + name + "'. Pig Latin identifiers"
+ + " must start with a letter, followed by letters, digits or"
+ + " underscores");
+ }
+ }
+
+ /**
+ * Escapes the given string for use inside a Pig Latin string literal.
+ *
+ * <p>Backslashes and single quotes are escaped with a backslash, so that
+ * the value cannot break out of the {@code '...'} literal or be
re-interpreted
+ * as a Pig Latin escape sequence such as {@code \t} or {@code \u0001}.
+ *
+ * @param s String to escape
+ * @return the escaped string
+ */
+ public static String escapeStringLiteral(String s) {
+ return s.replace("\\", "\\\\").replace("'", "\\'");
+ }
+
+ /**
+ * Converts the given string to a Pig Latin string literal, escaping it
+ * first and then wrapping it in single quotes.
+ *
+ * @param s String to quote
+ * @return the quoted and escaped string literal
+ */
+ public static String quoteStringLiteral(String s) {
+ return "'" + escapeStringLiteral(s) + "'";
+ }
+}
diff --git a/pig/src/test/java/org/apache/calcite/adapter/pig/PigUtilsTest.java
b/pig/src/test/java/org/apache/calcite/adapter/pig/PigUtilsTest.java
new file mode 100644
index 0000000000..e26bd97b1a
--- /dev/null
+++ b/pig/src/test/java/org/apache/calcite/adapter/pig/PigUtilsTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.pig;
+
+import org.junit.jupiter.api.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Unit tests for {@link PigUtils}.
+ */
+class PigUtilsTest {
+
+ @Test void validIdentifiersAccepted() {
+ // Examples of valid identifiers from the Pig Latin reference
+ for (String name : new String[] {"A", "A123", "abc_123_BeX_", "t", "tc0"})
{
+ assertDoesNotThrow(() -> PigUtils.checkValidIdentifier(name),
+ () -> "expected '" + name + "' to be accepted");
+ }
+ }
+
+ @Test void invalidIdentifiersRejected() {
+ // Examples of invalid identifiers from the Pig Latin reference, plus
+ // names that would be mis-parsed as Pig Latin operators if passed through
+ for (String name : new String[] {"_A123", "abc_$", "A!B", "1abc", "",
+ "a b", "a-b", "a;b", "a=b", "a.b", "EXPR$0"}) {
+ final IllegalArgumentException e =
+ assertThrows(IllegalArgumentException.class,
+ () -> PigUtils.checkValidIdentifier(name),
+ () -> "expected '" + name + "' to be rejected");
+ assertThat(
+ e.getMessage(), is("Invalid Pig Latin identifier: '" + name
+ + "'. Pig Latin identifiers must start with a letter, followed by"
+ + " letters, digits or underscores"));
+ }
+ }
+
+ @Test void plainValueQuoted() {
+ assertThat(PigUtils.quoteStringLiteral("alice"), is("'alice'"));
+ }
+
+ @Test void valueWithBackslash() {
+ assertThat(PigUtils.quoteStringLiteral("a\\b"), is("'a\\\\b'"));
+ }
+
+ @Test void valueWithBackslashBeforeApostrophe() {
+ assertThat(PigUtils.quoteStringLiteral("a\\'b"), is("'a\\\\\\'b'"));
+ }
+
+ @Test void valueWithTrailingBackslash() {
+ assertThat(PigUtils.quoteStringLiteral("a\\"), is("'a\\\\'"));
+ }
+
+ @Test void valueWithApostrophe() {
+ assertThat(PigUtils.quoteStringLiteral("O'Brien"), is("'O\\'Brien'"));
+ }
+
+ @Test void valueWithApostropheAtTheEnd() {
+ assertThat(PigUtils.quoteStringLiteral("a'"), is("'a\\''"));
+ }
+
+ @Test void valueWithApostropheAtTheStart() {
+ assertThat(PigUtils.quoteStringLiteral("'a"), is("'\\'a'"));
+ }
+
+ @Test void valueWithMultipleApostrophes() {
+ assertThat(PigUtils.quoteStringLiteral("a''b"), is("'a\\'\\'b'"));
+ }
+
+ @Test void emptyValue() {
+ assertThat(PigUtils.quoteStringLiteral(""), is("''"));
+ }
+}
diff --git a/pig/src/test/java/org/apache/calcite/test/AbstractPigTest.java
b/pig/src/test/java/org/apache/calcite/test/AbstractPigTest.java
index e3abc3072b..4b7f651df2 100644
--- a/pig/src/test/java/org/apache/calcite/test/AbstractPigTest.java
+++ b/pig/src/test/java/org/apache/calcite/test/AbstractPigTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.calcite.test;
+import org.apache.calcite.adapter.pig.PigUtils;
import org.apache.calcite.util.Sources;
import java.net.URL;
@@ -32,4 +33,17 @@ protected String getFullPathForTestDataFile(String fileName)
{
requireNonNull(url, "url");
return Sources.of(url).file().getAbsolutePath();
}
+
+ /**
+ * Returns the full path of the given test data file, escaped for inclusion
+ * in a Pig Latin string literal, as in the {@code LOAD} statement of an
+ * expected script.
+ *
+ * <p>Paths are platform-dependent (a Windows path contains backslashes), so
+ * expected scripts must build them with the same escaping as
+ * {@link PigUtils} uses when generating scripts.
+ */
+ protected String getEscapedFullPathForTestDataFile(String fileName) {
+ return PigUtils.escapeStringLiteral(getFullPathForTestDataFile(fileName));
+ }
}
diff --git a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
index 18a61e468b..08c3178174 100644
--- a/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
+++ b/pig/src/test/java/org/apache/calcite/test/PigAdapterTest.java
@@ -52,7 +52,7 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
+ "t = FILTER t BY (tc0 > 'abc');"));
}
@@ -67,11 +67,55 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
+ "t = FILTER t BY (tc0 == 'a\\'b');"));
}
+ @Test void testLoadPathWithSingleQuote() {
+ // A LOAD path containing a single quote must be escaped per Pig Latin
+ // string-literal rules so it does not break out of the '...' literal in
+ // the generated LOAD statement
+ CalciteAssert.that()
+ .with(MODEL)
+ .query("select * from \"tq\"")
+ .runs()
+ .queryContains(
+ pigScriptChecker("tq = LOAD '"
+ + getEscapedFullPathForTestDataFile("data'quote.txt")
+ + "' USING PigStorage() AS (tqc0:chararray,
tqc1:chararray);"));
+ }
+
+ @Test void testInvalidTableName() {
+ // A table name is used as a relation alias, which cannot be quoted or
+ // escaped in Pig Latin, so script generation is rejected instead of
+ // producing a script that Pig cannot parse
+ CalciteAssert.that()
+ .with(MODEL)
+ .query("select * from \"bad-name\"")
+ .throws_("Invalid Pig Latin identifier: 'bad-name'");
+ }
+
+ @Test void testInvalidColumnName() {
+ // Field names cannot be quoted or escaped in Pig Latin, so script
+ // generation is rejected instead of producing a script that Pig cannot
+ // parse
+ CalciteAssert.that()
+ .with(MODEL)
+ .query("select * from \"badcolumn\"")
+ .throws_("Invalid Pig Latin identifier: 'bad$column'");
+ }
+
+ @Test void testUnaliasedAggregateCall() {
+ // 'EXPR$0' (the name Calcite derives for an unaliased aggregate call) is
+ // not a valid Pig Latin identifier and cannot be used after AS; the query
+ // has to be re-written with an explicit alias instead
+ CalciteAssert.that()
+ .with(MODEL)
+ .query("select count(\"tc0\") from \"t\"")
+ .throws_("Invalid Pig Latin identifier: 'EXPR$0'");
+ }
+
@Test void testImplWithMultipleFilters() {
CalciteAssert.that()
.with(MODEL)
@@ -82,7 +126,7 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
+ "t = FILTER t BY (tc0 > 'abc') AND (tc1 == '3');"));
}
@@ -97,7 +141,7 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
+ "t = GROUP t BY (tc0);\n"
+ "t = FOREACH t {\n"
@@ -115,7 +159,7 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
+ "t = GROUP t ALL;\n"
+ "t = FOREACH t {\n"
@@ -133,7 +177,7 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
+ "t = GROUP t BY (tc0, tc1);\n"
+ "t = FOREACH t {\n"
@@ -151,7 +195,7 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
+ "t = GROUP t BY (tc1);\n"
+ "t = FOREACH t {\n"
@@ -173,9 +217,9 @@ class PigAdapterTest extends AbstractPigTest {
.runs()
.queryContains(
pigScriptChecker("t = LOAD '"
- + getFullPathForTestDataFile("data.txt")
+ + getEscapedFullPathForTestDataFile("data.txt")
+ "' USING PigStorage() AS (tc0:chararray, tc1:chararray);\n"
- + "s = LOAD '" + getFullPathForTestDataFile("data2.txt")
+ + "s = LOAD '" + getEscapedFullPathForTestDataFile("data2.txt")
+ "' USING PigStorage() AS (sc0:chararray, sc1:chararray);\n"
+ "t = JOIN t BY tc1 , s BY sc0;"));
}
diff --git a/pig/src/test/resources/data'quote.txt
b/pig/src/test/resources/data'quote.txt
new file mode 100644
index 0000000000..2398dad264
--- /dev/null
+++ b/pig/src/test/resources/data'quote.txt
@@ -0,0 +1,2 @@
+a 1
+b 2
diff --git a/pig/src/test/resources/model.json
b/pig/src/test/resources/model.json
index 851888e9af..05bd5a9b66 100644
--- a/pig/src/test/resources/model.json
+++ b/pig/src/test/resources/model.json
@@ -40,6 +40,33 @@
"file": "data2.txt",
"columns": ["sc0", "sc1"]
}
+ },
+ {
+ "name": "tq",
+ "type": "custom",
+ "factory": "org.apache.calcite.adapter.pig.PigTableFactory",
+ "operand": {
+ "file": "data'quote.txt",
+ "columns": ["tqc0", "tqc1"]
+ }
+ },
+ {
+ "name": "bad-name",
+ "type": "custom",
+ "factory": "org.apache.calcite.adapter.pig.PigTableFactory",
+ "operand": {
+ "file": "data.txt",
+ "columns": ["bc0"]
+ }
+ },
+ {
+ "name": "badcolumn",
+ "type": "custom",
+ "factory": "org.apache.calcite.adapter.pig.PigTableFactory",
+ "operand": {
+ "file": "data.txt",
+ "columns": ["bad$column"]
+ }
}
]
}
