kbendick commented on code in PR #5305:
URL: https://github.com/apache/iceberg/pull/5305#discussion_r927003817
##########
spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/BaseCatalog.java:
##########
@@ -46,4 +54,56 @@ public Procedure loadProcedure(Identifier ident) throws
NoSuchProcedureException
throw new NoSuchProcedureException(ident);
}
+
+ @Override
+ public Identifier[] listFunctions(String[] namespace) throws
NoSuchNamespaceException {
+ if (isValidNamespaceForFunction(namespace)) {
+ return Arrays.stream(SparkFunctions.list())
+ .map(name -> Identifier.of(namespace, name))
+ .toArray(Identifier[]::new);
+ } else {
+ throw new NoSuchNamespaceException(namespace);
+ }
+ }
+
+ @Override
+ public UnboundFunction loadFunction(Identifier ident) throws
NoSuchFunctionException {
+ String[] namespace = ident.namespace();
+ String name = ident.name();
+
+ if (isValidNamespaceForFunction(namespace)) {
+ UnboundFunction func = SparkFunctions.load(name);
+ if (func != null) {
+ return func;
+ }
+ }
+
+ throw new NoSuchFunctionException(ident);
+ }
+
+ /**
+ * When in an Iceberg catalog, allow for using the built-in Iceberg
functions provided that:
+ * <ol>
+ * <li>
+ * The namespace is not specified,
+ * i.e. {@code SELECT truncate(1, 4)} or {@code SELECT
my_catalog.truncate(1, 4)}
+ * </li>
+ * <li>
+ * The implicit <b>system</b> namespace is used, to mirror call
procedure syntax,
+ * i.e. {@code SELECT system.truncate(1, 4)} or {@code SELECT
my_catalog.system.truncate(1, 4)}
+ * </li>
+ * <li>
+ * A namespace that exists within the catalog is referenced,
+ * i.e. {@code SELECT ns.truncate(1, 4)} or {@code SELECT
my_catalog.ns.truncate(1, 4)}
Review Comment:
Full stack trace:
```
Undefined function: 'truncate'. This function is neither a registered
temporary function nor a permanent function registered in the database
'system'.; line 1 pos 7
org.apache.spark.sql.AnalysisException: Undefined function: 'truncate'. This
function is neither a registered temporary function nor a permanent function
registered in the database 'system'.; line 1 pos 7
at
app//org.apache.spark.sql.catalyst.catalog.SessionCatalog.failFunctionLookup(SessionCatalog.scala:1561)
at
app//org.apache.spark.sql.catalyst.catalog.SessionCatalog.resolvePersistentFunctionInternal(SessionCatalog.scala:1704)
at
app//org.apache.spark.sql.catalyst.catalog.SessionCatalog.resolvePersistentFunction(SessionCatalog.scala:1673)
at
app//org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$.org$apache$spark$sql$catalyst$analysis$Analyzer$ResolveFunctions$$resolveV1Function(Analyzer.scala:2172)
at
app//org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$25$$anonfun$applyOrElse$103.$anonfun$applyOrElse$109(Analyzer.scala:2122)
at app//scala.Option.getOrElse(Option.scala:189)
at
app//org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$25$$anonfun$applyOrElse$103.$anonfun$applyOrElse$108(Analyzer.scala:2119)
```
Link to the code in the analyzer that's being hit:
https://github.com/apache/spark/blob/f74867bddfbcdd4d08076db36851e88b15e66556/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala#L2118-L2126
##########
spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/functions/TruncateFunction.java:
##########
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+import org.apache.iceberg.spark.SparkSchemaUtil;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.iceberg.util.UnicodeUtil;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.functions.BoundFunction;
+import org.apache.spark.sql.connector.catalog.functions.ScalarFunction;
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.DecimalType;
+import org.apache.spark.sql.types.IntegerType;
+import org.apache.spark.sql.types.LongType;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.unsafe.types.UTF8String;
+
+/**
+ * Implmenetation of {@link UnboundFunction} that matches the {@code truncate}
transformation.
+ * The various implementations are registered with the {@link
org.apache.iceberg.spark.SparkCatalog}
+ * such that the function can be used as {@code truncate(col, width)} or
{@code truncate(col, 2)}.
+ */
+public class TruncateFunction implements UnboundFunction {
+
+ public TruncateFunction() {
Review Comment:
Omitted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
