soumilshah1995 commented on issue #10110:
URL: https://github.com/apache/hudi/issues/10110#issuecomment-1814482504
Here is the entirety of my code, and I'm encountering persistent issues. I
attempted to contact you through a personal message on Slack for more rapid
communication, hoping to expedite the resolution process. Please inform me if
I've made any errors, as I'm still in the early stages of learning and might be
overlooking certain aspects. I'm eager to learn, progress, and eventually
contribute my knowledge to the Hudi community.
```
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, StringType,
TimestampType, FloatType
from datetime import datetime
import os
import sys
HUDI_VERSION = '1.0.0-beta1'
SPARK_VERSION = '3.4'
SUBMIT_ARGS = f"--packages
org.apache.hudi:hudi-spark{SPARK_VERSION}-bundle_2.12:{HUDI_VERSION}
pyspark-shell"
os.environ["PYSPARK_SUBMIT_ARGS"] = SUBMIT_ARGS
os.environ['PYSPARK_PYTHON'] = sys.executable
# Spark session
spark = SparkSession.builder \
.config('spark.serializer',
'org.apache.spark.serializer.KryoSerializer') \
.config('spark.sql.extensions',
'org.apache.spark.sql.hudi.HoodieSparkSessionExtension') \
.config('className', 'org.apache.hudi') \
.config('spark.sql.hive.convertMetastoreParquet', 'false') \
.getOrCreate()
# Sample data
data = [
['2023-09-20 03:58:59', '334e26e9-8355-45cc-97c6-c31daf0df330',
'rider-A', 'driver-K', 19.10, 'san_francisco'],
['2023-09-19 08:46:34', 'e96c4396-3fad-413a-a942-4cb36106d721',
'rider-C', 'driver-M', 27.70, 'san_francisco'],
]
# Define schema for the DataFrame
schema = StructType([
StructField("ts", StringType(), True),
StructField("transaction_id", StringType(), True),
StructField("rider", StringType(), True),
StructField("driver", StringType(), True),
StructField("price", FloatType(), True),
StructField("location", StringType(), True),
])
# Create Spark DataFrame
df = spark.createDataFrame(data, schema=schema)
df.show()
path = 'file:///Users/soumilnitinshah/Downloads/hudidb/hudi_table_func_index'
hudi_options = {
'hoodie.table.name': 'hudi_table_func_index',
'hoodie.datasource.write.table.type': 'COPY_ON_WRITE',
'hoodie.datasource.write.operation': 'upsert',
'hoodie.datasource.write.recordkey.field': 'transaction_id',
'hoodie.datasource.write.precombine.field': 'ts',
'hoodie.table.metadata.enable': 'true',
'hoodie.datasource.write.partitionpath.field': 'location'
}
df.write.format("hudi").options(**hudi_options).mode("append").save(path)
# Register the Hudi table
spark.read.format("hudi").load(path).createOrReplaceTempView("hudi_table_func_index")
# Create the functional index
functional_index_sql = """
CREATE INDEX ts_functional_index
ON hudi_table_func_index(ts)
USING column_stats
OPTIONS (func='from_unixtime', format='yyyy-MM-dd HH:mm:ss');
"""
spark.sql(functional_index_sql)
```
# Error
```
---------------------------------------------------------------------------
ParseException Traceback (most recent call last)
Cell In[2], line 12
4 # Create the functional index
5 functional_index_sql = """
6 CREATE INDEX ts_functional_index
7 ON hudi_table_func_index(ts)
8 USING column_stats
9 OPTIONS (func='from_unixtime', format='yyyy-MM-dd HH:mm:ss');
10 """
---> 12 spark.sql(functional_index_sql)
File ~/anaconda3/lib/python3.11/site-packages/pyspark/sql/session.py:1440,
in SparkSession.sql(self, sqlQuery, args, **kwargs)
1438 try:
1439 litArgs = {k: _to_java_column(lit(v)) for k, v in (args or
{}).items()}
-> 1440 return DataFrame(self._jsparkSession.sql(sqlQuery, litArgs),
self)
1441 finally:
1442 if len(kwargs) > 0:
File ~/anaconda3/lib/python3.11/site-packages/py4j/java_gateway.py:1322, in
JavaMember.__call__(self, *args)
1316 command = proto.CALL_COMMAND_NAME +\
1317 self.command_header +\
1318 args_command +\
1319 proto.END_COMMAND_PART
1321 answer = self.gateway_client.send_command(command)
-> 1322 return_value = get_return_value(
1323 answer, self.gateway_client, self.target_id, self.name)
1325 for temp_arg in temp_args:
1326 if hasattr(temp_arg, "_detach"):
File
~/anaconda3/lib/python3.11/site-packages/pyspark/errors/exceptions/captured.py:175,
in capture_sql_exception.<locals>.deco(*a, **kw)
171 converted = convert_exception(e.java_exception)
172 if not isinstance(converted, UnknownException):
173 # Hide where the exception came from that shows a non-Pythonic
174 # JVM exception message.
--> 175 raise converted from None
176 else:
177 raise
ParseException:
Operation not allowed: CREATE INDEX.(line 2, pos 4)
== SQL ==
CREATE INDEX ts_functional_index
----^^^
ON hudi_table_func_index(ts)
USING column_stats
OPTIONS (func='from_unixtime', format='yyyy-MM-dd HH:mm:ss');
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
