jackye1995 commented on a change in pull request #3275:
URL: https://github.com/apache/iceberg/pull/3275#discussion_r727346630
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
##########
@@ -107,14 +108,27 @@ private void initializeCatalogTables() throws
InterruptedException, SQLException
LOG.trace("Creating database tables (if missing) to store iceberg
catalog");
connections.run(conn -> {
DatabaseMetaData dbMeta = conn.getMetaData();
- ResultSet tableExists = dbMeta.getTables(null, null,
JdbcUtil.CATALOG_TABLE_NAME, null);
+ ResultSet tableExists = dbMeta.getTables(null, null,
JdbcCatalogTable.CATALOG_TABLE_NAME, null);
if (tableExists.next()) {
return true;
}
- LOG.debug("Creating table {} to store iceberg catalog",
JdbcUtil.CATALOG_TABLE_NAME);
- return conn.prepareStatement(JdbcUtil.CREATE_CATALOG_TABLE).execute();
+ LOG.debug("Creating table {} to store iceberg catalog tables",
JdbcCatalogTable.CATALOG_TABLE_NAME);
+ return
conn.prepareStatement(JdbcCatalogTable.CREATE_CATALOG_TABLE).execute();
+ });
+ connections.run(conn -> {
Review comment:
nit: newline after control statement like if/for
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalogNamespace.java
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.jdbc;
+
+public class JdbcCatalogNamespace {
Review comment:
My current opinion is that there is no need to separate `JdbcUtil` into
`JdbcCatalogNamespace` and `JdbcCatalogTable`. All these static variables can
just continue to live under `JdbcUtil`. That would make this PR much cleaner.
What would be the benefits of separating into 2 classes?
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java
##########
@@ -99,4 +65,26 @@ public static Properties filterAndRemovePrefix(Map<String,
String> properties,
return result;
}
+
+ public static Map<String, String> convertJsonStringToMap(String jsonString) {
Review comment:
I wonder if we can just use `SerializationUtil.serializeToBase64`.
Maybe there is some benefit in using a JSON map for readability purpose. In
that case, use `JsonUtil.mapper()` instead of initializing a mapper here.
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java
##########
@@ -99,4 +65,26 @@ public static Properties filterAndRemovePrefix(Map<String,
String> properties,
return result;
}
+
+ public static Map<String, String> convertJsonStringToMap(String jsonString) {
+ Map<String, String> resultMap;
+ try {
+ resultMap = objectMapper.readValue(
+ jsonString, new TypeReference<HashMap<String, String>>() {
+ });
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
Review comment:
throw new UncheckedIOException(e);
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
##########
@@ -113,9 +114,23 @@ private void initializeCatalogTables() throws
InterruptedException, SQLException
return true;
}
- LOG.debug("Creating table {} to store iceberg catalog",
JdbcUtil.CATALOG_TABLE_NAME);
+ LOG.debug("Creating table {} to store iceberg catalog tables",
JdbcUtil.CATALOG_TABLE_NAME);
return conn.prepareStatement(JdbcUtil.CREATE_CATALOG_TABLE).execute();
});
+
+ connections.run(conn -> {
+ DatabaseMetaData dbMeta = conn.getMetaData();
+ ResultSet tableExists = dbMeta.getTables(null, null,
+ JdbcUtil.CATALOG_NAMESPACE_TABLE_NAME, null);
Review comment:
nit: do we need newline here?
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java
##########
@@ -27,48 +31,91 @@
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.base.Splitter;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.util.JsonUtil;
final class JdbcUtil {
+ // Catalog Table
protected static final String CATALOG_TABLE_NAME = "iceberg_tables";
protected static final String CATALOG_NAME = "catalog_name";
protected static final String TABLE_NAMESPACE = "table_namespace";
protected static final String TABLE_NAME = "table_name";
protected static final String METADATA_LOCATION = "metadata_location";
protected static final String PREVIOUS_METADATA_LOCATION =
"previous_metadata_location";
+
public static final String DO_COMMIT_SQL = "UPDATE " + CATALOG_TABLE_NAME +
- " SET " + METADATA_LOCATION + " = ? , " + PREVIOUS_METADATA_LOCATION + "
= ? " +
- " WHERE " + CATALOG_NAME + " = ? AND " +
- TABLE_NAMESPACE + " = ? AND " +
- TABLE_NAME + " = ? AND " +
- METADATA_LOCATION + " = ?";
+ " SET " + METADATA_LOCATION + " = ? , " + PREVIOUS_METADATA_LOCATION
+ " = ? " +
Review comment:
nit: looks like spacings are all off? newline in the same sentence
should be 4 spaces.
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java
##########
@@ -27,48 +31,91 @@
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.base.Splitter;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.util.JsonUtil;
final class JdbcUtil {
+ // Catalog Table
protected static final String CATALOG_TABLE_NAME = "iceberg_tables";
protected static final String CATALOG_NAME = "catalog_name";
protected static final String TABLE_NAMESPACE = "table_namespace";
protected static final String TABLE_NAME = "table_name";
protected static final String METADATA_LOCATION = "metadata_location";
protected static final String PREVIOUS_METADATA_LOCATION =
"previous_metadata_location";
+
public static final String DO_COMMIT_SQL = "UPDATE " + CATALOG_TABLE_NAME +
- " SET " + METADATA_LOCATION + " = ? , " + PREVIOUS_METADATA_LOCATION + "
= ? " +
- " WHERE " + CATALOG_NAME + " = ? AND " +
- TABLE_NAMESPACE + " = ? AND " +
- TABLE_NAME + " = ? AND " +
- METADATA_LOCATION + " = ?";
+ " SET " + METADATA_LOCATION + " = ? , " + PREVIOUS_METADATA_LOCATION
+ " = ? " +
+ " WHERE " + CATALOG_NAME + " = ? AND " +
+ TABLE_NAMESPACE + " = ? AND " +
+ TABLE_NAME + " = ? AND " +
+ METADATA_LOCATION + " = ?";
protected static final String CREATE_CATALOG_TABLE =
- "CREATE TABLE " + CATALOG_TABLE_NAME +
- "(" +
- CATALOG_NAME + " VARCHAR(255) NOT NULL," +
- TABLE_NAMESPACE + " VARCHAR(255) NOT NULL," +
- TABLE_NAME + " VARCHAR(255) NOT NULL," +
- METADATA_LOCATION + " VARCHAR(5500)," +
- PREVIOUS_METADATA_LOCATION + " VARCHAR(5500)," +
- "PRIMARY KEY (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ", " +
TABLE_NAME + ")" +
- ")";
+ "CREATE TABLE " + CATALOG_TABLE_NAME +
+ "(" +
+ CATALOG_NAME + " VARCHAR(255) NOT NULL," +
+ TABLE_NAMESPACE + " VARCHAR(255) NOT NULL," +
+ TABLE_NAME + " VARCHAR(255) NOT NULL," +
+ METADATA_LOCATION + " VARCHAR(5500)," +
+ PREVIOUS_METADATA_LOCATION + " VARCHAR(5500)," +
+ "PRIMARY KEY (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ",
" + TABLE_NAME + ")" +
+ ")";
protected static final String GET_TABLE_SQL = "SELECT * FROM " +
CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND " +
TABLE_NAME + " = ? ";
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND
" + TABLE_NAME + " = ? ";
protected static final String LIST_TABLES_SQL = "SELECT * FROM " +
CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ?";
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ?";
protected static final String RENAME_TABLE_SQL = "UPDATE " +
CATALOG_TABLE_NAME +
- " SET " + TABLE_NAMESPACE + " = ? , " + TABLE_NAME + " = ? " +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND " +
TABLE_NAME + " = ? ";
+ " SET " + TABLE_NAMESPACE + " = ? , " + TABLE_NAME + " = ? " +
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND
" + TABLE_NAME + " = ? ";
protected static final String DROP_TABLE_SQL = "DELETE FROM " +
CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND " +
TABLE_NAME + " = ? ";
- protected static final String GET_NAMESPACE_SQL = "SELECT " +
TABLE_NAMESPACE + " FROM " + CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " LIKE ?
LIMIT 1";
- protected static final String LIST_NAMESPACES_SQL = "SELECT DISTINCT " +
TABLE_NAMESPACE +
- " FROM " + CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " LIKE ?";
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND
" + TABLE_NAME + " = ? ";
+
protected static final String DO_COMMIT_CREATE_TABLE_SQL = "INSERT INTO " +
CATALOG_TABLE_NAME +
- " (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ", " + TABLE_NAME +
- ", " + METADATA_LOCATION + ", " + PREVIOUS_METADATA_LOCATION + ") " +
- " VALUES (?,?,?,?,null)";
+ " (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ", " + TABLE_NAME +
+ ", " + METADATA_LOCATION + ", " + PREVIOUS_METADATA_LOCATION + ") " +
+ " VALUES (?,?,?,?,null)";
+
+ // Catalog Namespace
+
+ protected static final String CATALOG_NAMESPACE_TABLE_NAME =
"iceberg_namespaces";
+ protected static final String NAMESPACE_NAME = "namespace_name";
+ protected static final String NAMESPACE_LOCATION = "location";
+ protected static final String NAMESPACE_METADATA = "metadata";
+ protected static final String NAMESPACE_PROPERTIES = "properties";
+
+ protected static final String CREATE_NAMESPACE_TABLE =
+ "CREATE TABLE " + CATALOG_NAMESPACE_TABLE_NAME +
+ "(" +
+ CATALOG_NAME + " VARCHAR(255) NOT NULL," +
+ NAMESPACE_NAME + " VARCHAR(255) NOT NULL," +
+ NAMESPACE_LOCATION + " VARCHAR(5500)," +
Review comment:
I think we don't need a specific location field, it can just be a part
of the metadata. Hive has a specific `LocationUri` field in its `Database`
model, that's why location is handled differently, but for this catalog I think
we don't need that.
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcTableOperations.java
##########
@@ -207,7 +209,8 @@ protected String tableName() {
table.put(JdbcUtil.TABLE_NAMESPACE,
rs.getString(JdbcUtil.TABLE_NAMESPACE));
table.put(JdbcUtil.TABLE_NAME, rs.getString(JdbcUtil.TABLE_NAME));
table.put(JdbcUtil.METADATA_LOCATION,
rs.getString(JdbcUtil.METADATA_LOCATION));
- table.put(JdbcUtil.PREVIOUS_METADATA_LOCATION,
rs.getString(JdbcUtil.PREVIOUS_METADATA_LOCATION));
+ table.put(JdbcUtil.PREVIOUS_METADATA_LOCATION,
+ rs.getString(JdbcUtil.PREVIOUS_METADATA_LOCATION));
Review comment:
nit: no need for newline
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
##########
@@ -252,8 +267,34 @@ public void setConf(Configuration conf) {
@Override
public void createNamespace(Namespace namespace, Map<String, String>
metadata) {
- throw new UnsupportedOperationException("Cannot create namespace " +
namespace +
- ": createNamespace is not supported");
+ if (namespaceExists(namespace)) {
+ throw new AlreadyExistsException("Namespace already exists: %s",
namespace);
+ }
+ String namespaceName = JdbcUtil.namespaceToString(namespace);
Review comment:
nit: newline after if
##########
File path: core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java
##########
@@ -27,48 +31,91 @@
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.base.Splitter;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.util.JsonUtil;
final class JdbcUtil {
+ // Catalog Table
protected static final String CATALOG_TABLE_NAME = "iceberg_tables";
protected static final String CATALOG_NAME = "catalog_name";
protected static final String TABLE_NAMESPACE = "table_namespace";
protected static final String TABLE_NAME = "table_name";
protected static final String METADATA_LOCATION = "metadata_location";
protected static final String PREVIOUS_METADATA_LOCATION =
"previous_metadata_location";
+
public static final String DO_COMMIT_SQL = "UPDATE " + CATALOG_TABLE_NAME +
- " SET " + METADATA_LOCATION + " = ? , " + PREVIOUS_METADATA_LOCATION + "
= ? " +
- " WHERE " + CATALOG_NAME + " = ? AND " +
- TABLE_NAMESPACE + " = ? AND " +
- TABLE_NAME + " = ? AND " +
- METADATA_LOCATION + " = ?";
+ " SET " + METADATA_LOCATION + " = ? , " + PREVIOUS_METADATA_LOCATION
+ " = ? " +
+ " WHERE " + CATALOG_NAME + " = ? AND " +
+ TABLE_NAMESPACE + " = ? AND " +
+ TABLE_NAME + " = ? AND " +
+ METADATA_LOCATION + " = ?";
protected static final String CREATE_CATALOG_TABLE =
- "CREATE TABLE " + CATALOG_TABLE_NAME +
- "(" +
- CATALOG_NAME + " VARCHAR(255) NOT NULL," +
- TABLE_NAMESPACE + " VARCHAR(255) NOT NULL," +
- TABLE_NAME + " VARCHAR(255) NOT NULL," +
- METADATA_LOCATION + " VARCHAR(5500)," +
- PREVIOUS_METADATA_LOCATION + " VARCHAR(5500)," +
- "PRIMARY KEY (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ", " +
TABLE_NAME + ")" +
- ")";
+ "CREATE TABLE " + CATALOG_TABLE_NAME +
+ "(" +
+ CATALOG_NAME + " VARCHAR(255) NOT NULL," +
+ TABLE_NAMESPACE + " VARCHAR(255) NOT NULL," +
+ TABLE_NAME + " VARCHAR(255) NOT NULL," +
+ METADATA_LOCATION + " VARCHAR(5500)," +
+ PREVIOUS_METADATA_LOCATION + " VARCHAR(5500)," +
+ "PRIMARY KEY (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ",
" + TABLE_NAME + ")" +
+ ")";
protected static final String GET_TABLE_SQL = "SELECT * FROM " +
CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND " +
TABLE_NAME + " = ? ";
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND
" + TABLE_NAME + " = ? ";
protected static final String LIST_TABLES_SQL = "SELECT * FROM " +
CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ?";
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ?";
protected static final String RENAME_TABLE_SQL = "UPDATE " +
CATALOG_TABLE_NAME +
- " SET " + TABLE_NAMESPACE + " = ? , " + TABLE_NAME + " = ? " +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND " +
TABLE_NAME + " = ? ";
+ " SET " + TABLE_NAMESPACE + " = ? , " + TABLE_NAME + " = ? " +
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND
" + TABLE_NAME + " = ? ";
protected static final String DROP_TABLE_SQL = "DELETE FROM " +
CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND " +
TABLE_NAME + " = ? ";
- protected static final String GET_NAMESPACE_SQL = "SELECT " +
TABLE_NAMESPACE + " FROM " + CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " LIKE ?
LIMIT 1";
- protected static final String LIST_NAMESPACES_SQL = "SELECT DISTINCT " +
TABLE_NAMESPACE +
- " FROM " + CATALOG_TABLE_NAME +
- " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " LIKE ?";
+ " WHERE " + CATALOG_NAME + " = ? AND " + TABLE_NAMESPACE + " = ? AND
" + TABLE_NAME + " = ? ";
+
protected static final String DO_COMMIT_CREATE_TABLE_SQL = "INSERT INTO " +
CATALOG_TABLE_NAME +
- " (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ", " + TABLE_NAME +
- ", " + METADATA_LOCATION + ", " + PREVIOUS_METADATA_LOCATION + ") " +
- " VALUES (?,?,?,?,null)";
+ " (" + CATALOG_NAME + ", " + TABLE_NAMESPACE + ", " + TABLE_NAME +
+ ", " + METADATA_LOCATION + ", " + PREVIOUS_METADATA_LOCATION + ") " +
+ " VALUES (?,?,?,?,null)";
+
+ // Catalog Namespace
+
+ protected static final String CATALOG_NAMESPACE_TABLE_NAME =
"iceberg_namespaces";
+ protected static final String NAMESPACE_NAME = "namespace_name";
+ protected static final String NAMESPACE_LOCATION = "location";
+ protected static final String NAMESPACE_METADATA = "metadata";
+ protected static final String NAMESPACE_PROPERTIES = "properties";
+
+ protected static final String CREATE_NAMESPACE_TABLE =
+ "CREATE TABLE " + CATALOG_NAMESPACE_TABLE_NAME +
+ "(" +
+ CATALOG_NAME + " VARCHAR(255) NOT NULL," +
+ NAMESPACE_NAME + " VARCHAR(255) NOT NULL," +
+ NAMESPACE_LOCATION + " VARCHAR(5500)," +
+ NAMESPACE_METADATA + " VARCHAR(65535)," +
+ NAMESPACE_PROPERTIES + " VARCHAR(65535)," +
Review comment:
what's the difference of metadata and properties?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
