This is an automated email from the ASF dual-hosted git repository. vitalii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push: new 41c2dcc DRILL-7995: Add Oracle HDFS lib to query OCI Object Storage (#2311) 41c2dcc is described below commit 41c2dcc498afcbac8aa584c4180445ebf62b34ff Author: Vitalii Diravka <vita...@apache.org> AuthorDate: Mon Sep 6 06:46:28 2021 +0300 DRILL-7995: Add Oracle HDFS lib to query OCI Object Storage (#2311) * DRILL-7995: Add Oracle HDFS lib to query OCI Object Storage - add OCI OS dependency with exclusions - update GlassFish Jersey dependencies version - small refactoring in ViewExpansionContext - add jersey-hk2 dependency (this injection framework was excluded from Jersey in latest versions) - decrease jdbc-all - update default plugins in bootstrap-storage-plugins.json - udpate core-site-example.xml - rename OCI OS Storage plugin "os" -> "oci" --- distribution/pom.xml | 25 +++++++++ .../src/main/resources/core-site-example.xml | 43 ++++++++++++++++ exec/java-exec/pom.xml | 24 +++++---- .../drill/exec/ops/ViewExpansionContext.java | 5 +- .../main/resources/bootstrap-storage-plugins.json | 59 ++++++++++++++++++++++ exec/jdbc-all/pom.xml | 6 ++- exec/jdbc/pom.xml | 2 +- pom.xml | 8 ++- 8 files changed, 157 insertions(+), 15 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 0c5a5fc..7c3e69b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -32,6 +32,7 @@ <properties> <aws.java.sdk.version>1.11.375</aws.java.sdk.version> + <oci.hdfs.version>3.3.0.7.0.1</oci.hdfs.version> </properties> <dependencies> @@ -147,6 +148,30 @@ <version>${aws.java.sdk.version}</version> </dependency> <dependency> + <groupId>com.oracle.oci.sdk</groupId> + <artifactId>oci-hdfs-connector</artifactId> + <version>${oci.hdfs.version}</version> <!--this version corresponds to the Drill Hadoop version--> + <exclusions> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-1.2-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <exclusions> diff --git a/distribution/src/main/resources/core-site-example.xml b/distribution/src/main/resources/core-site-example.xml index 3ce2e29..0261658 100644 --- a/distribution/src/main/resources/core-site-example.xml +++ b/distribution/src/main/resources/core-site-example.xml @@ -130,4 +130,47 @@ </property> --> + <!-- + HDFS Connector for Object Storage: + https://docs.oracle.com/en-us/iaas/Content/API/SDKDocs/hdfsconnector.htm + --> + <!-- + <property> + <name>fs.oci.client.hostname</name> + <value>https://objectstorage.us-ashburn-1.oraclecloud.com</value> + <description> + The URL of the host endpoint. For example, https://www.example.com. + </description> + </property> + <property> + <name>fs.oci.client.auth.tenantId</name> + <value>ocid1.tenancy.oc1..exampleuniqueID</value> + <description> + The OCID of your tenancy. To get the value, see Required Keys and OCIDs: + https://docs.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm + </description> + </property> + <property> + <name>fs.oci.client.auth.userId</name> + <value>ocid1.user.oc1..exampleuniqueID</value> + <description> + The OCID of the user calling the API. To get the value, see Required Keys and OCIDs. + </description> + </property> + <property> + <name>fs.oci.client.auth.fingerprint</name> + <value>20:3b:97:13:55:1c:5b:0d:d3:37:d8:50:4e:c5:3a:34</value> + <description> + The fingerprint for the key pair being used. To get the value, see Required Keys and OCIDs. + </description> + </property> + <property> + <name>fs.oci.client.auth.pemfilepath</name> + <value>/opt/drill/conf/oci_api_key.pem</value> + <description> + The full path and file name of the private key used for authentication. The file should be on the local file system. + </description> + </property> + --> + </configuration> diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index a022c7a..2660f60 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -198,6 +198,20 @@ <artifactId>jersey-media-json-jackson</artifactId> </dependency> <dependency> + <groupId>org.glassfish.jersey.ext</groupId> + <artifactId>jersey-mvc-freemarker</artifactId> + <exclusions> + <exclusion> + <artifactId>servlet-api</artifactId> + <groupId>javax.servlet</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.inject</groupId> + <artifactId>jersey-hk2</artifactId> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.jaxrs</groupId> <artifactId>jackson-jaxrs-json-provider</artifactId> <version>${jackson.version}</version> @@ -212,16 +226,6 @@ <artifactId>jackson-dataformat-hocon</artifactId> </dependency> <dependency> - <groupId>org.glassfish.jersey.ext</groupId> - <artifactId>jersey-mvc-freemarker</artifactId> - <exclusions> - <exclusion> - <artifactId>servlet-api</artifactId> - <groupId>javax.servlet</groupId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>${calcite.groupId}</groupId> <artifactId>calcite-core</artifactId> </dependency> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java index 6b4191c..07fbae3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ViewExpansionContext.java @@ -31,8 +31,9 @@ import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; /** * Contains context information about view expansion(s) in a query. Part of {@link org.apache.drill.exec.ops * .QueryContext}. Before expanding a view into its definition, as part of the - * {@link org.apache.drill.exec.planner.logical.DrillViewTable#toRel(ToRelContext, RelOptTable)}, first a - * {@link ViewExpansionToken} is requested from ViewExpansionContext through {@link #reserveViewExpansionToken(String)}. + * {@link org.apache.drill.exec.planner.logical.DrillViewTable#toRel(org.apache.calcite.plan.RelOptTable.ToRelContext, + * org.apache.calcite.plan.RelOptTable)}, first a {@link ViewExpansionToken} is requested from ViewExpansionContext + * through {@link #reserveViewExpansionToken(String)}. * Once view expansion is complete, a token is released through {@link ViewExpansionToken#release()}. A view definition * itself may contain zero or more views for expanding those nested views also a token is obtained. * diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json index 81f63f2..f171fc2 100644 --- a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json +++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json @@ -111,6 +111,65 @@ }, "enabled" : false }, + "oci" : { + "type" : "file", + "connection": "oci://{bucket_name}@{namespace_name}", + "config" : { + "fs.oci.client.hostname": "HOSTNAME", + "fs.oci.client.auth.tenantId": "TENANT_ID", + "fs.oci.client.auth.userId": "USER_ID", + "fs.oci.client.auth.fingerprint": "FINGERPRINT", + "fs.oci.client.auth.pemfilepath": "PEM_FILEPATH" + }, + "workspaces" : { + "root" : { + "location" : "/", + "writable" : false + }, + "tmp" : { + "location" : "/tmp", + "writable" : true + } + }, + "formats" : { + "psv" : { + "type" : "text", + "extensions" : [ "tbl" ], + "fieldDelimiter" : "|" + }, + "csv" : { + "type" : "text", + "extensions" : [ "csv" ], + "fieldDelimiter" : "," + }, + "tsv" : { + "type" : "text", + "extensions" : [ "tsv" ], + "fieldDelimiter" : "\t" + }, + "parquet" : { + "type" : "parquet" + }, + "json" : { + "type" : "json", + "extensions" : [ "json" ] + }, + "avro" : { + "type" : "avro" + }, + "sequencefile" : { + "type" : "sequencefile", + "extensions" : [ "seq" ] + }, + "csvh" : { + "type" : "text", + "extensions" : [ "csvh" ], + "fieldDelimiter" : ",", + "extractHeader" : true + } + }, + "enabled" : false + }, "cp" : { "type" : "file", "connection" : "classpath:///", diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index 7506d94..a69c4aa 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -81,6 +81,10 @@ <artifactId>commons-codec</artifactId> </exclusion> <exclusion> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-common</artifactId> + </exclusion> + <exclusion> <groupId>org.glassfish.jersey.containers</groupId> <artifactId>jersey-container-jetty-servlet</artifactId> </exclusion> @@ -544,7 +548,7 @@ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users. </message> - <maxsize>46900000</maxsize> + <maxsize>46600000</maxsize> <minsize>15000000</minsize> <files> <file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file> diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml index d6d259d..052ab10 100644 --- a/exec/jdbc/pom.xml +++ b/exec/jdbc/pom.xml @@ -102,7 +102,7 @@ <dependency> <groupId>javax.validation</groupId> <artifactId>validation-api</artifactId> - <version>2.0.1.Final</version> + <version>${javax.validation.api}</version> </dependency> </dependencies> diff --git a/pom.xml b/pom.xml index 00cd1e3..9894041 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,8 @@ <avro.version>1.9.1</avro.version> <metrics.version>4.0.2</metrics.version> <jetty.version>9.4.41.v20210516</jetty.version> - <jersey.version>2.25.1</jersey.version> + <jersey.version>2.34</jersey.version> + <javax.validation.api>2.0.1.Final</javax.validation.api> <asm.version>7.3.1</asm.version> <excludedGroups /> <memoryMb>4000</memoryMb> @@ -2933,6 +2934,11 @@ <artifactId>jersey-container-servlet</artifactId> <version>${jersey.version}</version> </dependency> + <dependency> + <groupId>org.glassfish.jersey.inject</groupId> + <artifactId>jersey-hk2</artifactId> + <version>${jersey.version}</version> + </dependency> <!--/GlassFish Jersey dependecies--> </dependencies> </dependencyManagement>