-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/26963/#review61857
-----------------------------------------------------------
I have a couple of high level comments and an exception.
It seems like the LinkConfiguration and the ToJobConfiguration are holding the
wrong pieces? For example, the dataset URI should be part of the
LinkConfiguration and the data format (Avro, CSV, etc.) should be part of the
job? Or perhaps they should all be part of the link configuration?
Also, I think there should be a validation step for the dataset for HDFS. We
should check for "hdfs" type and require a host. It's fine for now for this to
be a runtime exception, but let's log a separate Jira?
Exception has occurred during processing command
Exception: org.apache.sqoop.common.SqoopException Message: CLIENT_0001:Server
has returned exception
Stack trace:
at org.apache.sqoop.client.request.ResourceRequest
(ResourceRequest.java:115)
at org.apache.sqoop.client.request.ResourceRequest
(ResourceRequest.java:148)
at org.apache.sqoop.client.request.JobResourceRequest
(JobResourceRequest.java:63)
at org.apache.sqoop.client.request.SqoopResourceRequests
(SqoopResourceRequests.java:116)
at org.apache.sqoop.client.SqoopClient (SqoopClient.java:406)
at org.apache.sqoop.shell.CreateJobFunction
(CreateJobFunction.java:99)
at org.apache.sqoop.shell.CreateJobFunction
(CreateJobFunction.java:64)
at org.apache.sqoop.shell.SqoopFunction (SqoopFunction.java:51)
at org.apache.sqoop.shell.SqoopCommand (SqoopCommand.java:127)
at org.apache.sqoop.shell.SqoopCommand (SqoopCommand.java:103)
at org.codehaus.groovy.tools.shell.Command$execute (null:-1)
at org.codehaus.groovy.tools.shell.Shell (Shell.groovy:101)
at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:-1)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:-2)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl
(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method (Method.java:606)
at org.codehaus.groovy.reflection.CachedMethod (CachedMethod.java:90)
at groovy.lang.MetaMethod (MetaMethod.java:233)
at groovy.lang.MetaClassImpl (MetaClassImpl.java:1054)
at org.codehaus.groovy.runtime.ScriptBytecodeAdapter
(ScriptBytecodeAdapter.java:128)
at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:173)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:-2)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl
(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method (Method.java:606)
at
org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite$PogoCachedMethodSiteNoUnwrapNoCoerce
(PogoMetaMethodSite.java:267)
at org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite
(PogoMetaMethodSite.java:52)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite
(AbstractCallSite.java:141)
at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:121)
at org.codehaus.groovy.tools.shell.Shell (Shell.groovy:114)
at org.codehaus.groovy.tools.shell.Shell$leftShift$0 (null:-1)
at org.codehaus.groovy.tools.shell.ShellRunner
(ShellRunner.groovy:88)
at org.codehaus.groovy.tools.shell.InteractiveShellRunner
(InteractiveShellRunner.groovy:-1)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:-2)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl
(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method (Method.java:606)
at org.codehaus.groovy.reflection.CachedMethod (CachedMethod.java:90)
at groovy.lang.MetaMethod (MetaMethod.java:233)
at groovy.lang.MetaClassImpl (MetaClassImpl.java:1054)
at org.codehaus.groovy.runtime.ScriptBytecodeAdapter
(ScriptBytecodeAdapter.java:128)
at org.codehaus.groovy.runtime.ScriptBytecodeAdapter
(ScriptBytecodeAdapter.java:148)
at org.codehaus.groovy.tools.shell.InteractiveShellRunner
(InteractiveShellRunner.groovy:100)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:-2)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl
(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method (Method.java:606)
at
org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite$PogoCachedMethodSiteNoUnwrapNoCoerce
(PogoMetaMethodSite.java:267)
at org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite
(PogoMetaMethodSite.java:52)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite
(AbstractCallSite.java:137)
at org.codehaus.groovy.tools.shell.ShellRunner
(ShellRunner.groovy:57)
at org.codehaus.groovy.tools.shell.InteractiveShellRunner
(InteractiveShellRunner.groovy:-1)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:-2)
at sun.reflect.NativeMethodAccessorImpl
(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl
(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method (Method.java:606)
at org.codehaus.groovy.reflection.CachedMethod (CachedMethod.java:90)
at groovy.lang.MetaMethod (MetaMethod.java:233)
at groovy.lang.MetaClassImpl (MetaClassImpl.java:1054)
at org.codehaus.groovy.runtime.ScriptBytecodeAdapter
(ScriptBytecodeAdapter.java:128)
at org.codehaus.groovy.runtime.ScriptBytecodeAdapter
(ScriptBytecodeAdapter.java:148)
at org.codehaus.groovy.tools.shell.InteractiveShellRunner
(InteractiveShellRunner.groovy:66)
at java_lang_Runnable$run (null:-1)
at org.codehaus.groovy.runtime.callsite.CallSiteArray
(CallSiteArray.java:42)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite
(AbstractCallSite.java:108)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite
(AbstractCallSite.java:112)
at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:463)
at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:402)
at org.apache.sqoop.shell.SqoopShell (SqoopShell.java:128)
Caused by: Exception: java.lang.NullPointerException Message:
Stack trace:
at java.util.regex.Matcher (Matcher.java:1234)
at java.util.regex.Matcher (Matcher.java:308)
at java.util.regex.Matcher (Matcher.java:228)
at java.util.regex.Pattern (Pattern.java:1088)
at org.apache.sqoop.connector.kite.util.InputValidation
(InputValidation.java:35)
at
org.apache.sqoop.connector.kite.configuration.ToJobConfig$ConfigValidator
(ToJobConfig.java:38)
at
org.apache.sqoop.connector.kite.configuration.ToJobConfig$ConfigValidator
(ToJobConfig.java:33)
at org.apache.sqoop.validation.ConfigValidationRunner
(ConfigValidationRunner.java:172)
at org.apache.sqoop.validation.ConfigValidationRunner
(ConfigValidationRunner.java:140)
at org.apache.sqoop.validation.ConfigValidationRunner
(ConfigValidationRunner.java:121)
at org.apache.sqoop.validation.ConfigValidationRunner
(ConfigValidationRunner.java:82)
at org.apache.sqoop.model.ConfigUtils (ConfigUtils.java:220)
at org.apache.sqoop.handler.JobRequestHandler
(JobRequestHandler.java:220)
at org.apache.sqoop.handler.JobRequestHandler
(JobRequestHandler.java:102)
at org.apache.sqoop.server.v1.JobServlet (JobServlet.java:91)
at org.apache.sqoop.server.SqoopProtocolServlet
(SqoopProtocolServlet.java:63)
at javax.servlet.http.HttpServlet (HttpServlet.java:643)
at javax.servlet.http.HttpServlet (HttpServlet.java:723)
at org.apache.catalina.core.ApplicationFilterChain
(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain
(ApplicationFilterChain.java:206)
at
org.apache.hadoop.security.authentication.server.AuthenticationFilter
(AuthenticationFilter.java:392)
at org.apache.catalina.core.ApplicationFilterChain
(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain
(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve
(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve
(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve
(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve
(ErrorReportValve.java:103)
at org.apache.catalina.core.StandardEngineValve
(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter
(CoyoteAdapter.java:293)
at org.apache.coyote.http11.Http11Processor
(Http11Processor.java:861)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler
(Http11Protocol.java:606)
at org.apache.tomcat.util.net.JIoEndpoint$Worker
(JIoEndpoint.java:489)
at java.lang.Thread (Thread.java:744)
- Abraham Elmahrek
On Nov. 17, 2014, 7:31 a.m., Qian Xu wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/26963/
> -----------------------------------------------------------
>
> (Updated Nov. 17, 2014, 7:31 a.m.)
>
>
> Review request for Sqoop.
>
>
> Bugs: SQOOP-1588
> https://issues.apache.org/jira/browse/SQOOP-1588
>
>
> Repository: sqoop-sqoop2
>
>
> Description
> -------
>
> Create a basic Kite connector that can write data (i.e. from a jdbc
> connection) to HDFS.
>
> The scope is defined as follows:
> * Destination: HDFS
> * File Format: Avro Parquet and CSV.
> * Compression Codec: Use default
> * Partitioner Strategy: Not supported
> * Column Mapping: Not supported
>
>
> Diffs
> -----
>
> connector/connector-kite/pom.xml PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnectorConstants.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnectorError.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToDestroyer.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/LinkConfig.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/LinkConfiguration.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ToJobConfig.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ToJobConfiguration.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/InputValidation.java
> PRE-CREATION
>
> connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java
> PRE-CREATION
> connector/connector-kite/src/main/resources/connector-configs.properties
> PRE-CREATION
>
> connector/connector-kite/src/main/resources/kite-connector-config.properties
> PRE-CREATION
> connector/connector-kite/src/main/resources/sqoopconnector.properties
> PRE-CREATION
>
> connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
> PRE-CREATION
>
> connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java
> PRE-CREATION
>
> connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteToDestroyer.java
> PRE-CREATION
>
> connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteToInitializer.java
> PRE-CREATION
> connector/connector-kite/src/test/resources/log4j.properties PRE-CREATION
>
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/FileFormat.java
> PRE-CREATION
>
> connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/JarUtil.java
> PRE-CREATION
> connector/pom.xml e98a0fc
> pom.xml 233d3ce
> server/pom.xml 4a5eb5e
> test/pom.xml 2dbb8c5
>
> Diff: https://reviews.apache.org/r/26963/diff/
>
>
> Testing
> -------
>
> New unittests included. All passed.
>
>
> Thanks,
>
> Qian Xu
>
>
