[
https://issues.apache.org/jira/browse/SQOOP-3396?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16707475#comment-16707475
]
ASF GitHub Bot commented on SQOOP-3396:
---------------------------------------
Github user szvasas commented on a diff in the pull request:
https://github.com/apache/sqoop/pull/60#discussion_r238334653
--- Diff:
src/test/org/apache/sqoop/importjob/numerictypes/NumericTypesImportTestBase.java
---
@@ -65,240 +46,79 @@
* 2. Decimal padding during avro or parquet import
* In case of Oracle and Postgres, Sqoop has to pad the values with 0s to
avoid errors.
*/
-public abstract class NumericTypesImportTestBase<T extends
AvroTestConfiguration & ParquetTestConfiguration> extends ImportJobTestCase
implements DatabaseAdapterFactory {
+public abstract class NumericTypesImportTestBase<T extends
ImportJobTestConfiguration> extends ThirdPartyTestBase<T> {
public static final Log LOG =
LogFactory.getLog(NumericTypesImportTestBase.class.getName());
- private Configuration conf = new Configuration();
-
- private final T configuration;
- private final DatabaseAdapter adapter;
private final boolean failWithoutExtraArgs;
private final boolean failWithPadding;
- // Constants for the basic test case, that doesn't use extra arguments
- // that are required to avoid errors, i.e. padding and default precision
and scale.
- protected final static boolean SUCCEED_WITHOUT_EXTRA_ARGS = false;
- protected final static boolean FAIL_WITHOUT_EXTRA_ARGS = true;
-
- // Constants for the test case that has padding specified but not
default precision and scale.
- protected final static boolean SUCCEED_WITH_PADDING_ONLY = false;
- protected final static boolean FAIL_WITH_PADDING_ONLY = true;
-
- private Path tableDirPath;
-
public NumericTypesImportTestBase(T configuration, boolean
failWithoutExtraArgs, boolean failWithPaddingOnly) {
- this.adapter = createAdapter();
- this.configuration = configuration;
+ super(configuration);
this.failWithoutExtraArgs = failWithoutExtraArgs;
this.failWithPadding = failWithPaddingOnly;
}
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Override
- protected Configuration getConf() {
- return conf;
- }
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- @Override
- protected String getConnectString() {
- return adapter.getConnectionString();
- }
-
- @Override
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opts = new SqoopOptions(conf);
- adapter.injectConnectionParameters(opts);
- return opts;
- }
-
- @Override
- protected void dropTableIfExists(String table) throws SQLException {
- adapter.dropTableIfExists(table, getManager());
- }
-
@Before
public void setUp() {
super.setUp();
- String[] names = configuration.getNames();
- String[] types = configuration.getTypes();
- createTableWithColTypesAndNames(names, types, new String[0]);
- List<String[]> inputData = configuration.getSampleData();
- for (String[] input : inputData) {
- insertIntoTable(names, types, input);
- }
tableDirPath = new Path(getWarehouseDir() + "/" + getTableName());
}
- @After
- public void tearDown() {
- try {
- dropTableIfExists(getTableName());
- } catch (SQLException e) {
- LOG.warn("Error trying to drop table on tearDown: " + e);
- }
- super.tearDown();
- }
+ public Path tableDirPath;
- private ArgumentArrayBuilder getArgsBuilder(SqoopOptions.FileLayout
fileLayout) {
- ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
- if (AvroDataFile.equals(fileLayout)) {
- builder.withOption("as-avrodatafile");
- }
- else if (ParquetFile.equals(fileLayout)) {
- builder.withOption("as-parquetfile");
- }
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ abstract public ArgumentArrayBuilder getArgsBuilder();
+ abstract public void verify();
+ public ArgumentArrayBuilder includeCommonOptions(ArgumentArrayBuilder
builder) {
return builder.withCommonHadoopFlags(true)
.withOption("warehouse-dir", getWarehouseDir())
.withOption("num-mappers", "1")
.withOption("table", getTableName())
.withOption("connect", getConnectString());
}
- /**
- * Adds properties to the given arg builder for decimal precision and
scale.
- * @param builder
- */
- private void addPrecisionAndScale(ArgumentArrayBuilder builder) {
-
builder.withProperty("sqoop.avro.logical_types.decimal.default.precision",
"38");
- builder.withProperty("sqoop.avro.logical_types.decimal.default.scale",
"3");
- }
-
- /**
- * Enables padding for decimals in avro and parquet import.
- * @param builder
- */
- private void addPadding(ArgumentArrayBuilder builder) {
- builder.withProperty("sqoop.avro.decimal_padding.enable", "true");
- }
-
- private void addEnableAvroDecimal(ArgumentArrayBuilder builder) {
- builder.withProperty("sqoop.avro.logical_types.decimal.enable",
"true");
- }
-
- private void addEnableParquetDecimal(ArgumentArrayBuilder builder) {
- builder.withProperty("sqoop.parquet.logical_types.decimal.enable",
"true");
- }
- private void configureJunitToExpectFailure(boolean failWithPadding) {
- if (failWithPadding) {
+ @Test
+ public void testImportWithoutPadding() throws IOException {
+ if(failWithoutExtraArgs){
thrown.expect(IOException.class);
thrown.expectMessage("Failure during job; return status 1");
+ NumericTypesTestUtils.configureJunitToExpectFailure(thrown);
}
- }
-
- @Test
- public void testAvroImportWithoutPadding() throws IOException {
- configureJunitToExpectFailure(failWithoutExtraArgs);
- ArgumentArrayBuilder builder = getArgsBuilder(AvroDataFile);
- addEnableAvroDecimal(builder);
+ ArgumentArrayBuilder builder = getArgsBuilder();
String[] args = builder.build();
runImport(args);
if (!failWithoutExtraArgs) {
- verify(AvroDataFile);
- }
- }
-
- @Test
- public void testAvroImportWithPadding() throws IOException {
- configureJunitToExpectFailure(failWithPadding);
- ArgumentArrayBuilder builder = getArgsBuilder(AvroDataFile);
- addEnableAvroDecimal(builder);
- addPadding(builder);
- runImport(builder.build());
- if (!failWithPadding) {
- verify(AvroDataFile);
+ verify();
}
}
@Test
- public void testAvroImportWithDefaultPrecisionAndScale() throws
IOException {
- ArgumentArrayBuilder builder = getArgsBuilder(AvroDataFile);
- addEnableAvroDecimal(builder);
- addPadding(builder);
- addPrecisionAndScale(builder);
- runImport(builder.build());
- verify(AvroDataFile);
- }
-
- @Test
- public void testParquetImportWithoutPadding() throws IOException {
- configureJunitToExpectFailure(failWithoutExtraArgs);
- ArgumentArrayBuilder builder = getArgsBuilder(ParquetFile);
- addEnableParquetDecimal(builder);
- String[] args = builder.build();
- runImport(args);
- if (!failWithoutExtraArgs) {
- verify(ParquetFile);
+ public void testImportWithPadding() throws IOException {
+ if(failWithPadding){
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Failure during job; return status 1");
+ NumericTypesTestUtils.configureJunitToExpectFailure(thrown);
--- End diff --
This method seems to be doing the same thing as the previous line.
> Add parquet numeric support for Parquet in Hive import
> ------------------------------------------------------
>
> Key: SQOOP-3396
> URL: https://issues.apache.org/jira/browse/SQOOP-3396
> Project: Sqoop
> Issue Type: Sub-task
> Reporter: Fero Szabo
> Assignee: Fero Szabo
> Priority: Major
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
