Repository: sqoop Updated Branches: refs/heads/trunk 302674d96 -> a50394977
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java new file mode 100644 index 0000000..65ed0eb --- /dev/null +++ b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.importjob.numerictypes.parquet; + +import org.apache.sqoop.importjob.configuration.AvroTestConfiguration; +import org.apache.sqoop.importjob.configuration.OracleImportJobTestConfiguration; +import org.apache.sqoop.importjob.configuration.OracleImportJobTestConfigurationForNumber; +import org.apache.sqoop.importjob.configuration.ParquetTestConfiguration; +import org.apache.sqoop.importjob.numerictypes.NumericTypesAvroImportTestBase; +import org.apache.sqoop.testcategories.thirdpartytest.OracleTest; +import org.apache.sqoop.testutil.adapter.DatabaseAdapter; +import org.apache.sqoop.testutil.adapter.OracleDatabaseAdapter; +import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; + +import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITHOUT_EXTRA_ARGS; +import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITH_PADDING_ONLY; +import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY; + +@Category(OracleTest.class) +@RunWith(Parameterized.class) [email protected](BlockJUnit4ClassRunnerWithParametersFactory.class) +public class OracleNumericTypesParquetImportTest<T extends AvroTestConfiguration & ParquetTestConfiguration> extends NumericTypesAvroImportTestBase<T> { + + @Override + public DatabaseAdapter createAdapter() { + return new OracleDatabaseAdapter(); + } + + @Parameterized.Parameters(name = "Config: {0}| failWithoutExtraArgs: {1}| failWithPadding: {2}") + public static Iterable<? extends Object> testConfigurations() { + return Arrays.asList( + new Object[]{new OracleImportJobTestConfigurationForNumber(), FAIL_WITHOUT_EXTRA_ARGS, FAIL_WITH_PADDING_ONLY}, + new Object[]{new OracleImportJobTestConfiguration(), FAIL_WITHOUT_EXTRA_ARGS, SUCCEED_WITH_PADDING_ONLY} + ); + } + + public OracleNumericTypesParquetImportTest(T configuration, boolean failWithoutExtraArgs, boolean failWithPaddingOnly) { + super(configuration, failWithoutExtraArgs, failWithPaddingOnly); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java new file mode 100644 index 0000000..6e84c83 --- /dev/null +++ b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.importjob.numerictypes.parquet; + +import org.apache.sqoop.importjob.configuration.ParquetTestConfiguration; +import org.apache.sqoop.importjob.configuration.PostgresqlImportJobTestConfigurationForNumeric; +import org.apache.sqoop.importjob.configuration.PostgresqlImportJobTestConfigurationPaddingShouldSucceed; +import org.apache.sqoop.importjob.numerictypes.NumericTypesParquetImportTestBase; +import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest; +import org.apache.sqoop.testutil.adapter.DatabaseAdapter; +import org.apache.sqoop.testutil.adapter.PostgresDatabaseAdapter; +import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; + +import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITHOUT_EXTRA_ARGS; +import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITH_PADDING_ONLY; +import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITHOUT_EXTRA_ARGS; +import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY; + +@Category(PostgresqlTest.class) +@RunWith(Parameterized.class) [email protected](BlockJUnit4ClassRunnerWithParametersFactory.class) +public class PostgresNumericTypesParquetImportTest<T extends ParquetTestConfiguration> extends NumericTypesParquetImportTestBase<T> { + + @Override + public DatabaseAdapter createAdapter() { + return new PostgresDatabaseAdapter(); + } + + @Parameterized.Parameters(name = "Config: {0}| failWithoutExtraArgs: {1}| failWithPadding: {2}") + public static Iterable<? extends Object> testConfigurations() { + return Arrays.asList( + new Object[]{new PostgresqlImportJobTestConfigurationForNumeric(), FAIL_WITHOUT_EXTRA_ARGS, FAIL_WITH_PADDING_ONLY}, + new Object[]{new PostgresqlImportJobTestConfigurationPaddingShouldSucceed(), SUCCEED_WITHOUT_EXTRA_ARGS, SUCCEED_WITH_PADDING_ONLY} + ); + } + + public PostgresNumericTypesParquetImportTest(T configuration, boolean failWithoutExtraArgs, boolean failWithPaddingOnly) { + super(configuration, failWithoutExtraArgs, failWithPaddingOnly); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java new file mode 100644 index 0000000..7f5eb35 --- /dev/null +++ b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.importjob.numerictypes.parquet; + +import org.apache.sqoop.importjob.configuration.SqlServerImportJobTestConfiguration; +import org.apache.sqoop.importjob.numerictypes.NumericTypesParquetImportTestBase; +import org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest; +import org.apache.sqoop.testutil.adapter.DatabaseAdapter; +import org.apache.sqoop.testutil.adapter.SqlServerDatabaseAdapter; +import org.junit.experimental.categories.Category; + +import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITHOUT_EXTRA_ARGS; +import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY; + +@Category(SqlServerTest.class) +public class SqlServerNumericTypesParquetImportTest extends NumericTypesParquetImportTestBase { + + @Override + public DatabaseAdapter createAdapter() { + return new SqlServerDatabaseAdapter(); + } + + public SqlServerNumericTypesParquetImportTest() { + super(new SqlServerImportJobTestConfiguration(), SUCCEED_WITHOUT_EXTRA_ARGS, SUCCEED_WITH_PADDING_ONLY); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java b/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java new file mode 100644 index 0000000..60371b9 --- /dev/null +++ b/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.testutil; + +import org.junit.rules.ExpectedException; + +import java.io.IOException; + +public class NumericTypesTestUtils { + // Constants for the basic test case, that doesn't use extra arguments + // that are required to avoid errors, i.e. padding and default precision and scale. + public final static boolean SUCCEED_WITHOUT_EXTRA_ARGS = false; + public final static boolean FAIL_WITHOUT_EXTRA_ARGS = true; + // Constants for the test case that has padding specified but not default precision and scale. + public final static boolean SUCCEED_WITH_PADDING_ONLY = false; + public final static boolean FAIL_WITH_PADDING_ONLY = true; + + /** + * Adds properties to the given arg builder for decimal precision and scale. + * @param builder + */ + public static void addPrecisionAndScale(ArgumentArrayBuilder builder) { + builder.withProperty("sqoop.avro.logical_types.decimal.default.precision", "38"); + builder.withProperty("sqoop.avro.logical_types.decimal.default.scale", "3"); + } + + /** + * Enables padding for decimals in avro and parquet import. + * @param builder + */ + public static void addPadding(ArgumentArrayBuilder builder) { + builder.withProperty("sqoop.avro.decimal_padding.enable", "true"); + } + + public static void addEnableAvroDecimal(ArgumentArrayBuilder builder) { + builder.withProperty("sqoop.avro.logical_types.decimal.enable", "true"); + } + + public static void addEnableParquetDecimal(ArgumentArrayBuilder builder) { + builder.withProperty("sqoop.parquet.logical_types.decimal.enable", "true"); + } + + public static void configureJunitToExpectFailure(ExpectedException thrown) { + thrown.expect(IOException.class); + thrown.expectMessage("Failure during job; return status 1"); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java b/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java new file mode 100644 index 0000000..a2087e3 --- /dev/null +++ b/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.testutil; + +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.importjob.DatabaseAdapterFactory; +import org.apache.sqoop.importjob.configuration.ImportJobTestConfiguration; +import org.apache.sqoop.testutil.adapter.DatabaseAdapter; +import org.junit.After; +import org.junit.Before; + +import java.sql.SQLException; +import java.util.List; + +public abstract class ThirdPartyTestBase<T extends ImportJobTestConfiguration> extends ImportJobTestCase implements DatabaseAdapterFactory { + + private final DatabaseAdapter adapter; + protected final T configuration; + + private Configuration conf = new Configuration(); + + public DatabaseAdapter getAdapter() { + return adapter; + } + + public T getConfiguration() { + return configuration; + } + + protected ThirdPartyTestBase(T configuration) { + this.adapter = createAdapter(); + this.configuration = configuration; + } + + @Override + protected String getConnectString() { + return adapter.getConnectionString(); + } + + @Override + protected SqoopOptions getSqoopOptions(Configuration conf) { + SqoopOptions opts = new SqoopOptions(conf); + adapter.injectConnectionParameters(opts); + return opts; + } + + @Override + protected void dropTableIfExists(String table) throws SQLException { + adapter.dropTableIfExists(table, getManager()); + } + + @Override + protected Configuration getConf() { + return conf; + } + + @Override + protected boolean useHsqldbTestServer() { + return false; + } + + @Before + public void setUp() { + super.setUp(); + String[] names = configuration.getNames(); + String[] types = configuration.getTypes(); + createTableWithColTypesAndNames(names, types, new String[0]); + List<String[]> inputData = configuration.getSampleData(); + for (String[] input : inputData) { + insertIntoTable(names, types, input); + } + } + + @After + public void tearDown() { + try { + dropTableIfExists(getTableName()); + } catch (SQLException e) { + LOG.warn("Error trying to drop table on tearDown: " + e); + } + super.tearDown(); + } + +}
