Repository: sqoop Updated Branches: refs/heads/sqoop2 0756d1366 -> 4bf7e0029
SQOOP-1331: Add method "findConnectors" to repository API (Gwen Shapira via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/4bf7e002 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/4bf7e002 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/4bf7e002 Branch: refs/heads/sqoop2 Commit: 4bf7e0029cafea7d5f7b99fa60dce7aafa6aaaef Parents: 0756d13 Author: Jarek Jarcec Cecho <[email protected]> Authored: Wed Jun 4 10:00:18 2014 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Wed Jun 4 10:00:18 2014 -0700 ---------------------------------------------------------------------- .../apache/sqoop/repository/JdbcRepository.java | 13 +++ .../sqoop/repository/JdbcRepositoryHandler.java | 7 ++ .../org/apache/sqoop/repository/Repository.java | 7 ++ .../sqoop/repository/derby/DerbyRepoError.java | 4 +- .../derby/DerbyRepositoryHandler.java | 99 ++++++++++++++------ .../repository/derby/DerbySchemaQuery.java | 8 ++ .../sqoop/repository/derby/DerbyTestCase.java | 9 ++ .../repository/derby/TestConnectorHandling.java | 19 ++++ 8 files changed, 134 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java index 7768b13..9b64661 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java @@ -205,6 +205,19 @@ public class JdbcRepository extends Repository { /** * {@inheritDoc} */ + @Override + public List<MConnector> findConnectors() { + return (List<MConnector>) doWithConnection(new DoWithConnection() { + @Override + public Object doIt(Connection conn) { + return handler.findConnectors(conn); + } + }); + } + + /** + * {@inheritDoc} + */ @Override public MFramework registerFramework(final MFramework mFramework, final boolean autoUpgrade) { return (MFramework) doWithConnection(new DoWithConnection() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java index 9299484..4de3134 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java @@ -52,6 +52,13 @@ public abstract class JdbcRepositoryHandler { public abstract MConnector findConnector(String shortName, Connection conn); /** + * Get all connectors in repository + * + * @return List will all connectors in repository + */ + public abstract List<MConnector> findConnectors(Connection conn); + + /** * Register given connector in repository. * * Save given connector data to the repository. Given connector should not be http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/core/src/main/java/org/apache/sqoop/repository/Repository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java index 4c7b866..ecf5004 100644 --- a/core/src/main/java/org/apache/sqoop/repository/Repository.java +++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java @@ -96,6 +96,13 @@ public abstract class Repository { */ public abstract MConnector findConnector(String shortName); + /** + * Get all connectors in repository + * + * @return List will all connectors in repository + */ + public abstract List<MConnector> findConnectors(); + /** * Registers given framework in the repository and return registered http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java index e4b30f9..aae89fe 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java @@ -179,7 +179,9 @@ public enum DerbyRepoError implements ErrorCode { DERBYREPO_0043("Can't enable/disable job"), DERBYREPO_0044("Update of framework failed"), - ; + + DERBYREPO_0045("Can't retrieve all connector metadata"), + ; private final String message; http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java index 4f002bb..5bce3a9 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java @@ -446,57 +446,54 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler { } MConnector mc = null; PreparedStatement baseConnectorFetchStmt = null; - PreparedStatement formFetchStmt = null; - PreparedStatement inputFetchStmt = null; try { baseConnectorFetchStmt = conn.prepareStatement(STMT_FETCH_BASE_CONNECTOR); baseConnectorFetchStmt.setString(1, shortName); - ResultSet rsetBaseConnector = baseConnectorFetchStmt.executeQuery(); - if (!rsetBaseConnector.next()) { + List<MConnector> connectors = loadConnectors(baseConnectorFetchStmt,conn); + + if (connectors.size()==0) { LOG.debug("No connector found by name: " + shortName); return null; + } else if (connectors.size()==1) { + LOG.debug("Looking up connector: " + shortName + ", found: " + mc); + return connectors.get(0); } - - long connectorId = rsetBaseConnector.getLong(1); - String connectorName = rsetBaseConnector.getString(2); - String connectorClassName = rsetBaseConnector.getString(3); - String connectorVersion = rsetBaseConnector.getString(4); - - formFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_CONNECTOR); - formFetchStmt.setLong(1, connectorId); - inputFetchStmt = conn.prepareStatement(STMT_FETCH_INPUT); - - List<MForm> connectionForms = new ArrayList<MForm>(); - Map<MJob.Type, List<MForm>> jobForms = - new HashMap<MJob.Type, List<MForm>>(); - - loadForms(connectionForms, jobForms, formFetchStmt, inputFetchStmt, 1); - - mc = new MConnector(connectorName, connectorClassName, connectorVersion, - new MConnectionForms(connectionForms), - convertToJobList(jobForms)); - mc.setPersistenceId(connectorId); - - if (rsetBaseConnector.next()) { + else { throw new SqoopException(DerbyRepoError.DERBYREPO_0005, shortName); } + } catch (SQLException ex) { logException(ex, shortName); throw new SqoopException(DerbyRepoError.DERBYREPO_0004, shortName, ex); } finally { - closeStatements(baseConnectorFetchStmt, - formFetchStmt, inputFetchStmt); + closeStatements(baseConnectorFetchStmt); } - - LOG.debug("Looking up connector: " + shortName + ", found: " + mc); - return mc; } + /** * {@inheritDoc} */ @Override + public List<MConnector> findConnectors(Connection conn) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(STMT_SELECT_CONNECTOR_ALL); + return loadConnectors(stmt,conn); + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(DerbyRepoError.DERBYREPO_0045, ex); + } finally { + closeStatements(stmt); + } + } + + + /** + * {@inheritDoc} + */ + @Override public void registerFramework(MFramework mf, Connection conn) { if (mf.hasPersistenceId()) { throw new SqoopException(DerbyRepoError.DERBYREPO_0011, @@ -1603,6 +1600,46 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler { } } + private List<MConnector> loadConnectors(PreparedStatement stmt,Connection conn) throws SQLException { + List<MConnector> connectors = new ArrayList<MConnector>(); + ResultSet rsConnectors = null; + PreparedStatement formFetchStmt = null; + PreparedStatement inputFetchStmt = null; + + try { + rsConnectors = stmt.executeQuery(); + formFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_CONNECTOR); + inputFetchStmt = conn.prepareStatement(STMT_FETCH_INPUT); + + while(rsConnectors.next()) { + long connectorId = rsConnectors.getLong(1); + String connectorName = rsConnectors.getString(2); + String connectorClassName = rsConnectors.getString(3); + String connectorVersion = rsConnectors.getString(4); + + formFetchStmt.setLong(1, connectorId); + + List<MForm> connectionForms = new ArrayList<MForm>(); + Map<MJob.Type, List<MForm>> jobForms = + new HashMap<MJob.Type, List<MForm>>(); + + loadForms(connectionForms, jobForms, formFetchStmt, inputFetchStmt, 1); + + MConnector mc = new MConnector(connectorName, connectorClassName, connectorVersion, + new MConnectionForms(connectionForms), + convertToJobList(jobForms)); + mc.setPersistenceId(connectorId); + + connectors.add(mc); + } + } finally { + closeResultSets(rsConnectors); + closeStatements(formFetchStmt,inputFetchStmt); + } + + return connectors; + } + private List<MConnection> loadConnections(PreparedStatement stmt, Connection conn) throws SQLException { http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java index c470211..7042a53 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java @@ -445,6 +445,14 @@ public final class DerbySchemaQuery { + " FROM " + TABLE_SQ_CONNECTOR + " WHERE " + COLUMN_SQC_NAME + " = ?"; + // DML: Select all connectors + public static final String STMT_SELECT_CONNECTOR_ALL = + "SELECT " + + COLUMN_SQC_ID + ", " + + COLUMN_SQC_NAME + ", " + + COLUMN_SQC_CLASS + ", " + + COLUMN_SQC_VERSION + + " FROM " + TABLE_SQ_CONNECTOR; // DML: Fetch all forms for a given connector public static final String STMT_FETCH_FORM_CONNECTOR = http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java index a1ad40d..20b87a1 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java @@ -265,6 +265,15 @@ abstract public class DerbyTestCase extends TestCase { } /** + * Add a second connector for testing with multiple connectors + */ + public void addConnector() throws Exception { + // Connector entry + runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)" + + "VALUES('B', 'org.apache.sqoop.test.B', '1.0-test')"); + } + + /** * Load testing submissions into the metadata repository. * * @throws Exception http://git-wip-us.apache.org/repos/asf/sqoop/blob/4bf7e002/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java index cc3fe60..745e128 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java @@ -19,6 +19,8 @@ package org.apache.sqoop.repository.derby; import org.apache.sqoop.model.MConnector; +import java.util.List; + /** * Test connector methods on Derby repository. */ @@ -55,6 +57,23 @@ public class TestConnectorHandling extends DerbyTestCase { assertEquals(original, connector); } + public void testFindAllConnectors() throws Exception { + // No connectors in an empty repository, we expect an empty list + assertEquals(handler.findConnectors(getDerbyConnection()).size(),0); + + loadConnectorAndFramework(); + addConnector(); + + // Retrieve connectors + List<MConnector> connectors = handler.findConnectors(getDerbyConnection()); + assertNotNull(connectors); + assertEquals(connectors.size(),2); + assertEquals(connectors.get(0).getUniqueName(),"A"); + assertEquals(connectors.get(1).getUniqueName(),"B"); + + + } + public void testRegisterConnector() throws Exception { MConnector connector = getConnector();
