Updated Branches: refs/heads/sqoop2 fd4be5f5e -> 681914c85
SQOOP-943: Sqoop2: Repository wide configuration of immutable disk structures (Jarek Jarcec Cecho via Kate Ting) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/681914c8 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/681914c8 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/681914c8 Branch: refs/heads/sqoop2 Commit: 681914c851cb275f1d803ffe057c2a7a53436330 Parents: fd4be5f Author: Kate Ting <[email protected]> Authored: Sat Jun 22 21:00:42 2013 -0400 Committer: Kate Ting <[email protected]> Committed: Sat Jun 22 21:00:42 2013 -0400 ---------------------------------------------------------------------- .../apache/sqoop/repository/JdbcRepository.java | 34 ++++++++++++++++++-- .../sqoop/repository/JdbcRepositoryContext.java | 9 ------ .../repository/JdbcRepositoryProvider.java | 10 +----- .../repository/RepoConfigurationConstants.java | 15 ++++----- .../org/apache/sqoop/repository/Repository.java | 20 ++++++++++++ .../sqoop/repository/RepositoryError.java | 3 ++ .../sqoop/repository/RepositoryManager.java | 9 ++++++ dist/src/main/server/conf/sqoop.properties | 4 +-- .../test/minicluster/SqoopMiniCluster.java | 2 +- 9 files changed, 75 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java index 135c2d2..aa1aa8d 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java @@ -31,8 +31,7 @@ import org.apache.sqoop.model.MSubmission; public class JdbcRepository extends Repository { - private static final Logger LOG = - Logger.getLogger(JdbcRepository.class); + private static final Logger LOG = Logger.getLogger(JdbcRepository.class); private final JdbcRepositoryHandler handler; private final JdbcRepositoryContext repoContext; @@ -124,6 +123,37 @@ public class JdbcRepository extends Repository { * {@inheritDoc} */ @Override + public void createOrUpdateInternals() { + doWithConnection(new DoWithConnection() { + @Override + public Object doIt(Connection conn) throws Exception { + if (!handler.schemaExists()) { + LOG.info("Creating repository schema objects"); + handler.createSchema(); + } + + return null; + } + }); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean haveSuitableInternals() { + return (Boolean) doWithConnection(new DoWithConnection() { + @Override + public Object doIt(Connection conn) throws Exception { + return handler.schemaExists(); + } + }); + } + + /** + * {@inheritDoc} + */ + @Override public MConnector registerConnector(final MConnector mConnector) { return (MConnector) doWithConnection(new DoWithConnection() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java index 8989fb6..0a8139a 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java @@ -34,7 +34,6 @@ public final class JdbcRepositoryContext { private final MapContext context; private final String handlerClassName; - private final boolean createSchema; private final String connectionUrl; private final String driverClassName; private final Properties connectionProperties; @@ -55,9 +54,6 @@ public final class JdbcRepositoryContext { RepoConfigurationConstants.SYSCFG_REPO_JDBC_HANDLER); } - createSchema = context.getBoolean( - RepoConfigurationConstants.SYSCFG_REPO_JDBC_CREATE_SCHEMA, false); - connectionUrl = context.getString( RepoConfigurationConstants.SYSCFG_REPO_JDBC_URL); @@ -132,7 +128,6 @@ public final class JdbcRepositoryContext { if (LOG.isInfoEnabled()) { StringBuilder sb = new StringBuilder("[repo-ctx] "); sb.append("handler=").append(handlerClassName).append(", "); - sb.append("create-schema=").append(createSchema).append(", "); sb.append("conn-url=").append(connectionUrl).append(", "); sb.append("driver=").append(driverClassName).append(", "); sb.append("user=").append(jdbcUserName).append(", "); @@ -189,10 +184,6 @@ public final class JdbcRepositoryContext { return driverClassName; } - public boolean shouldCreateSchema() { - return createSchema; - } - public JdbcTransactionIsolation getTransactionIsolation() { return transactionIsolation; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java index 3339c59..1fd092a 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java @@ -40,8 +40,7 @@ import org.apache.sqoop.utils.ClassUtils; public class JdbcRepositoryProvider implements RepositoryProvider { - private static final Logger LOG = - Logger.getLogger(JdbcRepositoryProvider.class); + private static final Logger LOG = Logger.getLogger(JdbcRepositoryProvider.class); private JdbcRepositoryContext repoContext; @@ -156,13 +155,6 @@ public class JdbcRepositoryProvider implements RepositoryProvider { handler.initialize(repoContext); - if (repoContext.shouldCreateSchema()) { - if (!handler.schemaExists()) { - LOG.info("Creating repository schema objects"); - handler.createSchema(); - } - } - repository = new JdbcRepository(handler, repoContext); LOG.info("JdbcRepositoryProvider initialized"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java index 8939f8a..12cb93e 100644 --- a/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java +++ b/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java @@ -43,6 +43,13 @@ public final class RepoConfigurationConstants { + "provider"; /** + * Boolean property defining whether repository is allowed to make changes + * on disk structures (schema in databases, changing file format, ...). + */ + public static final String SYSCFG_REPO_SCHEMA_IMMUTABLE= + PREFIX_REPO_CONFIG + "schema.immutable"; + + /** * Class name for the JDBC repository handler specified by: * <tt>org.apache.sqoop.repository.jdbc.handler</tt>. */ @@ -50,14 +57,6 @@ public final class RepoConfigurationConstants { + "jdbc.handler"; /** - * Indicates if the repository should create the schema objects as necessary, - * specified as a boolean value for the key: - * <tt>org.apache.sqoop.repository.jdbc.create.schema</tt> - */ - public static final String SYSCFG_REPO_JDBC_CREATE_SCHEMA = - PREFIX_REPO_CONFIG + "jdbc.create.schema"; - - /** * JDBC connection URL specified by: * <tt>org.apache.sqoop.repository.jdbc.url</tt> */ http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/Repository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java index 1ba947a..a2911c5 100644 --- a/core/src/main/java/org/apache/sqoop/repository/Repository.java +++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java @@ -56,6 +56,26 @@ public abstract class Repository { public abstract RepositoryTransaction getTransaction(); /** + * Create or update disk data structures. + * + * This method will be called only if Sqoop server is enabled with changing + * repository on disk structures. Repository should not change its disk structures + * outside of this method. This method must be no-op in case that the structures + * do not need any maintenance. + */ + public abstract void createOrUpdateInternals(); + + /** + * Return true if internal repository structures exists and are suitable for use. + * + * This method should return false in case that the structures do exists, but + * are not suitable for use or if they requires upgrade. + * + * @return Boolean values if internal structures are suitable for use + */ + public abstract boolean haveSuitableInternals(); + + /** * Registers given connector in the repository and return registered * variant. This method might return an exception in case that metadata for * given connector are already registered with different structure. http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java index 4cae7ba..ab5e5a1 100644 --- a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java +++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java @@ -29,6 +29,9 @@ public enum RepositoryError implements ErrorCode { /** The system was unable to find or load the repository provider. */ REPO_0001("Invalid repository provider specified"), + /** Repository on disk structures are not suitable for use */ + REPO_0002("Repository structures are not initialized or requires upgrade"), + // JDBC Repository Errors: Prefix JDBCREP /** An unknown error has occurred. */ http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java index 955306d..a178238 100644 --- a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java +++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java @@ -111,6 +111,15 @@ public class RepositoryManager { provider.initialize(context); + if(!context.getBoolean(RepoConfigurationConstants.SYSCFG_REPO_SCHEMA_IMMUTABLE, true)) { + LOG.info("Creating or upgrading on disk structures if necessary"); + provider.getRepository().createOrUpdateInternals(); + } + + if(!provider.getRepository().haveSuitableInternals()) { + throw new SqoopException(RepositoryError.REPO_0002); + } + LOG.info("Repository initialized: OK"); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/dist/src/main/server/conf/sqoop.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties index 5131aad..07b148e 100755 --- a/dist/src/main/server/conf/sqoop.properties +++ b/dist/src/main/server/conf/sqoop.properties @@ -62,20 +62,20 @@ org.apache.sqoop.log4j.category.org.apache.derby=INFO # is "org.apache.sqoop.repository.sysprop". Any property that # is specified with this prefix is parsed out and set as a # system property. For example, if the built in Derby repository -# is being used, the sysprop prefixed proeprties can be used +# is being used, the sysprop prefixed properties can be used # to affect Derby configuration at startup time by setting # the appropriate system properties. # # Repository provider org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider +org.apache.sqoop.repository.schema.immutable=false # JDBC repository provider configuration org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED org.apache.sqoop.repository.jdbc.maximum.connections=10 org.apache.sqoop.repository.jdbc.url=jdbc:derby:@BASEDIR@/repository/db;create=true -org.apache.sqoop.repository.jdbc.create.schema=true org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver org.apache.sqoop.repository.jdbc.user=sa org.apache.sqoop.repository.jdbc.password= http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java index 3620c28..9c71688 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java @@ -169,11 +169,11 @@ public abstract class SqoopMiniCluster { Map<String, String> properties = new HashMap<String, String>(); properties.put("org.apache.sqoop.repository.provider", "org.apache.sqoop.repository.JdbcRepositoryProvider"); + properties.put("org.apache.sqoop.repository.schema.immutable", "false"); properties.put("org.apache.sqoop.repository.jdbc.handler", "org.apache.sqoop.repository.derby.DerbyRepositoryHandler"); properties.put("org.apache.sqoop.repository.jdbc.transaction.isolation", "READ_COMMITTED"); properties.put("org.apache.sqoop.repository.jdbc.maximum.connections", "10"); properties.put("org.apache.sqoop.repository.jdbc.url=jdbc:derby:memory:myDB;create", "true"); - properties.put("org.apache.sqoop.repository.jdbc.create.schema", "true"); properties.put("org.apache.sqoop.repository.jdbc.driver", "org.apache.derby.jdbc.EmbeddedDriver"); properties.put("org.apache.sqoop.repository.jdbc.user", "sa"); properties.put("org.apache.sqoop.repository.jdbc.password", "");
