Repository: sqoop Updated Branches: refs/heads/sqoop2 98d0db1f5 -> 70a08a029
SQOOP-1577: Sqoop2: Refactor repository dump and load tools (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/70a08a02 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/70a08a02 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/70a08a02 Branch: refs/heads/sqoop2 Commit: 70a08a029a2c48cb9b23ad5e06654c7325872575 Parents: 98d0db1 Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Feb 20 08:44:57 2015 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Feb 20 08:44:57 2015 -0800 ---------------------------------------------------------------------- .../apache/sqoop/tools/tool/JSONConstants.java | 3 +- .../sqoop/tools/tool/RepositoryDumpTool.java | 38 ++++++---- .../sqoop/tools/tool/RepositoryLoadTool.java | 78 +++++++++++--------- 3 files changed, 69 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java index 9bec0d0..9301707 100644 --- a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java +++ b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java @@ -25,8 +25,9 @@ public final class JSONConstants { } public static final String CONNECTOR_ID = "connector-id"; + public static final String FROM_CONNECTOR_ID = "from-connector-id"; + public static final String TO_CONNECTOR_ID = "from-connector-id"; public static final String CONNECTOR_NAME = "connector-name"; - public static final String ALL = "all"; public static final String NAME = "name"; public static final String LINK_ID = "link-id"; public static final String JOB_ID = "job-id"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java index 53dac1b..1027baf 100644 --- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java +++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java @@ -21,6 +21,7 @@ import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; import java.util.Iterator; +import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -31,10 +32,10 @@ import org.apache.commons.cli.ParseException; import org.apache.log4j.Logger; import org.apache.sqoop.common.VersionInfo; import org.apache.sqoop.connector.ConnectorManager; -import org.apache.sqoop.json.JobBean; -import org.apache.sqoop.json.JsonBean; -import org.apache.sqoop.json.LinkBean; +import org.apache.sqoop.json.JobsBean; +import org.apache.sqoop.json.LinksBean; import org.apache.sqoop.json.SubmissionsBean; +import org.apache.sqoop.model.MLink; import org.apache.sqoop.repository.Repository; import org.apache.sqoop.repository.RepositoryManager; import org.apache.sqoop.tools.ConfiguredTool; @@ -101,17 +102,26 @@ public class RepositoryDumpTool extends ConfiguredTool { JSONObject result = new JSONObject(); - LOG.info("Dumping Connections with skipSensitive=" + String.valueOf(skipSensitive)); - LinkBean links = new LinkBean(repository.findLinks()); - result.put(JSONConstants.LINKS, addConnectorName(links.extract(skipSensitive))); + LOG.info("Dumping Links with skipSensitive=" + String.valueOf(skipSensitive)); + List<MLink> links = repository.findLinks(); + LinksBean linkBeans = new LinksBean(links); + JSONObject linksJsonObject = linkBeans.extract(skipSensitive); + JSONArray linksJsonArray = (JSONArray)linksJsonObject.get(JSONConstants.LINKS); + addConnectorName(linksJsonArray, JSONConstants.CONNECTOR_ID); + result.put(JSONConstants.LINKS, linksJsonObject); LOG.info("Dumping Jobs with skipSensitive=" + String.valueOf(skipSensitive)); - JobBean jobs = new JobBean(repository.findJobs()); - result.put(JSONConstants.JOBS, addConnectorName(jobs.extract(skipSensitive))); + JobsBean jobs = new JobsBean(repository.findJobs()); + JSONObject jobsJsonObject = jobs.extract(skipSensitive); + JSONArray jobsJsonArray = (JSONArray)jobsJsonObject.get(JSONConstants.JOBS); + addConnectorName(jobsJsonArray, JSONConstants.FROM_CONNECTOR_ID); + addConnectorName(jobsJsonArray, JSONConstants.TO_CONNECTOR_ID); + result.put(JSONConstants.JOBS, jobsJsonObject); LOG.info("Dumping Submissions with skipSensitive=" + String.valueOf(skipSensitive)); SubmissionsBean submissions = new SubmissionsBean(repository.findSubmissions()); - result.put(JSONConstants.SUBMISSIONS, submissions.extract(skipSensitive)); + JSONObject submissionsJsonObject = submissions.extract(skipSensitive); + result.put(JSONConstants.SUBMISSIONS, submissionsJsonObject); result.put(JSONConstants.METADATA, repoMetadata(skipSensitive)); @@ -129,19 +139,17 @@ public class RepositoryDumpTool extends ConfiguredTool { return metadata; } - private JSONObject addConnectorName(JSONObject json) { + private JSONArray addConnectorName(JSONArray jsonArray, String connectorKey) { ConnectorManager connectorManager = ConnectorManager.getInstance(); - JSONArray results = (JSONArray) json.get(JsonBean.ALL); - - Iterator<JSONObject> iterator = results.iterator(); + Iterator<JSONObject> iterator = jsonArray.iterator(); while (iterator.hasNext()) { JSONObject result = iterator.next(); - Long connectorId = (Long) result.get(JSONConstants.CONNECTOR_ID); + Long connectorId = (Long) result.get(connectorKey); result.put(JSONConstants.CONNECTOR_NAME, connectorManager.getConnectorConfigurable(connectorId).getUniqueName()); } - return json; + return jsonArray; } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java index d0ebaed..e13cf1e 100644 --- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java +++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java @@ -43,9 +43,9 @@ import org.apache.sqoop.connector.spi.SqoopConnector; import org.apache.sqoop.driver.Driver; import org.apache.sqoop.driver.DriverUpgrader; import org.apache.sqoop.json.JSONUtils; -import org.apache.sqoop.json.JobBean; -import org.apache.sqoop.json.LinkBean; -import org.apache.sqoop.json.SubmissionBean; +import org.apache.sqoop.json.JobsBean; +import org.apache.sqoop.json.LinksBean; +import org.apache.sqoop.json.SubmissionsBean; import org.apache.sqoop.model.ConfigUtils; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConnector; @@ -136,19 +136,23 @@ public class RepositoryLoadTool extends ConfiguredTool { ConnectorManager.getInstance().initialize(); LOG.info("Loading Connections"); - JSONObject jsonConns = (JSONObject) repo.get(JSONConstants.LINKS); + JSONObject jsonLinks = (JSONObject) repo.get(JSONConstants.LINKS); - if (jsonConns == null) { + if (jsonLinks == null) { LOG.error("Malformed JSON file. Key " + JSONConstants.LINKS + " not found."); return false; } - LinkBean linkBean = new LinkBean(); - linkBean.restore(updateConnectorIDUsingName(jsonConns)); + updateConnectorIDUsingName( + (JSONArray)jsonLinks.get(JSONConstants.LINKS), + JSONConstants.CONNECTOR_ID); - HashMap<Long, Long> connectionIds = new HashMap<Long, Long>(); + LinksBean linksBean = new LinksBean(); + linksBean.restore(jsonLinks); - for (MLink link : linkBean.getLinks()) { + HashMap<Long, Long> linkIds = new HashMap<Long, Long>(); + + for (MLink link : linksBean.getLinks()) { long oldId = link.getPersistenceId(); long newId = loadLink(link); if (newId == link.PERSISTANCE_ID_DEFAULT) { @@ -156,9 +160,9 @@ public class RepositoryLoadTool extends ConfiguredTool { + " failed. Aborting repository load. Check log for details."); return false; } - connectionIds.put(oldId, newId); + linkIds.put(oldId, newId); } - LOG.info("Loaded " + connectionIds.size() + " connections"); + LOG.info("Loaded " + linkIds.size() + " links"); LOG.info("Loading Jobs"); JSONObject jsonJobs = (JSONObject) repo.get(JSONConstants.JOBS); @@ -168,12 +172,20 @@ public class RepositoryLoadTool extends ConfiguredTool { return false; } - JobBean jobBean = new JobBean(); - jobBean.restore(updateIdUsingMap(updateConnectorIDUsingName(jsonJobs), connectionIds, - JSONConstants.LINK_ID)); + updateConnectorIDUsingName( + (JSONArray)jsonJobs.get(JSONConstants.JOBS), + JSONConstants.FROM_CONNECTOR_ID); + updateConnectorIDUsingName( + (JSONArray)jsonJobs.get(JSONConstants.JOBS), + JSONConstants.TO_CONNECTOR_ID); + updateIdUsingMap((JSONArray)jsonJobs.get(JSONConstants.JOBS), + linkIds, JSONConstants.LINK_ID); + + JobsBean jobsBean = new JobsBean(); + jobsBean.restore(jsonJobs); HashMap<Long, Long> jobIds = new HashMap<Long, Long>(); - for (MJob job : jobBean.getJobs()) { + for (MJob job : jobsBean.getJobs()) { long oldId = job.getPersistenceId(); long newId = loadJob(job); @@ -195,10 +207,12 @@ public class RepositoryLoadTool extends ConfiguredTool { return false; } - SubmissionBean submissionBean = new SubmissionBean(); - submissionBean.restore(updateIdUsingMap(jsonSubmissions, jobIds, JSONConstants.JOB_ID)); + updateIdUsingMap((JSONArray)jsonSubmissions.get(JSONConstants.SUBMISSIONS), jobIds, JSONConstants.JOB_ID); + + SubmissionsBean submissionsBean = new SubmissionsBean(); + submissionsBean.restore(jsonSubmissions); int submissionCount = 0; - for (MSubmission submission : submissionBean.getSubmissions()) { + for (MSubmission submission : submissionsBean.getSubmissions()) { resetPersistenceId(submission); repository.createSubmission(submission); submissionCount++; @@ -238,7 +252,7 @@ public class RepositoryLoadTool extends ConfiguredTool { private long loadLink(MLink link) { - // starting by pretending we have a brand new connection + // starting by pretending we have a brand new link resetPersistenceId(link); Repository repository = RepositoryManager.getInstance().getRepository(); @@ -345,9 +359,7 @@ public class RepositoryLoadTool extends ConfiguredTool { } - private JSONObject updateConnectorIDUsingName(JSONObject json) { - JSONArray array = (JSONArray) json.get(JSONConstants.ALL); - + private JSONArray updateConnectorIDUsingName(JSONArray jsonArray, String connectorIdKey) { Repository repository = RepositoryManager.getInstance().getRepository(); List<MConnector> connectors = repository.findConnectors(); @@ -357,34 +369,32 @@ public class RepositoryLoadTool extends ConfiguredTool { connectorMap.put(connector.getUniqueName(), connector.getPersistenceId()); } - for (Object obj : array) { + for (Object obj : jsonArray) { JSONObject object = (JSONObject) obj; - long connectorId = (Long) object.get(JSONConstants.CONNECTOR_ID); + long connectorId = (Long) object.get(connectorIdKey); String connectorName = (String) object.get(JSONConstants.CONNECTOR_NAME); long currentConnectorId = connectorMap.get(connectorName); - String connectionName = (String) object.get(JSONConstants.NAME); + String linkName = (String) object.get(JSONConstants.NAME); // If a given connector now has a different ID, we need to update the ID if (connectorId != currentConnectorId) { - LOG.warn("Connection " + connectionName + " uses connector " + connectorName + ". " + LOG.warn("Link " + linkName + " uses connector " + connectorName + ". " + "Replacing previous ID " + connectorId + " with new ID " + currentConnectorId); - object.put(JSONConstants.CONNECTOR_ID, currentConnectorId); + object.put(connectorIdKey, currentConnectorId); } } - return json; - } - private JSONObject updateIdUsingMap(JSONObject json, HashMap<Long, Long> idMap, String fieldName) { - JSONArray array = (JSONArray) json.get(JSONConstants.ALL); + return jsonArray; + } - for (Object obj : array) { + private JSONArray updateIdUsingMap(JSONArray jsonArray, HashMap<Long, Long> idMap, String fieldName) { + for (Object obj : jsonArray) { JSONObject object = (JSONObject) obj; object.put(fieldName, idMap.get(object.get(fieldName))); } - return json; + return jsonArray; } - }
