SQOOP-1509: Sqoop2: Sqoop2 Rest API refactoring (Veena Basavaraj via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/24e9b4c5 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/24e9b4c5 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/24e9b4c5 Branch: refs/heads/sqoop2 Commit: 24e9b4c5b8a13d544a38e5a1d1fe997898a99b5d Parents: a79ec05 Author: Abraham Elmahrek <[email protected]> Authored: Fri Oct 24 13:03:29 2014 -0700 Committer: Abraham Elmahrek <[email protected]> Committed: Fri Oct 24 13:03:29 2014 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/client/SqoopClient.java | 3 +- .../request/ConnectorResourceRequest.java | 20 +- .../request/DriverConfigResourceRequest.java | 39 ---- .../client/request/DriverResourceRequest.java | 39 ++++ .../client/request/SqoopResourceRequests.java | 12 +- .../org/apache/sqoop/json/ConfigurableBean.java | 25 ++ .../org/apache/sqoop/json/ConnectorBean.java | 109 +++++---- .../org/apache/sqoop/json/ConnectorsBean.java | 61 +++++ .../java/org/apache/sqoop/json/DriverBean.java | 37 ++- .../java/org/apache/sqoop/json/JobBean.java | 73 ++---- .../java/org/apache/sqoop/json/JsonBean.java | 14 ++ .../java/org/apache/sqoop/json/LinkBean.java | 36 +-- .../json/util/ConfigBundleSerialization.java | 75 ++++++ .../sqoop/json/util/ConfigInputConstants.java | 40 ++++ .../json/util/ConfigInputSerialization.java | 203 +++++++++++++++++ .../sqoop/json/util/ConfigSerialization.java | 226 ------------------- .../json/util/ResourceBundleSerialization.java | 78 ------- .../sqoop/json/util/SchemaSerialization.java | 6 +- .../org/apache/sqoop/model/MConfigList.java | 54 +++-- .../org/apache/sqoop/model/MDriverConfig.java | 2 +- .../org/apache/sqoop/model/MFromConfig.java | 2 +- .../org/apache/sqoop/model/MLinkConfig.java | 2 +- .../java/org/apache/sqoop/model/MToConfig.java | 2 +- .../apache/sqoop/utils/MapResourceBundle.java | 2 +- .../org/apache/sqoop/json/ConfigTestUtil.java | 17 +- .../apache/sqoop/json/TestConnectorBean.java | 13 +- .../org/apache/sqoop/json/TestLinkBean.java | 20 +- .../json/util/TestConfigSerialization.java | 31 +-- .../org/apache/sqoop/model/TestMConfigList.java | 2 +- .../sqoop/connector/ConnectorManager.java | 61 +++-- .../java/org/apache/sqoop/driver/Driver.java | 8 +- .../sqoop/driver/DriverConfigValidator.java | 3 +- .../org/apache/sqoop/driver/DriverError.java | 6 +- .../apache/sqoop/repository/JdbcRepository.java | 1 - .../sqoop/handler/ConnectorRequestHandler.java | 87 ++++--- .../handler/DriverConfigRequestHandler.java | 49 ---- .../sqoop/handler/DriverRequestHandler.java | 45 ++++ .../apache/sqoop/handler/JobRequestHandler.java | 7 +- .../sqoop/server/v1/ConfigurableServlet.java | 57 +++++ .../sqoop/server/v1/ConnectorServlet.java | 12 +- .../sqoop/server/v1/ConnectorsServlet.java | 43 ++++ .../sqoop/server/v1/DriverConfigServlet.java | 41 ---- .../apache/sqoop/server/v1/DriverServlet.java | 44 ++++ server/src/main/webapp/WEB-INF/web.xml | 35 ++- .../sqoop/tools/tool/RepositoryDumpTool.java | 22 +- 45 files changed, 981 insertions(+), 783 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/client/src/main/java/org/apache/sqoop/client/SqoopClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java index bffb599..33a0c3c 100644 --- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java +++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java @@ -65,7 +65,7 @@ public class SqoopClient { */ private Map<Long, MConnector> connectors; /** - * All cached bundles for all connectors. + * All cached config params for every registered connector in the sqoop system. */ private Map<Long, ResourceBundle> connectorConfigBundles; @@ -135,7 +135,6 @@ public class SqoopClient { if(connectors.containsKey(cid)) { return connectors.get(cid).clone(false); } - retrieveConnector(cid); return connectors.get(cid).clone(false); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java index e2aae98..5d41b3f 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java @@ -18,16 +18,16 @@ package org.apache.sqoop.client.request; import org.apache.sqoop.json.ConnectorBean; +import org.apache.sqoop.json.ConnectorsBean; import org.json.simple.JSONObject; import org.json.simple.JSONValue; /** - * Provide Read semantics over RESTfull HTTP API for connectors. Only read - * is supported as creation, update and delete might be done only directly on + * Provide Read semantics over RESTfull HTTP API for connectors. Only read is + * supported as creation, update and delete might be done only directly on * server side. */ -public class ConnectorResourceRequest extends ResourceRequest -{ +public class ConnectorResourceRequest extends ResourceRequest { public static final String RESOURCE = "v1/connector/"; public ConnectorBean read(String serverUrl, Long cid) { @@ -37,9 +37,13 @@ public class ConnectorResourceRequest extends ResourceRequest } else { response = super.get(serverUrl + RESOURCE + cid); } - JSONObject jsonObject = (JSONObject)JSONValue.parse(response); - ConnectorBean connectorBean = new ConnectorBean(); - connectorBean.restore(jsonObject); - return connectorBean; + JSONObject jsonObject = (JSONObject) JSONValue.parse(response); + // defaults to all + ConnectorBean bean = new ConnectorsBean(); + if (cid != null) { + bean = new ConnectorBean(); + } + bean.restore(jsonObject); + return bean; } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java deleted file mode 100644 index f6066fc..0000000 --- a/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.client.request; - -import org.apache.sqoop.json.DriverBean; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; - -/** - * Provide CRUD semantics over RESTfull HTTP API for driverConfig - */ -public class DriverConfigResourceRequest extends ResourceRequest { - - public static final String RESOURCE = "v1/config/driver"; - - public DriverBean read(String serverUrl) { - String response = null; - response = super.get(serverUrl + RESOURCE); - JSONObject jsonObject = (JSONObject) JSONValue.parse(response); - DriverBean driverBean = new DriverBean(); - driverBean.restore(jsonObject); - return driverBean; - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/client/src/main/java/org/apache/sqoop/client/request/DriverResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/DriverResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/DriverResourceRequest.java new file mode 100644 index 0000000..ca82f59 --- /dev/null +++ b/client/src/main/java/org/apache/sqoop/client/request/DriverResourceRequest.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.client.request; + +import org.apache.sqoop.json.DriverBean; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +/** + * Provide read semantics over RESTfull HTTP API for driver + */ +public class DriverResourceRequest extends ResourceRequest { + + public static final String RESOURCE = "v1/driver/"; + + public DriverBean read(String serverUrl) { + String response = null; + response = super.get(serverUrl + RESOURCE); + JSONObject jsonObject = (JSONObject) JSONValue.parse(response); + DriverBean driverBean = new DriverBean(); + driverBean.restore(jsonObject); + return driverBean; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java index a4dee75..4a56bb7 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java +++ b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java @@ -33,7 +33,7 @@ public class SqoopResourceRequests { private String serverUrl; - private DriverConfigResourceRequest driverConfigRequest; + private DriverResourceRequest driverRequest; private ConnectorResourceRequest connectorRequest; private LinkResourceRequest linkRequest; private JobResourceRequest jobRequest; @@ -43,12 +43,12 @@ public class SqoopResourceRequests { this.serverUrl = serverUrl; } - public DriverConfigResourceRequest getDriverConfigResourceRequest() { - if (driverConfigRequest == null) { - driverConfigRequest = new DriverConfigResourceRequest(); + public DriverResourceRequest getDriverResourceRequest() { + if (driverRequest == null) { + driverRequest = new DriverResourceRequest(); } - return driverConfigRequest; + return driverRequest; } public ConnectorResourceRequest getConnectorResourceRequest() { @@ -84,7 +84,7 @@ public class SqoopResourceRequests { } public DriverBean readDriver() { - return getDriverConfigResourceRequest().read(serverUrl); + return getDriverResourceRequest().read(serverUrl); } public ConnectorBean readConnector(Long cid) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/ConfigurableBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/ConfigurableBean.java b/common/src/main/java/org/apache/sqoop/json/ConfigurableBean.java new file mode 100644 index 0000000..49dd057 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/json/ConfigurableBean.java @@ -0,0 +1,25 @@ +package org.apache.sqoop.json; +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Marker class for the configurables supported in sqoop + */ +public abstract class ConfigurableBean implements JsonBean { + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java index d12b6b0..845cafa 100644 --- a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java +++ b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java @@ -17,25 +17,16 @@ */ package org.apache.sqoop.json; -import static org.apache.sqoop.json.util.ConfigSerialization.ALL; -import static org.apache.sqoop.json.util.ConfigSerialization.CLASS; -import static org.apache.sqoop.json.util.ConfigSerialization.ID; -import static org.apache.sqoop.json.util.ConfigSerialization.CONNECTOR_JOB_CONFIG; -import static org.apache.sqoop.json.util.ConfigSerialization.CONNECTOR_LINK_CONFIG; -import static org.apache.sqoop.json.util.ConfigSerialization.NAME; -import static org.apache.sqoop.json.util.ConfigSerialization.VERSION; -import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList; -import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONNECTOR_CONFIGS; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle; +import static org.apache.sqoop.json.util.ConfigInputSerialization.extractConfigList; +import static org.apache.sqoop.json.util.ConfigInputSerialization.restoreConfigList; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.extractConfigParamBundle; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.restoreConfigParamBundle; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.ResourceBundle; -import java.util.Set; import org.apache.sqoop.common.Direction; import org.apache.sqoop.model.MConfig; @@ -50,10 +41,14 @@ import org.json.simple.JSONObject; * Json representation of the connector object * */ -public class ConnectorBean implements JsonBean { +public class ConnectorBean extends ConfigurableBean { - private List<MConnector> connectors; + // to represent the config and inputs with values + public static final String CONNECTOR_LINK_CONFIG_VALUES = "link-config-values"; + public static final String CONNECTOR_JOB_CONFIG_VALUES = "job-config-values"; + private static final String CONNECTOR = "connector"; + private List<MConnector> connectors; private Map<Long, ResourceBundle> connectorConfigBundles; // for "extract" @@ -77,53 +72,60 @@ public class ConnectorBean implements JsonBean { @SuppressWarnings("unchecked") @Override public JSONObject extract(boolean skipSensitive) { + JSONArray connectorArray = extractConnectors(skipSensitive); + JSONObject connector = new JSONObject(); + connector.put(CONNECTOR, connectorArray); + return connector; + } + @SuppressWarnings("unchecked") + protected JSONArray extractConnectors(boolean skipSensitive) { JSONArray connectorArray = new JSONArray(); - for (MConnector connector : connectors) { JSONObject connectorJsonObject = new JSONObject(); - connectorJsonObject.put(ID, connector.getPersistenceId()); connectorJsonObject.put(NAME, connector.getUniqueName()); connectorJsonObject.put(CLASS, connector.getClassName()); - connectorJsonObject.put(VERSION, connector.getVersion()); - connectorJsonObject.put(CONNECTOR_LINK_CONFIG, - extractConfigList(connector.getLinkConfig().getConfigs(), skipSensitive)); + connectorJsonObject.put(CONFIGURABLE_VERSION, connector.getVersion()); + connectorJsonObject.put( + CONNECTOR_LINK_CONFIG_VALUES, + extractConfigList(connector.getLinkConfig().getConfigs(), connector.getLinkConfig() + .getType(), skipSensitive)); - connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject()); + connectorJsonObject.put(CONNECTOR_JOB_CONFIG_VALUES, new JSONObject()); // add sub fields to the job config for from and to if (connector.getFromConfig() != null) { - ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(Direction.FROM, - extractConfigList(connector.getFromConfig().getConfigs(), skipSensitive)); + ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG_VALUES)).put( + Direction.FROM, + extractConfigList(connector.getFromConfig().getConfigs(), connector.getFromConfig() + .getType(), skipSensitive)); } if (connector.getToConfig() != null) { - ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(Direction.TO, - extractConfigList(connector.getToConfig().getConfigs(), skipSensitive)); + ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG_VALUES)).put( + Direction.TO, + extractConfigList(connector.getToConfig().getConfigs(), connector.getToConfig() + .getType(), skipSensitive)); } - connectorArray.add(connectorJsonObject); - } - - JSONObject all = new JSONObject(); - all.put(ALL, connectorArray); - - if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) { - JSONObject jsonBundles = new JSONObject(); - - for (Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) { - jsonBundles.put(entry.getKey().toString(), extractResourceBundle(entry.getValue())); + // add the config-param inside each connector + connectorJsonObject.put(ALL_CONFIGS, new JSONObject()); + if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) { + connectorJsonObject.put(ALL_CONFIGS, + extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId()))); } - all.put(CONNECTOR_CONFIGS, jsonBundles); + connectorArray.add(connectorJsonObject); } - - return all; + return connectorArray; } @Override - @SuppressWarnings("unchecked") public void restore(JSONObject jsonObject) { - connectors = new ArrayList<MConnector>(); + JSONArray array = (JSONArray) jsonObject.get(CONNECTOR); + restoreConnectors(array); + } - JSONArray array = (JSONArray) jsonObject.get(ALL); + protected void restoreConnectors(JSONArray array) { + connectors = new ArrayList<MConnector>(); + connectorConfigBundles = new HashMap<Long, ResourceBundle>(); for (Object obj : array) { JSONObject object = (JSONObject) obj; @@ -131,12 +133,13 @@ public class ConnectorBean implements JsonBean { long connectorId = (Long) object.get(ID); String uniqueName = (String) object.get(NAME); String className = (String) object.get(CLASS); - String version = (String) object.get(VERSION); + String version = (String) object.get(CONFIGURABLE_VERSION); - List<MConfig> linkConfigs = restoreConfigList((JSONArray) object.get(CONNECTOR_LINK_CONFIG)); + List<MConfig> linkConfigs = restoreConfigList((JSONArray) object + .get(CONNECTOR_LINK_CONFIG_VALUES)); - // parent that encapsualtes both the from/to configs - JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG); + // parent that encapsulates both the from/to configs + JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG_VALUES); JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name()); JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name()); @@ -158,18 +161,12 @@ public class ConnectorBean implements JsonBean { toConfig); connector.setPersistenceId(connectorId); - connectors.add(connector); - } + if (object.containsKey(ALL_CONFIGS)) { - if (jsonObject.containsKey(CONNECTOR_CONFIGS)) { - connectorConfigBundles = new HashMap<Long, ResourceBundle>(); - - JSONObject jsonBundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS); - Set<Map.Entry<String, JSONObject>> entrySet = jsonBundles.entrySet(); - for (Map.Entry<String, JSONObject> entry : entrySet) { - connectorConfigBundles.put(Long.parseLong(entry.getKey()), - restoreResourceBundle(entry.getValue())); + JSONObject jsonConfigBundle = (JSONObject) object.get(ALL_CONFIGS); + connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle)); } + connectors.add(connector); } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java new file mode 100644 index 0000000..4cd3698 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.json; + +import java.util.List; +import java.util.Map; +import java.util.ResourceBundle; + +import org.apache.sqoop.model.MConnector; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +/** + * Json representation of the connectors object + * + */ +public class ConnectorsBean extends ConnectorBean { + + // to represent the config and inputs with values + private static final String CONNECTORS = "connectors"; + + // for "extract" + public ConnectorsBean(List<MConnector> connectors, Map<Long, ResourceBundle> bundles) { + super(connectors, bundles); + } + + // for "restore" + public ConnectorsBean() { + } + + @SuppressWarnings("unchecked") + @Override + public JSONObject extract(boolean skipSensitive) { + + JSONArray connectorArray = extractConnectors(skipSensitive); + JSONObject connectors = new JSONObject(); + connectors.put(CONNECTORS, connectorArray); + return connectors; + } + + @Override + public void restore(JSONObject jsonObject) { + JSONArray array = (JSONArray) jsonObject.get(CONNECTORS); + super.restoreConnectors(array); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/DriverBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/DriverBean.java b/common/src/main/java/org/apache/sqoop/json/DriverBean.java index 90cdbef..f200268 100644 --- a/common/src/main/java/org/apache/sqoop/json/DriverBean.java +++ b/common/src/main/java/org/apache/sqoop/json/DriverBean.java @@ -17,14 +17,10 @@ */ package org.apache.sqoop.json; -import static org.apache.sqoop.json.util.ConfigSerialization.DRIVER_CONFIG; -import static org.apache.sqoop.json.util.ConfigSerialization.DRIVER_VERSION; -import static org.apache.sqoop.json.util.ConfigSerialization.ID; -import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList; -import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONFIGS; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle; +import static org.apache.sqoop.json.util.ConfigInputSerialization.extractConfigList; +import static org.apache.sqoop.json.util.ConfigInputSerialization.restoreConfigList; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.extractConfigParamBundle; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.restoreConfigParamBundle; import java.util.List; import java.util.ResourceBundle; @@ -36,20 +32,19 @@ import org.json.simple.JSONArray; import org.json.simple.JSONObject; /** * Json representation of the driver - * */ -public class DriverBean implements JsonBean { +public class DriverBean extends ConfigurableBean { public static final String CURRENT_DRIVER_VERSION = "1"; + static final String DRIVER_JOB_CONFIG_VALUES = "job-config-values"; private MDriver driver; - - private ResourceBundle bundle; + private ResourceBundle driverConfigBundle; // for "extract" public DriverBean(MDriver driver, ResourceBundle bundle) { this.driver = driver; - this.bundle = bundle; + this.driverConfigBundle = bundle; } // for "restore" @@ -61,30 +56,30 @@ public class DriverBean implements JsonBean { } public ResourceBundle getDriverConfigResourceBundle() { - return bundle; + return driverConfigBundle; } @SuppressWarnings("unchecked") @Override public JSONObject extract(boolean skipSensitive) { JSONArray configs = - extractConfigList(driver.getDriverConfig().getConfigs(), skipSensitive); + extractConfigList(driver.getDriverConfig().getConfigs(), driver.getDriverConfig().getType(), skipSensitive); JSONObject result = new JSONObject(); result.put(ID, driver.getPersistenceId()); - result.put(DRIVER_VERSION, driver.getVersion()); - result.put(DRIVER_CONFIG, configs); - result.put(CONFIGS, extractResourceBundle(bundle)); + result.put(CONFIGURABLE_VERSION, driver.getVersion()); + result.put(DRIVER_JOB_CONFIG_VALUES, configs); + result.put(ALL_CONFIGS, extractConfigParamBundle(driverConfigBundle)); return result; } @Override public void restore(JSONObject jsonObject) { long id = (Long) jsonObject.get(ID); - String driverVersion = (String) jsonObject.get(DRIVER_VERSION); - List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_CONFIG)); + String driverVersion = (String) jsonObject.get(CONFIGURABLE_VERSION); + List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_JOB_CONFIG_VALUES)); driver = new MDriver(new MDriverConfig(driverConfig), driverVersion); driver.setPersistenceId(id); - bundle = restoreResourceBundle((JSONObject) jsonObject.get(CONFIGS)); + driverConfigBundle = restoreConfigParamBundle((JSONObject) jsonObject.get(ALL_CONFIGS)); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/JobBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/JobBean.java b/common/src/main/java/org/apache/sqoop/json/JobBean.java index 8f42edb..082d591 100644 --- a/common/src/main/java/org/apache/sqoop/json/JobBean.java +++ b/common/src/main/java/org/apache/sqoop/json/JobBean.java @@ -17,17 +17,10 @@ */ package org.apache.sqoop.json; -import static org.apache.sqoop.json.util.ConfigSerialization.CREATION_DATE; -import static org.apache.sqoop.json.util.ConfigSerialization.CREATION_USER; -import static org.apache.sqoop.json.util.ConfigSerialization.ENABLED; -import static org.apache.sqoop.json.util.ConfigSerialization.UPDATE_DATE; -import static org.apache.sqoop.json.util.ConfigSerialization.UPDATE_USER; -import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList; -import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONNECTOR_CONFIGS; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.DRIVER_CONFIGS; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle; +import static org.apache.sqoop.json.util.ConfigInputSerialization.extractConfigList; +import static org.apache.sqoop.json.util.ConfigInputSerialization.restoreConfigList; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.extractConfigParamBundle; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.restoreConfigParamBundle; import java.util.ArrayList; import java.util.Date; @@ -35,11 +28,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.ResourceBundle; -import java.util.Set; import org.apache.sqoop.common.Direction; import org.apache.sqoop.model.MConfig; -import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MFromConfig; import org.apache.sqoop.model.MJob; @@ -52,16 +43,13 @@ import org.json.simple.JSONObject; */ public class JobBean implements JsonBean { - private static final String ALL = "all"; - private static final String ID = "id"; - private static final String NAME = "name"; - private static final String FROM_LINK_ID = "from-link-id"; - private static final String TO_LINK_ID = "to-link-id"; - private static final String FROM_CONNECTOR_ID = "from-connector-id"; - private static final String TO_CONNECTOR_ID = "to-connector-id"; - private static final String FROM_CONFIG = "from-config"; - private static final String TO_CONFIG = "to-config"; - private static final String DRIVER_CONFIG = "driver-config"; + static final String FROM_LINK_ID = "from-link-id"; + static final String TO_LINK_ID = "to-link-id"; + static final String FROM_CONNECTOR_ID = "from-connector-id"; + static final String TO_CONNECTOR_ID = "to-connector-id"; + static final String FROM_CONFIG = "from-config"; + static final String TO_CONFIG = "to-config"; + static final String DRIVER_CONFIG = "driver-config"; // Required private List<MJob> jobs; @@ -133,36 +121,22 @@ public class JobBean implements JsonBean { object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM)); object.put(TO_LINK_ID, job.getLinkId(Direction.TO)); // job configs - object.put(FROM_CONFIG, extractConfigList(job - .getJobConfig(Direction.FROM).getConfigs(), skipSensitive)); - object.put(TO_CONFIG, - extractConfigList(job.getJobConfig(Direction.TO).getConfigs(), skipSensitive)); - object.put(DRIVER_CONFIG, - extractConfigList(job.getDriverConfig().getConfigs(), skipSensitive)); + MFromConfig fromConfigList = job.getFromJobConfig(); + object.put(FROM_CONFIG, extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive)); + MToConfig toConfigList = job.getToJobConfig(); + object.put(TO_CONFIG, extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive)); + MDriverConfig driverConfigList = job.getDriverConfig(); + object.put(DRIVER_CONFIG, extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(), skipSensitive)); array.add(object); } JSONObject all = new JSONObject(); all.put(ALL, array); - - if(!connectorConfigBundles.isEmpty()) { - JSONObject bundles = new JSONObject(); - - for(Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) { - bundles.put(entry.getKey().toString(), - extractResourceBundle(entry.getValue())); - } - all.put(CONNECTOR_CONFIGS, bundles); - } - if(driverConfigBundle != null) { - all.put(DRIVER_CONFIGS,extractResourceBundle(driverConfigBundle)); - } return all; } @Override - @SuppressWarnings("unchecked") public void restore(JSONObject jsonObject) { jobs = new ArrayList<MJob>(); @@ -203,18 +177,5 @@ public class JobBean implements JsonBean { jobs.add(job); } - - if(jsonObject.containsKey(CONNECTOR_CONFIGS)) { - JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS); - Set<Map.Entry<String, JSONObject>> entrySet = bundles.entrySet(); - for (Map.Entry<String, JSONObject> entry : entrySet) { - connectorConfigBundles.put(Long.parseLong(entry.getKey()), - restoreResourceBundle(entry.getValue())); - } - } - if(jsonObject.containsKey(DRIVER_CONFIGS)) { - driverConfigBundle = restoreResourceBundle( - (JSONObject) jsonObject.get(DRIVER_CONFIGS)); - } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/JsonBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/JsonBean.java b/common/src/main/java/org/apache/sqoop/json/JsonBean.java index 8189259..ba86511 100644 --- a/common/src/main/java/org/apache/sqoop/json/JsonBean.java +++ b/common/src/main/java/org/apache/sqoop/json/JsonBean.java @@ -21,6 +21,20 @@ import org.json.simple.JSONObject; public interface JsonBean { + // common JSON constants for the rest-api response + static final String CONFIGURABLE_VERSION = "version"; + static final String ALL_CONFIGS = "all-configs"; + + static final String ALL = "all"; + static final String ID = "id"; + static final String NAME = "name"; + static final String CLASS = "class"; + static final String ENABLED = "enabled"; + static final String CREATION_USER = "creation-user"; + static final String CREATION_DATE = "creation-date"; + static final String UPDATE_USER = "update-user"; + static final String UPDATE_DATE = "update-date"; + JSONObject extract(boolean skipSensitive); void restore(JSONObject jsonObject); http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/LinkBean.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/LinkBean.java b/common/src/main/java/org/apache/sqoop/json/LinkBean.java index af0fc9d..2762500 100644 --- a/common/src/main/java/org/apache/sqoop/json/LinkBean.java +++ b/common/src/main/java/org/apache/sqoop/json/LinkBean.java @@ -17,11 +17,10 @@ */ package org.apache.sqoop.json; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MLinkConfig; -import org.apache.sqoop.model.MConfig; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; +import static org.apache.sqoop.json.util.ConfigInputSerialization.extractConfigList; +import static org.apache.sqoop.json.util.ConfigInputSerialization.restoreConfigList; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.extractConfigParamBundle; +import static org.apache.sqoop.json.util.ConfigBundleSerialization.restoreConfigParamBundle; import java.util.ArrayList; import java.util.Date; @@ -29,10 +28,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.ResourceBundle; -import java.util.Set; -import static org.apache.sqoop.json.util.ConfigSerialization.*; -import static org.apache.sqoop.json.util.ResourceBundleSerialization.*; +import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.model.MLinkConfig; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; /** * Link representation that is being send across the network between @@ -101,25 +102,17 @@ public class LinkBean implements JsonBean { linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime()); linkJsonObject.put(CONNECTOR_ID, link.getConnectorId()); linkJsonObject.put(LINK_CONFIG, - extractConfigList(link.getConnectorLinkConfig().getConfigs(), skipSensitive)); + extractConfigList(link.getConnectorLinkConfig().getConfigs(), link.getConnectorLinkConfig().getType(), skipSensitive)); linkArray.add(linkJsonObject); } JSONObject all = new JSONObject(); all.put(ALL, linkArray); - if (!linkConfigBundles.isEmpty()) { - JSONObject bundles = new JSONObject(); - for (Map.Entry<Long, ResourceBundle> entry : linkConfigBundles.entrySet()) { - bundles.put(entry.getKey().toString(), extractResourceBundle(entry.getValue())); - } - all.put(CONNECTOR_CONFIGS, bundles); - } return all; } @Override - @SuppressWarnings("unchecked") public void restore(JSONObject jsonObject) { links = new ArrayList<MLink>(); @@ -145,14 +138,5 @@ public class LinkBean implements JsonBean { links.add(link); } - - if(jsonObject.containsKey(CONNECTOR_CONFIGS)) { - JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS); - Set<Map.Entry<String, JSONObject>> entrySet = bundles.entrySet(); - for (Map.Entry<String, JSONObject> entry : entrySet) { - linkConfigBundles.put(Long.parseLong(entry.getKey()), - restoreResourceBundle(entry.getValue())); - } - } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/util/ConfigBundleSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigBundleSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigBundleSerialization.java new file mode 100644 index 0000000..6133798 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigBundleSerialization.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.json.util; + +import org.apache.sqoop.utils.MapResourceBundle; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +import java.util.Enumeration; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.ResourceBundle; + +/** + * + */ +public final class ConfigBundleSerialization { + + @SuppressWarnings("unchecked") + public static JSONArray extractConfigParamBundles(List<ResourceBundle> bundles) { + JSONArray array = new JSONArray(); + for (ResourceBundle bundle : bundles) { + array.add(extractConfigParamBundle(bundle)); + } + return array; + } + + @SuppressWarnings("unchecked") + public static JSONObject extractConfigParamBundle(ResourceBundle bundle) { + // TODO:(SQOOP-1618) can we preserve the order of the config params and use jackson + JSONObject json = new JSONObject(); + Enumeration<String> keys = bundle.getKeys(); + while(keys.hasMoreElements()) { + String key = keys.nextElement(); + json.put(key, bundle.getString(key)); + } + return json; + } + + public static List<ResourceBundle> restoreConfigParamBundles(JSONArray array) { + List<ResourceBundle> bundles = new LinkedList<ResourceBundle>(); + for (Object item : array) { + bundles.add(restoreConfigParamBundle((JSONObject) item)); + } + return bundles; + } + + @SuppressWarnings("unchecked") + public static ResourceBundle restoreConfigParamBundle(JSONObject json) { + Map<String, Object> map = new HashMap<String, Object>(); + map.putAll(json); + return new MapResourceBundle(map); + } + + private ConfigBundleSerialization() { + // Instantiation of this class is prohibited + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java new file mode 100644 index 0000000..df8a870 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.json.util; + +/** + * Constants related to the configs + */ +public class ConfigInputConstants { + + public static final String CONFIG_ID = "id"; + public static final String INPUT_ID = "id"; + public static final String CONFIG_NAME = "name"; + public static final String CONFIG_TYPE = "type"; + public static final String CONFIG_INPUTS = "inputs"; + public static final String CONFIG_INPUT_NAME = "name"; + public static final String CONFIG_INPUT_TYPE = "type"; + public static final String CONFIG_INPUT_SENSITIVE = "sensitive"; + public static final String CONFIG_INPUT_SIZE = "size"; + public static final String CONFIG_INPUT_VALUE = "value"; + public static final String CONFIG_INPUT_VALUES = "values"; + + private ConfigInputConstants() { + + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java new file mode 100644 index 0000000..4667f5c --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.json.util; + +import org.apache.commons.lang.StringUtils; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.model.MBooleanInput; +import org.apache.sqoop.model.MEnumInput; +import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MConfigType; +import org.apache.sqoop.model.MInput; +import org.apache.sqoop.model.MInputType; +import org.apache.sqoop.model.MIntegerInput; +import org.apache.sqoop.model.MMapInput; +import org.apache.sqoop.model.MStringInput; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Convenient static methods for serializing config and input objects. + */ +public final class ConfigInputSerialization { + + /** + * Transform given list of configs to JSON Array object. + * + * @param mConfigs List of configs. + * @return JSON object with serialized config of the list. + */ + @SuppressWarnings("unchecked") + public static JSONArray extractConfigList(List<MConfig> mConfigs, MConfigType type, + boolean skipSensitive) { + JSONArray configs = new JSONArray(); + + for (MConfig mConfig : mConfigs) { + configs.add(extractConfig(mConfig, type, skipSensitive)); + } + return configs; + } + + /** + * Transform given config to JSON Object. + * + * @param mConfig Given MConfig instance + * @param skipSensitive conditionally add sensitive input values + * @return Serialized JSON object. + */ + @SuppressWarnings("unchecked") + static JSONObject extractConfig(MConfig mConfig, MConfigType type, boolean skipSensitive) { + JSONObject config = new JSONObject(); + config.put(ConfigInputConstants.CONFIG_ID, mConfig.getPersistenceId()); + config.put(ConfigInputConstants.CONFIG_NAME, mConfig.getName()); + config.put(ConfigInputConstants.CONFIG_TYPE, type.name()); + JSONArray mInputs = new JSONArray(); + config.put(ConfigInputConstants.CONFIG_INPUTS, mInputs); + + for (MInput<?> mInput : mConfig.getInputs()) { + JSONObject input = new JSONObject(); + input.put(ConfigInputConstants.CONFIG_ID, mInput.getPersistenceId()); + input.put(ConfigInputConstants.CONFIG_INPUT_NAME, mInput.getName()); + input.put(ConfigInputConstants.CONFIG_INPUT_TYPE, mInput.getType().toString()); + input.put(ConfigInputConstants.CONFIG_INPUT_SENSITIVE, mInput.isSensitive()); + + // String specific serialization + if (mInput.getType() == MInputType.STRING) { + input.put(ConfigInputConstants.CONFIG_INPUT_SIZE, + ((MStringInput)mInput).getMaxLength()); + } + + // Enum specific serialization + if(mInput.getType() == MInputType.ENUM) { + input.put(ConfigInputConstants.CONFIG_INPUT_VALUES, + StringUtils.join(((MEnumInput)mInput).getValues(), ",")); + } + + // Serialize value if is there + // Skip if sensitive + if (!mInput.isEmpty() && !(skipSensitive && mInput.isSensitive())) { + if (mInput.getType() == MInputType.MAP) { + input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mInput.getValue()); + } else { + input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mInput.getUrlSafeValueString()); + } + } + + mInputs.add(input); + } + + return config; + } + + /** + * Restore List of MConfigs from JSON Array. + * + * @param configs JSON array representing list of MConfigs + * @return Restored list of MConfigs + */ + public static List<MConfig> restoreConfigList(JSONArray configs) { + List<MConfig> mConfigs = new ArrayList<MConfig>(); + + for (int i = 0; i < configs.size(); i++) { + mConfigs.add(restoreConfig((JSONObject) configs.get(i))); + } + + return mConfigs; + } + + /** + * Restore one MConfig from JSON Object. + * + * @param config JSON representation of the MConfig. + * @return Restored MConfig. + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + static MConfig restoreConfig(JSONObject config) { + JSONArray inputs = (JSONArray) config.get(ConfigInputConstants.CONFIG_INPUTS); + + List<MInput<?>> mInputs = new ArrayList<MInput<?>>(); + for (int i = 0; i < inputs.size(); i++) { + JSONObject input = (JSONObject) inputs.get(i); + MInputType type = + MInputType.valueOf((String) input.get(ConfigInputConstants.CONFIG_INPUT_TYPE)); + String name = (String) input.get(ConfigInputConstants.CONFIG_INPUT_NAME); + Boolean sensitive = (Boolean) input.get(ConfigInputConstants.CONFIG_INPUT_SENSITIVE); + MInput mInput = null; + switch (type) { + case STRING: { + long size = (Long) input.get(ConfigInputConstants.CONFIG_INPUT_SIZE); + mInput = new MStringInput(name, sensitive.booleanValue(), (short) size); + break; + } + case MAP: { + mInput = new MMapInput(name, sensitive.booleanValue()); + break; + } + case INTEGER: { + mInput = new MIntegerInput(name, sensitive.booleanValue()); + break; + } + case BOOLEAN: { + mInput = new MBooleanInput(name, sensitive.booleanValue()); + break; + } + case ENUM: { + String values = (String) input.get(ConfigInputConstants.CONFIG_INPUT_VALUES); + mInput = new MEnumInput(name, sensitive.booleanValue(), values.split(",")); + break; + } + default: + // do nothing + break; + } + + // Propagate config ID + mInput.setPersistenceId((Long)input.get(ConfigInputConstants.INPUT_ID)); + + // Propagate config optional value + if(input.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE)) { + switch (type) { + case MAP: + try { + mInput.setValue((Map<String, String>)input.get(ConfigInputConstants.CONFIG_INPUT_VALUE)); + } catch (ClassCastException e) { + throw new SqoopException(SerializationError.SERIALIZATION_001, name + " requires a 'map' value."); + } + break; + default: + mInput.restoreFromUrlSafeValueString( + (String) input.get(ConfigInputConstants.CONFIG_INPUT_VALUE)); + break; + } + } + mInputs.add(mInput); + } + + MConfig mConfig = new MConfig((String) config.get(ConfigInputConstants.CONFIG_NAME), mInputs); + mConfig.setPersistenceId((Long) config.get(ConfigInputConstants.CONFIG_ID)); + return mConfig; + } + + private ConfigInputSerialization() { + // Do not instantiate + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java deleted file mode 100644 index cec46f6..0000000 --- a/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.json.util; - -import org.apache.commons.lang.StringUtils; -import org.apache.sqoop.common.SqoopException; -import org.apache.sqoop.model.MBooleanInput; -import org.apache.sqoop.model.MEnumInput; -import org.apache.sqoop.model.MConfig; -import org.apache.sqoop.model.MConfigType; -import org.apache.sqoop.model.MInput; -import org.apache.sqoop.model.MInputType; -import org.apache.sqoop.model.MIntegerInput; -import org.apache.sqoop.model.MMapInput; -import org.apache.sqoop.model.MStringInput; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Convenient static methods for serializing config objects. - */ -public final class ConfigSerialization { - - public static final String ALL = "all"; - public static final String ID = "id"; - public static final String NAME = "name"; - public static final String VERSION = "version"; - public static final String CLASS = "class"; - public static final String ENABLED = "enabled"; - public static final String CREATION_USER = "creation-user"; - public static final String CREATION_DATE = "creation-date"; - public static final String UPDATE_USER = "update-user"; - public static final String UPDATE_DATE = "update-date"; - // TODO(VB): Move these constants to connector bean - public static final String CONNECTOR_LINK_CONFIG = "link-config"; - public static final String CONNECTOR_JOB_CONFIG = "job-config"; - // TODO:move these configs to driver bean - public static final String DRIVER_VERSION = "driver-version"; - public static final String DRIVER_CONFIG = "driver-config"; - - public static final String CONFIG_NAME = "name"; - public static final String CONFIG_TYPE = "type"; - public static final String CONFIG_INPUTS = "inputs"; - public static final String CONFIG_INPUT_NAME = "name"; - public static final String CONFIG_INPUT_TYPE = "type"; - public static final String CONFIG_INPUT_SENSITIVE = "sensitive"; - public static final String CONFIG_INPUT_SIZE = "size"; - public static final String CONFIG_INPUT_VALUE = "value"; - public static final String CONFIG_INPUT_VALUES = "values"; - - /** - * Transform given list of configs to JSON Array object. - * - * @param mConfigs List of configs. - * @return JSON object with serialized config of the list. - */ - @SuppressWarnings("unchecked") - public static JSONArray extractConfigList(List<MConfig> mConfigs, boolean skipSensitive) { - JSONArray configs = new JSONArray(); - - for (MConfig mConfig : mConfigs) { - configs.add(extractConfig(mConfig, skipSensitive)); - } - - return configs; - } - - /** - * Transform given config to JSON Object. - * - * @param mConfig Given MConfig instance - * @param skipSensitive conditionally add sensitive input values - * @return Serialized JSON object. - */ - @SuppressWarnings("unchecked") - static JSONObject extractConfig(MConfig mConfig, boolean skipSensitive) { - JSONObject config = new JSONObject(); - config.put(ID, mConfig.getPersistenceId()); - config.put(CONFIG_NAME, mConfig.getName()); - config.put(CONFIG_TYPE, MConfigType.LINK.toString()); - JSONArray mInputs = new JSONArray(); - config.put(CONFIG_INPUTS, mInputs); - - for (MInput<?> mInput : mConfig.getInputs()) { - JSONObject input = new JSONObject(); - input.put(ID, mInput.getPersistenceId()); - input.put(CONFIG_INPUT_NAME, mInput.getName()); - input.put(CONFIG_INPUT_TYPE, mInput.getType().toString()); - input.put(CONFIG_INPUT_SENSITIVE, mInput.isSensitive()); - - // String specific serialization - if (mInput.getType() == MInputType.STRING) { - input.put(CONFIG_INPUT_SIZE, - ((MStringInput)mInput).getMaxLength()); - } - - // Enum specific serialization - if(mInput.getType() == MInputType.ENUM) { - input.put(CONFIG_INPUT_VALUES, - StringUtils.join(((MEnumInput)mInput).getValues(), ",")); - } - - // Serialize value if is there - // Skip if sensitive - if (!mInput.isEmpty() && !(skipSensitive && mInput.isSensitive())) { - if (mInput.getType() == MInputType.MAP) { - input.put(CONFIG_INPUT_VALUE, mInput.getValue()); - } else { - input.put(CONFIG_INPUT_VALUE, mInput.getUrlSafeValueString()); - } - } - - mInputs.add(input); - } - - return config; - } - - /** - * Restore List of MConfigs from JSON Array. - * - * @param configs JSON array representing list of MConfigs - * @return Restored list of MConfigs - */ - public static List<MConfig> restoreConfigList(JSONArray configs) { - List<MConfig> mConfigs = new ArrayList<MConfig>(); - - for (int i = 0; i < configs.size(); i++) { - mConfigs.add(restoreConfig((JSONObject) configs.get(i))); - } - - return mConfigs; - } - - /** - * Restore one MConfig from JSON Object. - * - * @param config JSON representation of the MConfig. - * @return Restored MConfig. - */ - static MConfig restoreConfig(JSONObject config) { - JSONArray inputs = (JSONArray) config.get(CONFIG_INPUTS); - - List<MInput<?>> mInputs = new ArrayList<MInput<?>>(); - for (int i = 0; i < inputs.size(); i++) { - JSONObject input = (JSONObject) inputs.get(i); - MInputType type = - MInputType.valueOf((String) input.get(CONFIG_INPUT_TYPE)); - String name = (String) input.get(CONFIG_INPUT_NAME); - Boolean sensitive = (Boolean) input.get(CONFIG_INPUT_SENSITIVE); - MInput mInput = null; - switch (type) { - case STRING: { - long size = (Long) input.get(CONFIG_INPUT_SIZE); - mInput = new MStringInput(name, sensitive.booleanValue(), (short) size); - break; - } - case MAP: { - mInput = new MMapInput(name, sensitive.booleanValue()); - break; - } - case INTEGER: { - mInput = new MIntegerInput(name, sensitive.booleanValue()); - break; - } - case BOOLEAN: { - mInput = new MBooleanInput(name, sensitive.booleanValue()); - break; - } - case ENUM: { - String values = (String) input.get(CONFIG_INPUT_VALUES); - mInput = new MEnumInput(name, sensitive.booleanValue(), values.split(",")); - break; - } - } - - // Propagate config ID - mInput.setPersistenceId((Long)input.get(ID)); - - // Propagate config optional value - if(input.containsKey(CONFIG_INPUT_VALUE)) { - switch (type) { - case MAP: - try { - mInput.setValue((Map<String, String>)input.get(CONFIG_INPUT_VALUE)); - } catch (ClassCastException e) { - throw new SqoopException(SerializationError.SERIALIZATION_001, name + " requires a 'map' value."); - } - break; - default: - mInput.restoreFromUrlSafeValueString( - (String) input.get(CONFIG_INPUT_VALUE)); - break; - } - } - mInputs.add(mInput); - } - - MConfig mConfig = new MConfig((String) config.get(CONFIG_NAME), mInputs); - mConfig.setPersistenceId((Long) config.get(ID)); - return mConfig; - } - - private ConfigSerialization() { - // Do not instantiate - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java deleted file mode 100644 index f27d81d..0000000 --- a/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.json.util; - -import org.apache.sqoop.utils.MapResourceBundle; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; - -import java.util.Enumeration; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.ResourceBundle; - -/** - * - */ -public final class ResourceBundleSerialization { - - public static final String CONFIGS = "configs"; - public static final String CONNECTOR_CONFIGS = "connector-configs"; - public static final String DRIVER_CONFIGS = "driver-configs"; - - @SuppressWarnings("unchecked") - public static JSONArray extractResourceBundles(List<ResourceBundle> bundles) { - JSONArray array = new JSONArray(); - for (ResourceBundle bundle : bundles) { - array.add(extractResourceBundle(bundle)); - } - return array; - } - - @SuppressWarnings("unchecked") - public static JSONObject extractResourceBundle(ResourceBundle bundle) { - JSONObject json = new JSONObject(); - Enumeration<String> keys = bundle.getKeys(); - while(keys.hasMoreElements()) { - String key = keys.nextElement(); - json.put(key, bundle.getString(key)); - } - return json; - } - - public static List<ResourceBundle> restoreResourceBundles(JSONArray array) { - List<ResourceBundle> bundles = new LinkedList<ResourceBundle>(); - for (Object item : array) { - bundles.add(restoreResourceBundle((JSONObject) item)); - } - return bundles; - } - - @SuppressWarnings("unchecked") - public static ResourceBundle restoreResourceBundle(JSONObject json) { - Map<String, Object> map = new HashMap<String, Object>(); - map.putAll(json); - return new MapResourceBundle(map); - } - - private ResourceBundleSerialization() { - // Instantiation of this class is prohibited - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java index 1e6da6d..4b4b932 100644 --- a/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java +++ b/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java @@ -60,6 +60,7 @@ public class SchemaSerialization { private static final String UNSIGNED = "unsigned"; private static final String JDBC_TYPE = "jdbc-type"; + @SuppressWarnings("unchecked") public static JSONObject extractSchema(Schema schema) { JSONObject object = new JSONObject(); object.put(NAME, schema.getName()); @@ -67,15 +68,11 @@ public class SchemaSerialization { if(schema.getNote() != null) { object.put(NOTE, schema.getNote()); } - JSONArray columnArray = new JSONArray(); - for(Column column : schema.getColumns()) { columnArray.add(extractColumn(column)); } - object.put(COLUMNS, columnArray); - return object; } @@ -96,6 +93,7 @@ public class SchemaSerialization { return schema; } + @SuppressWarnings("unchecked") private static JSONObject extractColumn(Column column) { JSONObject ret = new JSONObject(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/model/MConfigList.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MConfigList.java b/common/src/main/java/org/apache/sqoop/model/MConfigList.java index 8747b55..20309e1 100644 --- a/common/src/main/java/org/apache/sqoop/model/MConfigList.java +++ b/common/src/main/java/org/apache/sqoop/model/MConfigList.java @@ -28,9 +28,11 @@ import java.util.List; public class MConfigList implements MClonable { private final List<MConfig> configObjects; + private final MConfigType type; - public MConfigList(List<MConfig> configObjects) { + public MConfigList(List<MConfig> configObjects, MConfigType type) { this.configObjects = configObjects; + this.type = type; } public List<MConfig> getConfigs() { @@ -38,18 +40,21 @@ public class MConfigList implements MClonable { } public MConfig getConfig(String configName) { - for(MConfig config: configObjects) { - if(configName.equals(config.getName())) { + for (MConfig config : configObjects) { + if (configName.equals(config.getName())) { return config; } } - throw new SqoopException(ModelError.MODEL_010, "config name: " + configName); } + public MConfigType getType() { + return type; + } + public MInput getInput(String name) { - String []parts = name.split("\\."); - if(parts.length != 2) { + String[] parts = name.split("\\."); + if (parts.length != 2) { throw new SqoopException(ModelError.MODEL_009, name); } @@ -57,68 +62,73 @@ public class MConfigList implements MClonable { } public MStringInput getStringInput(String name) { - return (MStringInput)getInput(name); + return (MStringInput) getInput(name); } public MEnumInput getEnumInput(String name) { - return (MEnumInput)getInput(name); + return (MEnumInput) getInput(name); } public MIntegerInput getIntegerInput(String name) { - return (MIntegerInput)getInput(name); + return (MIntegerInput) getInput(name); } public MMapInput getMapInput(String name) { - return (MMapInput)getInput(name); + return (MMapInput) getInput(name); } public MBooleanInput getBooleanInput(String name) { - return (MBooleanInput)getInput(name); + return (MBooleanInput) getInput(name); } @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof MConfigList)) return false; + if (this == o) { + return true; + } + if (!(o instanceof MConfigList)) { + return false; + } MConfigList mConfigList = (MConfigList) o; - - if (!configObjects.equals(mConfigList.configObjects)) return false; - + if (!configObjects.equals(mConfigList.configObjects) || !type.equals(mConfigList.type)) { + return false; + } return true; } @Override public int hashCode() { int result = super.hashCode(); - for(MConfig config : configObjects) { + result = 31 * result + type.hashCode(); + for (MConfig config : configObjects) { result = 31 * result + config.hashCode(); } - return result; } @Override public String toString() { StringBuilder sb = new StringBuilder("Configs: "); - for(MConfig config : configObjects) { + for (MConfig config : configObjects) { sb.append(config.toString()); } + sb.append("Type: " + type); return sb.toString(); } @Override public MConfigList clone(boolean cloneWithValue) { List<MConfig> copyConfigs = null; - if(this.getConfigs() != null) { + if (this.getConfigs() != null) { copyConfigs = new ArrayList<MConfig>(); - for(MConfig itr : this.getConfigs()) { + for (MConfig itr : this.getConfigs()) { MConfig newConfig = itr.clone(cloneWithValue); newConfig.setPersistenceId(itr.getPersistenceId()); copyConfigs.add(newConfig); } } - MConfigList copyConfigList = new MConfigList(copyConfigs); + MConfigList copyConfigList = new MConfigList(copyConfigs, type); return copyConfigList; } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java b/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java index 679859a..3faf1d3 100644 --- a/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java +++ b/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java @@ -27,7 +27,7 @@ import java.util.List; */ public class MDriverConfig extends MConfigList { public MDriverConfig(List<MConfig> configs) { - super(configs); + super(configs, MConfigType.JOB); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/model/MFromConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MFromConfig.java b/common/src/main/java/org/apache/sqoop/model/MFromConfig.java index 1b450d6..1e9e845 100644 --- a/common/src/main/java/org/apache/sqoop/model/MFromConfig.java +++ b/common/src/main/java/org/apache/sqoop/model/MFromConfig.java @@ -27,7 +27,7 @@ import java.util.List; */ public class MFromConfig extends MConfigList { public MFromConfig(List<MConfig> configs) { - super(configs); + super(configs, MConfigType.JOB); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java b/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java index 318b63c..040dca4 100644 --- a/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java +++ b/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java @@ -27,7 +27,7 @@ import java.util.List; public class MLinkConfig extends MConfigList { public MLinkConfig(List<MConfig> configs) { - super(configs); + super(configs, MConfigType.LINK); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/model/MToConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MToConfig.java b/common/src/main/java/org/apache/sqoop/model/MToConfig.java index b4fbe41..1d0c91f 100644 --- a/common/src/main/java/org/apache/sqoop/model/MToConfig.java +++ b/common/src/main/java/org/apache/sqoop/model/MToConfig.java @@ -27,7 +27,7 @@ import java.util.List; */ public class MToConfig extends MConfigList { public MToConfig(List<MConfig> configs) { - super(configs); + super(configs, MConfigType.JOB); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/main/java/org/apache/sqoop/utils/MapResourceBundle.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/utils/MapResourceBundle.java b/common/src/main/java/org/apache/sqoop/utils/MapResourceBundle.java index b0a15e7..8dbeb70 100644 --- a/common/src/main/java/org/apache/sqoop/utils/MapResourceBundle.java +++ b/common/src/main/java/org/apache/sqoop/utils/MapResourceBundle.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.ResourceBundle; /** - * + * Wrapper class to hold the resource bundle key-value pairs in a collections map object */ public class MapResourceBundle extends ResourceBundle { http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java b/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java index 4372171..30778f5 100644 --- a/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java +++ b/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java @@ -25,7 +25,6 @@ import java.util.ResourceBundle; import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConnector; -import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MFromConfig; import org.apache.sqoop.model.MInput; @@ -41,11 +40,11 @@ import org.apache.sqoop.utils.MapResourceBundle; * */ public class ConfigTestUtil { - public static MConnector getConnector(String name) { - return getConnector(name, true, true); + public static MConnector getConnector(Long id, String name) { + return getConnector(id, name, true, true); } - public static MConnector getConnector(String name, boolean from, boolean to) { + public static MConnector getConnector(Long id, String name, boolean from, boolean to) { MFromConfig fromConfig = null; MToConfig toConfig = null; if (from) { @@ -54,17 +53,21 @@ public class ConfigTestUtil { if (to) { toConfig = getToConfig(); } - return new MConnector(name, name + ".class", "1.0-test", + + MConnector connector = new MConnector(name, name + ".class", "1.0-test", getLinkConfig(), fromConfig, toConfig); + // simulate a persistence id + connector.setPersistenceId(id); + return connector; } public static MLink getLink(String name) { - return new MLink(1, getConnector(name).getLinkConfig()); + return new MLink(1, getConnector(1L, name).getLinkConfig()); } public static MJob getJob(String name) { - return new MJob(1, 2, 1, 2, getConnector(name).getFromConfig(), getConnector(name) + return new MJob(1, 2, 1, 2, getConnector(1L, name).getFromConfig(), getConnector(1L, name) .getToConfig(), getDriverConfig()); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java index 9fd2fe3..308e17e 100644 --- a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java +++ b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java @@ -45,8 +45,8 @@ public class TestConnectorBean { public void testSerialization() { // Create testing connector List<MConnector> connectors = new LinkedList<MConnector>(); - connectors.add(getConnector("jdbc")); - connectors.add(getConnector("mysql")); + connectors.add(getConnector(1L, "jdbc")); + connectors.add(getConnector(2L, "mysql")); // Create testing bundles Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>(); @@ -67,6 +67,7 @@ public class TestConnectorBean { assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size()); assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0)); + assertEquals(connectors.get(1), parsedConnectorBean.getConnectors().get(1)); ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L); assertNotNull(retrievedBundle); @@ -78,8 +79,8 @@ public class TestConnectorBean { public void testSingleDirection() { // Create testing connector List<MConnector> connectors = new LinkedList<MConnector>(); - connectors.add(getConnector("jdbc", true, false)); - connectors.add(getConnector("mysql", false, true)); + connectors.add(getConnector(1L, "jdbc", true, false)); + connectors.add(getConnector(2L, "mysql", false, true)); // Create testing bundles Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); @@ -107,8 +108,8 @@ public class TestConnectorBean { public void testNoDirection() { // Create testing connector List<MConnector> connectors = new LinkedList<MConnector>(); - connectors.add(getConnector("jdbc", false, false)); - connectors.add(getConnector("mysql", false, false)); + connectors.add(getConnector(1L, "jdbc", false, false)); + connectors.add(getConnector(2L, "mysql", false, false)); // Create testing bundles Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java index ac07137..526ec52 100644 --- a/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java +++ b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; import java.util.Date; -import org.apache.sqoop.json.util.ConfigSerialization; +import org.apache.sqoop.json.util.ConfigInputConstants; import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MStringInput; import org.json.simple.JSONArray; @@ -59,13 +59,13 @@ public class TestLinkBean { JSONObject json = linkBean.extract(false); // Check for sensitivity - JSONArray all = (JSONArray)json.get(ConfigSerialization.ALL); + JSONArray all = (JSONArray)json.get(JsonBean.ALL); JSONObject allItem = (JSONObject)all.get(0); JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG); JSONObject connector = (JSONObject)connectors.get(0); - JSONArray inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS); + JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS); for (Object input1 : inputs) { - assertTrue(((JSONObject)input1).containsKey(ConfigSerialization.CONFIG_INPUT_SENSITIVE)); + assertTrue(((JSONObject)input1).containsKey(ConfigInputConstants.CONFIG_INPUT_SENSITIVE)); } // "Move" it across network in text form @@ -116,25 +116,25 @@ public class TestLinkBean { JSONObject jsonFiltered = bean.extract(true); // Sensitive values should exist - JSONArray all = (JSONArray)json.get(ConfigSerialization.ALL); + JSONArray all = (JSONArray)json.get(JsonBean.ALL); JSONObject allItem = (JSONObject)all.get(0); JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG); JSONObject connector = (JSONObject)connectors.get(0); - JSONArray inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS); + JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS); assertEquals(3, inputs.size()); // Inputs are ordered when creating link JSONObject password = (JSONObject)inputs.get(2); - assertTrue(password.containsKey(ConfigSerialization.CONFIG_INPUT_VALUE)); + assertTrue(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE)); // Sensitive values should not exist - all = (JSONArray)jsonFiltered.get(ConfigSerialization.ALL); + all = (JSONArray)jsonFiltered.get(JsonBean.ALL); allItem = (JSONObject)all.get(0); connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG); connector = (JSONObject)connectors.get(0); - inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS); + inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS); assertEquals(3, inputs.size()); // Inputs are ordered when creating link password = (JSONObject)inputs.get(2); - assertFalse(password.containsKey(ConfigSerialization.CONFIG_INPUT_VALUE)); + assertFalse(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE)); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java index 4f0c84d..61fffad 100644 --- a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java +++ b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java @@ -17,10 +17,19 @@ */ package org.apache.sqoop.json.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MBooleanInput; -import org.apache.sqoop.model.MEnumInput; import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MConfigType; +import org.apache.sqoop.model.MEnumInput; import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MIntegerInput; import org.apache.sqoop.model.MMapInput; @@ -29,14 +38,6 @@ import org.json.simple.JSONObject; import org.json.simple.JSONValue; import org.junit.Test; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - /** * */ @@ -57,7 +58,7 @@ public class TestConfigSerialization { config.getEnumInput("Enum").setValue("YES"); // Serialize that into JSON - JSONObject jsonObject = ConfigSerialization.extractConfig(config, false); + JSONObject jsonObject = ConfigInputSerialization.extractConfig(config, MConfigType.JOB, false); assertNotNull(jsonObject); // Exchange the data on string level @@ -65,7 +66,7 @@ public class TestConfigSerialization { JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson); // And retrieve back from JSON representation - MConfig retrieved = ConfigSerialization.restoreConfig(retrievedJson); + MConfig retrieved = ConfigInputSerialization.restoreConfig(retrievedJson); // Verify all expected values assertEquals("A", retrieved.getStringInput("String").getValue()); @@ -85,12 +86,12 @@ public class TestConfigSerialization { config.getMapInput("Map").setValue(map); // Serialize - JSONObject jsonObject = ConfigSerialization.extractConfig(config, false); + JSONObject jsonObject = ConfigInputSerialization.extractConfig(config, MConfigType.JOB, false); String serializedJson = jsonObject.toJSONString(); // Deserialize JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson); - MConfig retrieved = ConfigSerialization.restoreConfig(retrievedJson); + MConfig retrieved = ConfigInputSerialization.restoreConfig(retrievedJson); assertEquals(map, retrieved.getMapInput("Map").getValue()); } @@ -104,14 +105,14 @@ public class TestConfigSerialization { config.getMapInput("Map").setValue(map); // Serialize - JSONObject jsonObject = ConfigSerialization.extractConfig(config, false); + JSONObject jsonObject = ConfigInputSerialization.extractConfig(config, MConfigType.JOB, false); String serializedJson = jsonObject.toJSONString(); // Replace map value with a fake string to force exception String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}", "\"nonsensical string\""); System.out.println(badSerializedJson); JSONObject retrievedJson = (JSONObject) JSONValue.parse(badSerializedJson); - ConfigSerialization.restoreConfig(retrievedJson); + ConfigInputSerialization.restoreConfig(retrievedJson); } protected MConfig getMapConfig() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/24e9b4c5/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java index 9b60055..46f4f81 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java @@ -46,7 +46,7 @@ public class TestMConfigList { inputs.add(enumInput); configs.add(new MConfig("Config2", inputs)); - MConfigList config = new MConfigList(configs); + MConfigList config = new MConfigList(configs, MConfigType.JOB); assertEquals(intInput, config.getIntegerInput("Config1.A")); assertEquals(mapInput, config.getMapInput("Config1.B")); assertEquals(stringInput, config.getStringInput("Config2.C"));
