http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java deleted file mode 100644 index a0a5dc2..0000000 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java +++ /dev/null @@ -1,939 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.shell.utils; - -import jline.ConsoleReader; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.lang.StringUtils; -import org.apache.sqoop.common.Direction; -import org.apache.sqoop.model.MBooleanInput; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MEnumInput; -import org.apache.sqoop.model.MForm; -import org.apache.sqoop.model.MInput; -import org.apache.sqoop.model.MIntegerInput; -import org.apache.sqoop.model.MMapInput; -import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MNamedElement; -import org.apache.sqoop.model.MStringInput; -import org.apache.sqoop.model.MValidatedElement; -import org.apache.sqoop.validation.Message; -import org.apache.sqoop.validation.Status; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.util.ResourceBundle; - -import static org.apache.sqoop.shell.ShellEnvironment.*; - -/** - * Convenient methods for retrieving user input and CLI options. - */ -public final class FormFiller { - - /** - * Internal input that will be reused for loading names for link and - * job objects. - */ - private static MStringInput nameInput = new MStringInput("object-name", false, (short)25); - - /** - * Fill job object based on CLI options. - * - * @param line Associated console reader object - * @param job Job that user is suppose to fill in - * @return True if we filled all inputs, false if user has stopped processing - * @throws IOException - */ - public static boolean fillJob(CommandLine line, - MJob job) - throws IOException { - - job.setName(line.getOptionValue("name")); - - // Fill in data from user - return fillForms(line, - job.getConnectorPart(Direction.FROM).getForms(), - job.getFrameworkPart().getForms()); - } - - /** - * Fill job object based on user input. - * - * @param reader Associated console reader object - * @param job Job that user is suppose to fill in - * @param fromConnectorBundle Connector resource bundle - * @param driverConfigBundle Driver config resource bundle - * @return True if we filled all inputs, false if user has stopped processing - * @throws IOException - */ - public static boolean fillJob(ConsoleReader reader, - MJob job, - ResourceBundle fromConnectorBundle, - ResourceBundle driverConfigBundle, - ResourceBundle toConnectorBundle) - throws IOException { - - job.setName(getName(reader, job.getName())); - - // Fill in data from user - return fillForms(reader, - job.getConnectorPart(Direction.FROM).getForms(), - fromConnectorBundle, - job.getFrameworkPart().getForms(), - driverConfigBundle, - job.getConnectorPart(Direction.TO).getForms(), - toConnectorBundle); - } - - /** - * Fill link object based on CLI options. - * - * @param line Associated command line options - * @param link Link that user is suppose to fill in - * @return True if we filled all inputs, false if user has stopped processing - * @throws IOException - */ - public static boolean fillConnection(CommandLine line, - MLink link) - throws IOException { - - link.setName(line.getOptionValue("name")); - - // Fill in data from user - return fillForms(line, - link.getConnectorPart().getForms(), - link.getFrameworkPart().getForms()); - } - - /** - * Fill link object based on user input. - * - * @param reader Associated console reader object - * @param link Link that user is suppose to fill in - * @param connectorConfigBundle Connector resource bundle - * @param driverConfigBundle Driver config resource bundle - * @return True if we filled all inputs, false if user has stopped processing - * @throws IOException - */ - public static boolean fillLink(ConsoleReader reader, - MLink link, - ResourceBundle connectorConfigBundle, - ResourceBundle driverConfigBundle) - throws IOException { - - link.setName(getName(reader, link.getName())); - - // Fill in data from user - return fillForms(reader, - link.getConnectorPart().getForms(), - connectorConfigBundle, - link.getFrameworkPart().getForms(), - driverConfigBundle); - } - - /** - * Load CLI options for framework forms and connector forms. - * - * @param line CLI options container - * @param connectorForms Connector forms to read or edit - * @param frameworkForms Framework forms to read or edit - * @return - * @throws IOException - */ - public static boolean fillForms(CommandLine line, - List<MForm> connectorForms, - List<MForm> frameworkForms) - throws IOException { - // Query connector forms and framework forms - return fillForms("connector", connectorForms, line) - && fillForms("framework", frameworkForms, line); - } - - /** - * Load all CLI options for a list of forms. - * - * @param prefix placed at the beginning of the CLI option key - * @param forms Forms to read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - public static boolean fillForms(String prefix, - List<MForm> forms, - CommandLine line) - throws IOException { - for (MForm form : forms) { - if (!fillForm(prefix, form, line)) { - return false; - } - } - return true; - } - - /** - * Load all CLI options for a particular form. - * - * @param prefix placed at the beginning of the CLI option key - * @param form Form to read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - @SuppressWarnings("rawtypes") - public static boolean fillForm(String prefix, - MForm form, - CommandLine line) throws IOException { - for (MInput input : form.getInputs()) { - if (!fillInput(prefix, input, line)) { - return false; - } - } - return true; - } - - /** - * Load CLI option. - * Chooses the appropriate 'fill' method to use based on input type. - * - * Keys for CLI options are automatically created from the 'prefix' argument - * and 'input' argument: <prefix>-<form name>-<input name> - * - * @param prefix placed at the beginning of the CLI option key - * @param input Input that we should read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - @SuppressWarnings("rawtypes") - public static boolean fillInput(String prefix, - MInput input, - CommandLine line) throws IOException { - // Based on the input type, let's perform specific load - switch (input.getType()) { - case STRING: - return fillInputString(prefix, (MStringInput) input, line); - case INTEGER: - return fillInputInteger(prefix, (MIntegerInput) input, line); - case BOOLEAN: - return fillInputBoolean(prefix, (MBooleanInput) input, line); - case MAP: - return fillInputMap(prefix, (MMapInput) input, line); - case ENUM: - return fillInputEnum(prefix, (MEnumInput) input, line); - default: - println("Unsupported data type " + input.getType()); - return true; - } - } - - /** - * Load CLI option for enum type. - * - * Currently only supports numeric values. - * - * @param prefix placed at the beginning of the CLI option key - * @param input Input that we should read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - private static boolean fillInputEnum(String prefix, - MEnumInput input, - CommandLine line) - throws IOException { - String opt = FormOptions.getOptionKey(prefix, input); - if (line.hasOption(opt)) { - String value = line.getOptionValue(opt); - int index = java.util.Arrays.asList(input.getValues()).indexOf(value); - - if(index < 0) { - errorMessage(input, String.format("Invalid option %s. Please use one of %s.", value, StringUtils.join(input.getValues(), ", "))); - return false; - } - - input.setValue(value); - } else { - input.setEmpty(); - } - return true; - } - - /** - * Load CLI options for map type. - * - * Parses Key-Value pairs that take the form "<key>=<value>&<key>=<value>&...". - * - * @param prefix placed at the beginning of the CLI option key - * @param input Input that we should read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - private static boolean fillInputMap(String prefix, - MMapInput input, - CommandLine line) - throws IOException { - String opt = FormOptions.getOptionKey(prefix, input); - if (line.hasOption(opt)) { - String value = line.getOptionValue(opt); - Map<String, String> values = new HashMap<String, String>(); - String[] entries = value.split("&"); - for (String entry : entries) { - if (entry.contains("=")) { - String[] keyValue = entry.split("="); - values.put(keyValue[0], keyValue[1]); - } else { - errorMessage(input, "Don't know what to do with " + entry); - return false; - } - } - input.setValue(values); - } else { - input.setEmpty(); - } - return true; - } - - /** - * Load integer input from CLI option. - * - * @param prefix placed at the beginning of the CLI option key - * @param input Input that we should read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - private static boolean fillInputInteger(String prefix, - MIntegerInput input, - CommandLine line) - throws IOException { - String opt = FormOptions.getOptionKey(prefix, input); - if (line.hasOption(opt)) { - try { - input.setValue(Integer.valueOf(line.getOptionValue(FormOptions.getOptionKey(prefix, input)))); - } catch (NumberFormatException ex) { - errorMessage(input, "Input is not valid integer number"); - return false; - } - } else { - input.setEmpty(); - } - return true; - } - - /** - * Load string input from CLI option. - * - * @param prefix placed at the beginning of the CLI option key - * @param input Input that we should read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - public static boolean fillInputString(String prefix, - MStringInput input, - CommandLine line) - throws IOException { - String opt = FormOptions.getOptionKey(prefix, input); - if (line.hasOption(opt)) { - String value = line.getOptionValue(FormOptions.getOptionKey(prefix, input)); - if(value.length() > input.getMaxLength()) { - errorMessage(input, "Size of input exceeds allowance for this input" - + " field. Maximal allowed size is " + input.getMaxLength()); - } - input.setValue(value); - } else { - input.setEmpty(); - } - return true; - } - - /** - * Load boolean input from CLI option. - * - * @param prefix placed at the beginning of the CLI option key - * @param input Input that we should read or edit - * @param line CLI options container - * @return - * @throws IOException - */ - public static boolean fillInputBoolean(String prefix, - MBooleanInput input, - CommandLine line) - throws IOException { - String opt = FormOptions.getOptionKey(prefix, input); - if (line.hasOption(opt)) { - input.setValue(Boolean.valueOf(line.getOptionValue(FormOptions.getOptionKey(prefix, input)))); - } else { - input.setEmpty(); - } - return true; - } - - public static boolean fillForms(ConsoleReader reader, - List<MForm> connectorForms, - ResourceBundle connectorConfigBundle, - List<MForm> frameworkForms, - ResourceBundle driverConfigBundle) throws IOException { - - - // Query connector forms - if(!fillForms(connectorForms, reader, connectorConfigBundle)) { - return false; - } - - // Query framework forms - if(!fillForms(frameworkForms, reader, driverConfigBundle)) { - return false; - } - return true; - } - - public static boolean fillForms(ConsoleReader reader, - List<MForm> fromConnectorForms, - ResourceBundle fromConnectorBundle, - List<MForm> frameworkForms, - ResourceBundle driverConfigBundle, - List<MForm> toConnectorForms, - ResourceBundle toConnectorBundle) throws IOException { - - - // From connector forms - if(!fillForms(fromConnectorForms, reader, fromConnectorBundle)) { - return false; - } - - // Query framework forms - if(!fillForms(frameworkForms, reader, driverConfigBundle)) { - return false; - } - - // To connector forms - if(!fillForms(toConnectorForms, reader, toConnectorBundle)) { - return false; - } - - return true; - } - - public static boolean fillForms(List<MForm> forms, - ConsoleReader reader, - ResourceBundle bundle) - throws IOException { - for (MForm form : forms) { - if(!fillForm(form, reader, bundle)) { - return false; - } - } - - return true; - } - - @SuppressWarnings("rawtypes") - public static boolean fillForm(MForm form, - ConsoleReader reader, - ResourceBundle bundle) throws IOException { - println(""); - println(bundle.getString(form.getLabelKey())); - - // Print out form validation - printValidationMessage(form, false); - println(""); - - for (MInput input : form.getInputs()) { - if(!fillInput(input, reader, bundle)) { - return false; - } - } - - return true; - } - - @SuppressWarnings("rawtypes") - public static boolean fillInput(MInput input, - ConsoleReader reader, - ResourceBundle bundle) throws IOException { - // Print out validation - printValidationMessage(input, false); - - // Based on the input type, let's perform specific load - switch (input.getType()) { - case STRING: - return fillInputString((MStringInput) input, reader, bundle); - case INTEGER: - return fillInputInteger((MIntegerInput) input, reader, bundle); - case BOOLEAN: - return fillInputBoolean((MBooleanInput) input, reader, bundle); - case MAP: - return fillInputMap((MMapInput) input, reader, bundle); - case ENUM: - return fillInputEnum((MEnumInput) input, reader, bundle); - default: - println("Unsupported data type " + input.getType()); - return true; - } - } - - /** - * Load user input for enum type. - * - * Print out numbered list of all available options and let user choose one - * item from that. - * - * @param input Input that we should read or edit - * @param reader Associated console reader - * @param bundle Resource bundle - * @return True if user with to continue with loading addtional inputs - * @throws IOException - */ - private static boolean fillInputEnum(MEnumInput input, - ConsoleReader reader, - ResourceBundle bundle) - throws IOException { - // Prompt in enum case - println(bundle.getString(input.getLabelKey()) + ": "); - - // Indexes - int i = -1; - int lastChoice = -1; - - // Print out all values as a numbered list - for(String value : input.getValues()) { - i++; - - println(" " + i + " : " + value); - - // Only show last choice if not sensitive - if(!input.isEmpty() && value.equals(input.getValue()) && !input.isSensitive()) { - lastChoice = i; - } - } - - // Prompt - reader.printString("Choose: "); - - // Fill previously filled index when available - if(lastChoice != -1) { - reader.putString(Integer.toString(lastChoice)); - } - - reader.flushConsole(); - String userTyped; - if(input.isSensitive()) { - userTyped = reader.readLine('*'); - } else { - userTyped = reader.readLine(); - } - - if (userTyped == null) { - return false; - } else if (userTyped.isEmpty()) { - input.setEmpty(); - } else { - Integer index; - try { - index = Integer.valueOf(userTyped); - - if(index < 0 || index >= input.getValues().length) { - errorMessage("Invalid index"); - return fillInputEnum(input, reader, bundle); - } - - input.setValue(input.getValues()[index]); - } catch (NumberFormatException ex) { - errorMessage("Input is not valid integer number"); - return fillInputEnum(input, reader, bundle); - } - } - - return true; - } - - /** - * Load user input for map type. - * - * This implementation will load one map entry at the time. Current flows is - * as follows: if user did not enter anything (empty input) finish loading - * and return from function. If user specified input with equal sign (=), - * lets add new key value pair. Otherwise consider entire input as a key name - * and try to remove it from the map. - * - * Please note that following code do not supports equal sign in property - * name. It's however perfectly fine to have equal sign in value. - * - * @param input Input that we should read or edit - * @param reader Associated console reader - * @param bundle Resource bundle - * @return True if user wish to continue with loading additional inputs - * @throws IOException - */ - private static boolean fillInputMap(MMapInput input, - ConsoleReader reader, - ResourceBundle bundle) - throws IOException { - // Special prompt in Map case - println(bundle.getString(input.getLabelKey()) + ": "); - - // Internal loading map - Map<String, String> values = input.getValue(); - if(values == null) { - values = new HashMap<String, String>(); - } - - String userTyped; - - while(true) { - // Print all current items in each iteration - // However do not printout if this input contains sensitive information. - println("There are currently " + values.size() + " values in the map:"); - if (!input.isSensitive()) { - for(Map.Entry<String, String> entry : values.entrySet()) { - println(entry.getKey() + " = " + entry.getValue()); - } - } - - // Special prompt for Map entry - reader.printString("entry# "); - reader.flushConsole(); - - if(input.isSensitive()) { - userTyped = reader.readLine('*'); - } else { - userTyped = reader.readLine(); - } - - if(userTyped == null) { - // Finish loading and return back to Sqoop shell - return false; - } else if(userTyped.isEmpty()) { - // User has finished loading data to Map input, either set input empty - // if there are no entries or propagate entries to the input - if(values.size() == 0) { - input.setEmpty(); - } else { - input.setValue(values); - } - return true; - } else { - // User has specified regular input, let's check if it contains equals - // sign. Save new entry (or update existing one) if it does. Otherwise - // try to remove entry that user specified. - if(userTyped.contains("=")) { - String []keyValue = userTyped.split("=", 2); - values.put(handleUserInput(keyValue[0]), handleUserInput(keyValue[1])); - } else { - String key = handleUserInput(userTyped); - if(values.containsKey(key)) { - values.remove(key); - } else { - errorMessage("Don't know what to do with " + userTyped); - } - } - } - - } - } - - /** - * Handle special cases in user input. - * - * Preserve null and empty values, remove whitespace characters before and - * after loaded string and de-quote the string if it's quoted (to preserve - * spaces for example). - * - * @param input String loaded from user - * @return Unquoted transformed string - */ - private static String handleUserInput(String input) { - // Preserve null and empty values - if(input == null) { - return null; - } - if(input.isEmpty()) { - return input; - } - - // Removes empty characters at the begging and end of loaded string - input = input.trim(); - - int lastIndex = input.length() - 1; - char first = input.charAt(0); - char last = input.charAt(lastIndex); - - // Remove quoting if present - if(first == '\'' && last == '\'') { - input = input.substring(1, lastIndex); - } else if(first == '"' && last == '"') { - input = input.substring(1, lastIndex); - } - - // Return final string - return input; - } - - private static boolean fillInputInteger(MIntegerInput input, - ConsoleReader reader, - ResourceBundle bundle) - throws IOException { - generatePrompt(reader, bundle, input); - - // Fill already filled data when available - // However do not printout if this input contains sensitive information. - if(!input.isEmpty() && !input.isSensitive()) { - reader.putString(input.getValue().toString()); - } - - // Get the data - String userTyped; - if(input.isSensitive()) { - userTyped = reader.readLine('*'); - } else { - userTyped = reader.readLine(); - } - - if (userTyped == null) { - return false; - } else if (userTyped.isEmpty()) { - input.setEmpty(); - } else { - Integer value; - try { - value = Integer.valueOf(userTyped); - input.setValue(value); - } catch (NumberFormatException ex) { - errorMessage("Input is not valid integer number"); - return fillInputInteger(input, reader, bundle); - } - - input.setValue(Integer.valueOf(userTyped)); - } - - return true; - } - - /** - * Load string input from the user. - * - * @param input Input that we should load in - * @param reader Associated console reader - * @param bundle Resource bundle for this input - * @return - * @throws IOException - */ - public static boolean fillInputString(MStringInput input, - ConsoleReader reader, - ResourceBundle bundle) - throws IOException { - generatePrompt(reader, bundle, input); - - // Fill already filled data when available - // However do not printout if this input contains sensitive information. - if(!input.isEmpty() && !input.isSensitive()) { - reader.putString(input.getValue()); - } - - // Get the data - String userTyped; - if(input.isSensitive()) { - userTyped = reader.readLine('*'); - } else { - userTyped = reader.readLine(); - } - - if (userTyped == null) { - // Propagate end of loading process - return false; - } else if (userTyped.isEmpty()) { - // Empty input in case that nothing was given - input.setEmpty(); - } else { - // Set value that user has entered - input.setValue(userTyped); - - // Check that it did not exceeds maximal allowance for given input - if(userTyped.length() > input.getMaxLength()) { - errorMessage("Size of input exceeds allowance for this input" - + " field. Maximal allowed size is " + input.getMaxLength()); - return fillInputString(input, reader, bundle); - } - } - - return true; - } - - /** - * Load boolean input from the user. - * - * @param input Input that we should load in - * @param reader Associated console reader - * @param bundle Resource bundle for this input - * @return - * @throws IOException - */ - public static boolean fillInputBoolean(MBooleanInput input, - ConsoleReader reader, - ResourceBundle bundle) - throws IOException { - generatePrompt(reader, bundle, input); - - // Fill already filled data when available - // However do not printout if this input contains sensitive information. - if(!input.isEmpty() && !input.isSensitive()) { - reader.putString(input.getValue().toString()); - } - - // Get the data - String userTyped; - if(input.isSensitive()) { - userTyped = reader.readLine('*'); - } else { - userTyped = reader.readLine(); - } - - if (userTyped == null) { - // Propagate end of loading process - return false; - } else if (userTyped.isEmpty()) { - // Empty input in case that nothing was given - input.setEmpty(); - } else { - // Set value that user has entered - input.setValue(Boolean.valueOf(userTyped)); - } - - return true; - } - - @SuppressWarnings("rawtypes") - public static void generatePrompt(ConsoleReader reader, - ResourceBundle bundle, - MInput input) - throws IOException { - reader.printString(bundle.getString(input.getLabelKey()) + ": "); - reader.flushConsole(); - } - - public static String getName(ConsoleReader reader, - String name) throws IOException { - if(name == null) { - nameInput.setEmpty(); - } else { - nameInput.setValue(name); - } - - fillInputString(nameInput, reader, getResourceBundle()); - - return nameInput.getValue(); - } - - /** - * Print validation message in cases that it's not in state "FINE" - * - * @param element Validated element - */ - public static void printValidationMessage(MValidatedElement element, boolean includeInputPrefix) { - if(element.getValidationStatus() == Status.getDefault()) { - return; - } - - for(Message message : element.getValidationMessages()) - switch (message.getStatus()) { - case UNACCEPTABLE: - if (includeInputPrefix) { - errorMessage(element, message.getMessage()); - } else { - errorMessage(message.getMessage()); - } - break; - case ACCEPTABLE: - if (includeInputPrefix) { - warningMessage(element, message.getMessage()); - } else { - warningMessage(message.getMessage()); - } - break; - default: - // Simply ignore all other states for the moment - break; - } - } - - public static void errorMessage(String message) { - println("Error message: @|red " + message + " |@"); - } - - public static void errorMessage(MNamedElement input, String message) { - print(input.getName()); - print(": "); - errorMessage(message); - } - - public static void warningMessage(String message) { - println("Warning message: @|yellow " + message + " |@"); - } - - public static void warningMessage(MNamedElement input, String message) { - print(input.getName()); - print(": "); - warningMessage(message); - } - - public static void errorIntroduction() { - println(); - println("@|red There are issues with entered data, please revise your input:|@"); - } - - public static void printLinkValidationMessages(MLink link) { - for (MForm form : link.getConnectorPart().getForms()) { - for (MInput<?> input : form.getInputs()) { - printValidationMessage(input, true); - } - } - for (MForm form : link.getFrameworkPart().getForms()) { - for (MInput<?> input : form.getInputs()) { - printValidationMessage(input, true); - } - } - } - - public static void printJobValidationMessages(MJob job) { - for (MForm form : job.getConnectorPart(Direction.FROM).getForms()) { - for (MInput<?> input : form.getInputs()) { - printValidationMessage(input, true); - } - } - for (MForm form : job.getFrameworkPart().getForms()) { - for (MInput<?> input : form.getInputs()) { - printValidationMessage(input, true); - } - } - for (MForm form : job.getConnectorPart(Direction.TO).getForms()) { - for (MInput<?> input : form.getInputs()) { - printValidationMessage(input, true); - } - } - } - - private FormFiller() { - // Do not instantiate - } -}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java deleted file mode 100644 index efd002e..0000000 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.shell.utils; - -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.sqoop.common.SqoopException; -import org.apache.sqoop.model.MForm; -import org.apache.sqoop.model.MInput; -import org.apache.sqoop.model.MInputType; -import org.apache.sqoop.shell.core.ShellError; - -/** - * Utilities for automatically creating org.apache.commons.cli.Option objects. - */ -public class FormOptions { - /** - * This method is used to automatically generate keys - * for a particular input. - * - * @param prefix Prefix to prepend to CLI option keys - * @param input - * @return - */ - @SuppressWarnings("rawtypes") - public static String getOptionKey(String prefix, MInput input) { - return prefix + "-" + input.getName().replace('.', '-'); - } - - /** - * This method is used to automatically generate CLI options - * for a list of forms. - * - * @param prefix Prefix to prepend to CLI option keys - * @param forms Forms to get options for - * @return - */ - public static List<Option> getFormsOptions(String prefix, List<MForm> forms) { - List<Option> options = new LinkedList<Option>(); - for (MForm form : forms) { - List<Option> formOptions = getFormOptions(prefix, form); - options.addAll(formOptions); - } - return options; - } - - /** - * This method is used to automatically generate CLI options - * for a particular form. - * - * @param prefix Prefix to prepend to CLI option keys - * @param form Form to get options for - * @return List<Option> - */ - @SuppressWarnings({ "rawtypes", "static-access" }) - public static List<Option> getFormOptions(String prefix, MForm form) { - List<Option> options = new LinkedList<Option>(); - for (MInput input : form.getInputs()) { - if (input.getType().equals(MInputType.BOOLEAN)) { - options.add(OptionBuilder - .withLongOpt(getOptionKey(prefix, input)) - .create()); - } else { - options.add(OptionBuilder - .withLongOpt(getOptionKey(prefix, input)) - .hasArg() - .create()); - } - } - return options; - } - - /** - * Parses command line options. - * - * @param options parse arglist against these. - * @param start beginning index in arglist. - * @param arglist arguments to parse. - * @param stopAtNonOption stop parsing when nonoption found in arglist. - * @return CommandLine object - */ - public static CommandLine parseOptions(Options options, int start, List<String> arglist, boolean stopAtNonOption) { - String[] args = arglist.subList(start, arglist.size()).toArray(new String[arglist.size() - start]); - - CommandLineParser parser = new GnuParser(); - CommandLine line; - try { - line = parser.parse(options, args, stopAtNonOption); - } catch (ParseException e) { - throw new SqoopException(ShellError.SHELL_0003, e.getMessage(), e); - } - return line; - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java new file mode 100644 index 0000000..49b09c7 --- /dev/null +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.shell.utils; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.sqoop.common.Direction; +import org.apache.sqoop.model.MJob; + +/** + * Automatically create dynamic options for jobs. + */ +@SuppressWarnings("serial") +public class JobDynamicConfigOptions extends DynamicConfigOptions<MJob> { + + @SuppressWarnings("static-access") + @Override + public void prepareOptions(MJob job) { + this.addOption(OptionBuilder + .withLongOpt("name") + .hasArg() + .create()); + for (Option option : ConfigOptions.getConfigsOptions("from", job.getJobConfig(Direction.FROM).getConfigs())) { + this.addOption(option); + } + for (Option option : ConfigOptions.getConfigsOptions("driver", job.getDriverConfig().getConfigs())) { + this.addOption(option); + } + for (Option option : ConfigOptions.getConfigsOptions("to", job.getJobConfig(Direction.TO).getConfigs())) { + this.addOption(option); + } + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java deleted file mode 100644 index 87c0776..0000000 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.shell.utils; - -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.sqoop.common.Direction; -import org.apache.sqoop.model.MJob; - -/** - * Automatically create dynamic options for jobs. - */ -@SuppressWarnings("serial") -public class JobDynamicFormOptions extends DynamicFormOptions<MJob> { - - @SuppressWarnings("static-access") - @Override - public void prepareOptions(MJob job) { - this.addOption(OptionBuilder - .withLongOpt("name") - .hasArg() - .create()); - for (Option option : FormOptions.getFormsOptions("connector", job.getConnectorPart(Direction.FROM).getForms())) { - this.addOption(option); - } - for (Option option : FormOptions.getFormsOptions("framework", job.getFrameworkPart().getForms())) { - this.addOption(option); - } - for (Option option : FormOptions.getFormsOptions("connector", job.getConnectorPart(Direction.TO).getForms())) { - this.addOption(option); - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java new file mode 100644 index 0000000..ef279f7 --- /dev/null +++ b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.shell.utils; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.sqoop.model.MLink; + +/** + * Automatically create dynamic options for connections. + */ +@SuppressWarnings("serial") +public class LinkDynamicConfigOptions extends DynamicConfigOptions<MLink> { + + @SuppressWarnings("static-access") + @Override + public void prepareOptions(MLink link) { + this.addOption(OptionBuilder.withLongOpt("name").hasArg().create()); + for (Option option : ConfigOptions.getConfigsOptions("link", link.getConnectorLinkConfig() + .getConfigs())) { + this.addOption(option); + } + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java deleted file mode 100644 index 2952be2..0000000 --- a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.shell.utils; - -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.sqoop.model.MLink; - -/** - * Automatically create dynamic options for connections. - */ -@SuppressWarnings("serial") -public class LinkDynamicFormOptions extends DynamicFormOptions<MLink> { - - @SuppressWarnings("static-access") - @Override - public void prepareOptions(MLink link) { - this.addOption(OptionBuilder.withLongOpt("name").hasArg().create()); - for (Option option : FormOptions.getFormsOptions("connector", link.getConnectorPart() - .getForms())) { - this.addOption(option); - } - for (Option option : FormOptions.getFormsOptions("framework", link.getFrameworkPart() - .getForms())) { - this.addOption(option); - } - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/resources/shell-resource.properties ---------------------------------------------------------------------- diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties index c0f86f7..e2381dd 100644 --- a/shell/src/main/resources/shell-resource.properties +++ b/shell/src/main/resources/shell-resource.properties @@ -16,7 +16,7 @@ # Client Resources in default language (english) ############################ -# Security Form +# Security Config ############################# object-name.label = Name object-name.help = Non unique name of the entity to help you remember \ @@ -200,22 +200,22 @@ table.header.status = Status table.header.date = Last Update Date table.header.enabled = Enabled -#Form displayer resources -formdisplayer.link = link -formdisplayer.job = Job -formdisplayer.form = form -formdisplayer.name = Name -formdisplayer.label = Label -formdisplayer.help = Help -formdisplayer.input = Input -formdisplayer.type = Type -formdisplayer.sensitive = Sensitive -formdisplayer.size = Size -formdisplayer.possible_values = Possible values -formdisplayer.unsupported_datatype = Unsupported data type -formdisplayer.input_sensitive = This input is sensitive - -formdisplayer.warning_message = There were warnings while create or update, but saved successfully. +#Config displayer resources +config.displayer.link = link +config.displayer.job = Job +config.displayer.config = config +config.displayer.name = Name +config.displayer.label = Label +config.displayer.help = Help +config.displayer.input = Input +config.displayer.type = Type +config.displayer.sensitive = Sensitive +config.displayer.size = Size +config.displayer.possible_values = Possible values +config.displayer.unsupported_datatype = Unsupported data type +config.displayer.input_sensitive = This input is sensitive + +config.displayer.warning_message = There were warnings while create or update, but saved successfully. submission.submission_detail = Submission details submission.job_id = Job ID http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java index e00b404..879e428 100644 --- a/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java +++ b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java @@ -18,8 +18,8 @@ */ package org.apache.sqoop.connector.spi; -import org.apache.sqoop.model.MConnectionForms; -import org.apache.sqoop.model.MJobForms; +import org.apache.sqoop.model.MConfigList; +import org.apache.sqoop.model.MLinkConfig; /** * Repository represents the sqoop entity store. Sqoop entities include @@ -29,23 +29,23 @@ import org.apache.sqoop.model.MJobForms; public abstract class RepositoryUpgrader { /** - * Upgrade the original connection and fill into the upgradeTarget. Note - * that any metadata already in {@code upgradeTarget} maybe overwritten. - * @param original - original connection metadata as in the repository + * Upgrade the original link config and fill into the upgradeTarget. Note + * that any data already in {@code upgradeTarget} maybe overwritten. + * @param original - original link config as in the repository * @param upgradeTarget - the instance that will be filled in with the - * upgraded metadata. + * upgraded link config. */ - public abstract void upgrade(MConnectionForms original, - MConnectionForms upgradeTarget); + public abstract void upgrade(MLinkConfig original, MLinkConfig upgradeTarget); /** - * Upgrade the original job and fill into the upgradeTarget. Note - * that any metadata already in {@code upgradeTarget} maybe overwritten. - * This method must be called only after the connection metadata has + * Upgrade the original job config and fill into the upgradeTarget. Note + * that any config data already in {@code upgradeTarget} maybe overwritten. + * This method must be called only after the link config has * already been upgraded. - * @param original - original connection metadata as in the repository + * @param original - original job config as in the repository * @param upgradeTarget - the instance that will be filled in with the - * upgraded metadata. + * upgraded job config. + * NOTE(VB): This api will be revisited to accomodate from and to job config update */ - public abstract void upgrade(MJobForms original, MJobForms upgradeTarget); + public abstract void upgrade(MConfigList original, MConfigList upgradeTarget); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java index 7b78ba0..849f091 100644 --- a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java +++ b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java @@ -58,12 +58,12 @@ public abstract class SqoopConnector { } /** - * @return Get connection configuration class + * @return Get link configuration group class */ public abstract Class getLinkConfigurationClass(); /** - * @return Get job configuration class for given type or null if not supported + * @return Get job configuration group class per direction type or null if not supported */ public abstract Class getJobConfigurationClass(Direction jobType); @@ -79,12 +79,10 @@ public abstract class SqoopConnector { /** * Returns validation object that Sqoop can use to validate user - * supplied forms before accepting them. This object will be used both for - * connection and job forms. - * + * supplied configs before accepting them. This object will be used both link and job configs * @return Validator object */ - public abstract Validator getValidator(); + public abstract Validator getConfigValidator(); /** * Returns an {@linkplain RepositoryUpgrader} object that can upgrade the http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java index f965bdd..a133106 100644 --- a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java +++ b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java @@ -28,7 +28,7 @@ public abstract class Destroyer<LinkConfiguration, JobConfiguration> { * * @param context Destroyer context * @param linkConfiguration link configuration object - * @param jobConfiguration Job configuration object + * @param jobConfiguration job configuration object */ public abstract void destroy(DestroyerContext context, LinkConfiguration linkConfiguration, http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java index 34f02c9..d6c186d 100644 --- a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java +++ b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java @@ -27,8 +27,8 @@ public abstract class Extractor<LinkConfiguration, JobConfiguration, Partition> * Extract data from source and pass them into the Sqoop. * * @param context Extractor context object - * @param linkConfiguration link configuration - * @param jobConfiguration Job configuration + * @param linkConfiguration link configuration object + * @param jobConfiguration job configuration object * @param partition Partition that this extract should work on */ public abstract void extract(ExtractorContext context, http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java index c9ae540..5c48fc3 100644 --- a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java +++ b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java @@ -34,28 +34,28 @@ public abstract class Initializer<LinkConfiguration, JobConfiguration> { * promoted to all other part of the workflow automatically. * * @param context Initializer context object - * @param linkConfiguration Connector's link configuration object - * @param jobConfiguration Connector's job configuration object + * @param linkConfiguration link configuration object + * @param jobConfiguration job configuration object */ - public abstract void initialize(InitializerContext context, - LinkConfiguration linkConfiguration, - JobConfiguration jobConfiguration); + public abstract void initialize(InitializerContext context, + LinkConfiguration linkConfiguration, + JobConfiguration jobConfiguration); - /** - * Return list of all jars that this particular connector needs to operate - * on following job. This method will be called after running initialize - * method. - * - * @return - */ - public List<String> getJars(InitializerContext context, - LinkConfiguration linkConfiguration, - JobConfiguration jobConfiguration) { - return new LinkedList<String>(); - } + /** + * Return list of all jars that this particular connector needs to operate + * on following job. This method will be called after running initialize + * method. + * + * @return + */ + public List<String> getJars(InitializerContext context, + LinkConfiguration linkConfiguration, + JobConfiguration jobConfiguration) { + return new LinkedList<String>(); + } - public abstract Schema getSchema(InitializerContext context, - LinkConfiguration linkConfiguration, - JobConfiguration jobConfiguration); + public abstract Schema getSchema(InitializerContext context, + LinkConfiguration linkConfiguration, + JobConfiguration jobConfiguration); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java index 434dcf6..cc32ada 100644 --- a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java +++ b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java @@ -26,12 +26,12 @@ public abstract class Loader<LinkConfiguration, JobConfiguration> { * Load data to target. * * @param context Loader context object - * @param linkConfiguration link configuration - * @param jobConfiguration Job configuration + * @param linkConfiguration link configuration object + * @param jobConfiguration job configuration object * @throws Exception */ - public abstract void load(LoaderContext context, - LinkConfiguration linkConfiguration, - JobConfiguration jobConfiguration) throws Exception; + public abstract void load(LoaderContext context, + LinkConfiguration linkConfiguration, + JobConfiguration jobConfiguration) throws Exception; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java index 8156762..57507df 100644 --- a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java +++ b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java @@ -31,12 +31,12 @@ public abstract class Partitioner<LinkConfiguration, JobConfiguration> { * Each partition will be then processed in separate extractor. * * @param context Partitioner context object - * @param linkConfiguration link configuration - * @param jobConfiguration Job configuration + * @param linkConfiguration link configuration object + * @param jobConfiguration job configuration object * @return */ - public abstract List<Partition> getPartitions(PartitionerContext context, - LinkConfiguration linkConfiguration, - JobConfiguration jobConfiguration); + public abstract List<Partition> getPartitions(PartitionerContext context, + LinkConfiguration linkConfiguration, + JobConfiguration jobConfiguration); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/validation/Validator.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/validation/Validator.java b/spi/src/main/java/org/apache/sqoop/validation/Validator.java index 894f412..2909905 100644 --- a/spi/src/main/java/org/apache/sqoop/validation/Validator.java +++ b/spi/src/main/java/org/apache/sqoop/validation/Validator.java @@ -29,21 +29,21 @@ public class Validator { /** * Validate link configuration object. * - * @param linkConfiguration Connection object to be validated + * @param linkConfiguration link config object to be validated * @return Validation status */ - public Validation validateLink(Object linkConfiguration) { - return new Validation(EmptyClass.class); + public ConfigValidator validateConfigForLink(Object linkConfiguration) { + return new ConfigValidator(EmptyClass.class); } /** * Validate configuration object for job . * - * @param jobConfiguration Job to be validated + * @param jobConfiguration Job config to be validated * @return Validation status */ - public Validation validateJob(Object jobConfiguration) { - return new Validation(EmptyClass.class); + public ConfigValidator validateConfigForJob(Object jobConfiguration) { + return new ConfigValidator(EmptyClass.class); } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java ---------------------------------------------------------------------- diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java index 25255ae..0c492ef 100644 --- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java +++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java @@ -141,7 +141,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine { * {@inheritDoc} */ @Override - public boolean isExecutionEngineSupported(Class executionEngineClass) { + public boolean isExecutionEngineSupported(Class<?> executionEngineClass) { return executionEngineClass == MapreduceExecutionEngine.class; } @@ -201,16 +201,16 @@ public class MapreduceSubmissionEngine extends SubmissionEngine { try { Job job = new Job(configuration); - // And finally put all configuration objects to credentials cache - ConfigurationUtils.setConnectorConnectionConfig(Direction.FROM, job, request.getConnectorLinkConfig(Direction.FROM)); - ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getConnectorJobConfig(Direction.FROM)); - ConfigurationUtils.setConnectorConnectionConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO)); - ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getConnectorJobConfig(Direction.TO)); + // link configs + ConfigurationUtils.setConnectorLinkConfig(Direction.FROM, job, request.getConnectorLinkConfig(Direction.FROM)); + ConfigurationUtils.setConnectorLinkConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO)); - ConfigurationUtils.setFrameworkConnectionConfig(Direction.FROM, job, request.getFrameworkLinkConfig(Direction.FROM)); - ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, request.getFrameworkLinkConfig(Direction.TO)); - ConfigurationUtils.setFrameworkJobConfig(job, request.getFrameworkJobConfig()); + // from and to configs + ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getJobConfig(Direction.FROM)); + ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getJobConfig(Direction.TO)); + ConfigurationUtils.setDriverConfig(job, request.getDriverConfig()); + // @TODO(Abe): Persist TO schema. ConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getFromSchema()); ConfigurationUtils.setConnectorSchema(Direction.TO, job, request.getSummary().getToSchema()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java index 9ecc9da..7504e76 100644 --- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java +++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java @@ -17,6 +17,12 @@ */ package org.apache.sqoop.test.minicluster; +import java.net.URL; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.codehaus.cargo.container.ContainerType; import org.codehaus.cargo.container.InstalledLocalContainer; @@ -25,16 +31,9 @@ import org.codehaus.cargo.container.configuration.LocalConfiguration; import org.codehaus.cargo.container.deployable.WAR; import org.codehaus.cargo.container.installer.Installer; import org.codehaus.cargo.container.installer.ZipURLInstaller; -import org.codehaus.cargo.container.property.GeneralPropertySet; import org.codehaus.cargo.generic.DefaultContainerFactory; import org.codehaus.cargo.generic.configuration.DefaultConfigurationFactory; -import java.net.URL; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - /** * Embedded tomcat Sqoop server mini cluster. * http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index 6074d36..af31769 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -17,15 +17,18 @@ */ package org.apache.sqoop.test.testcases; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; + import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.JobConf; import org.apache.log4j.Logger; import org.apache.sqoop.client.SubmissionCallback; import org.apache.sqoop.common.Direction; import org.apache.sqoop.connector.hdfs.configuration.ToFormat; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.test.asserts.ProviderAsserts; @@ -39,9 +42,6 @@ import org.apache.sqoop.validation.Status; import org.junit.AfterClass; import org.junit.BeforeClass; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; - /** * Base test case suitable for connector testing. * @@ -121,41 +121,50 @@ abstract public class ConnectorTestCase extends TomcatTestCase { } /** - * Fill link form based on currently active provider. + * Fill link config based on currently active provider. * * @param link MLink object to fill */ - protected void fillRdbmsLinkForm(MLink link) { - MFormList forms = link.getConnectorPart(); - forms.getStringInput("link.jdbcDriver").setValue(provider.getJdbcDriver()); - forms.getStringInput("link.connectionString").setValue(provider.getConnectionUrl()); - forms.getStringInput("link.username").setValue(provider.getConnectionUsername()); - forms.getStringInput("link.password").setValue(provider.getConnectionPassword()); + protected void fillRdbmsLinkConfig(MLink link) { + MConfigList configs = link.getConnectorLinkConfig(); + configs.getStringInput("linkConfig.jdbcDriver").setValue(provider.getJdbcDriver()); + configs.getStringInput("linkConfig.connectionString").setValue(provider.getConnectionUrl()); + configs.getStringInput("linkConfig.username").setValue(provider.getConnectionUsername()); + configs.getStringInput("linkConfig.password").setValue(provider.getConnectionPassword()); } /** - * Fill TO form with specific storage and output type. + * Fill TO config with specific storage and output type. * * @param job MJob object to fill * @param output Output type that should be set */ - protected void fillToJobForm(MJob job, ToFormat output) { - MFormList toForms = job.getConnectorPart(Direction.TO); - toForms.getEnumInput("toJobConfig.outputFormat").setValue(output); - toForms.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory()); + protected void fillHdfsToConfig(MJob job, ToFormat output) { + MConfigList toConfig = job.getJobConfig(Direction.TO); + toConfig.getEnumInput("toJobConfig.outputFormat").setValue(output); + toConfig.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory()); } /** - * Fill FROM form + * Fill FROM config * * @param job MJob object to fill */ - protected void fillFromJobForm(MJob job) { - MFormList forms = job.getConnectorPart(Direction.FROM); - forms.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory()); + protected void fillHdfsFromConfig(MJob job) { + MConfigList fromConfig = job.getJobConfig(Direction.FROM); + fromConfig.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory()); } /** + * Fill Driver config + * @param job + */ + protected void fillDriverConfig(MJob job) { + job.getDriverConfig().getStringInput("throttlingConfig.numExtractors").setValue("3"); + } + + + /** * Create table cities. */ protected void createTableCities() { @@ -186,7 +195,7 @@ abstract public class ConnectorTestCase extends TomcatTestCase { /** * Assert row in testing table. * - * @param conditions Conditions in form that are expected by the database provider + * @param conditions Conditions in config that are expected by the database provider * @param values Values that are expected in the table (with corresponding types) */ protected void assertRow(Object []conditions, Object ...values) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java index 63736ab..5e1d564 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java @@ -165,7 +165,7 @@ abstract public class TomcatTestCase { * @throws IOException */ protected void assertTo(String... lines) throws IOException { - // TODO(VB): fix this to be not directly dependent on mapreduce + // TODO(VB): fix this to be not directly dependent on hdfs/MR HdfsAsserts.assertMapreduceOutput(hdfsClient, getMapreduceDirectory(), lines); } @@ -175,7 +175,7 @@ abstract public class TomcatTestCase { * @param expectedFiles Expected number of files */ protected void assertToFiles(int expectedFiles) throws IOException { - // TODO(VB): fix this to be not directly dependent on mapreduce + // TODO(VB): fix this to be not directly dependent on hdfs/MR HdfsAsserts.assertMapreduceOutputFiles(hdfsClient, getMapreduceDirectory(), expectedFiles); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java index 5e1abc1..b1b3b16 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java @@ -19,8 +19,9 @@ package org.apache.sqoop.integration.connector.jdbc.generic; import org.apache.sqoop.common.Direction; import org.apache.sqoop.test.testcases.ConnectorTestCase; +import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MJob; import org.junit.Test; @@ -43,7 +44,7 @@ public class FromHDFSToRDBMSTest extends ConnectorTestCase { // RDBMS link MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkForm(rdbmsLink); + fillRdbmsLinkConfig(rdbmsLink); saveLink(rdbmsLink); // HDFS link @@ -53,10 +54,16 @@ public class FromHDFSToRDBMSTest extends ConnectorTestCase { // Job creation MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); - // Connector values - MFormList toForms = job.getConnectorPart(Direction.TO); - toForms.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - fillFromJobForm(job); + // set hdfs "FROM" config for the job, since the connector test case base class only has utilities for hdfs! + fillHdfsFromConfig(job); + + // set the rdms "TO" config here + MConfigList toConfig = job.getJobConfig(Direction.TO); + toConfig.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); + + // driver config + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); saveJob(job); executeJob(job); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java index 2dc0613..36f7443 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java @@ -17,17 +17,18 @@ */ package org.apache.sqoop.integration.connector.jdbc.generic; +import static org.junit.Assert.assertTrue; + import org.apache.sqoop.common.Direction; import org.apache.sqoop.connector.hdfs.configuration.ToFormat; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MConfigList; +import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MSubmission; import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.junit.Test; -import static org.junit.Assert.assertTrue; - /** * Import simple table with various configurations. */ @@ -39,7 +40,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { // RDBMS link MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkForm(rdbmsConnection); + fillRdbmsLinkConfig(rdbmsConnection); saveLink(rdbmsConnection); // HDFS link @@ -49,11 +50,17 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { // Job creation MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); - // Connector values - MFormList forms = job.getConnectorPart(Direction.FROM); - forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); - fillToJobForm(job, ToFormat.TEXT_FILE); + // srt rdbms "FROM" config + MConfigList fromConfig = job.getJobConfig(Direction.FROM); + fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); + fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + + // fill the hdfs "TO" config + fillHdfsToConfig(job, ToFormat.TEXT_FILE); + // driver config + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); + saveJob(job); executeJob(job); @@ -76,7 +83,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { // RDBMS link MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkForm(rdbmsLink); + fillRdbmsLinkConfig(rdbmsLink); saveLink(rdbmsLink); // HDFS link @@ -87,11 +94,11 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); // Connector values - MFormList forms = job.getConnectorPart(Direction.FROM); - forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); - forms.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country")); - fillToJobForm(job, ToFormat.TEXT_FILE); + MConfigList configs = job.getJobConfig(Direction.FROM); + configs.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); + configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + configs.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country")); + fillHdfsToConfig(job, ToFormat.TEXT_FILE); saveJob(job); MSubmission submission = getClient().startSubmission(job.getPersistenceId()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java index 729f95e..824a51d 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java @@ -19,8 +19,9 @@ package org.apache.sqoop.integration.connector.jdbc.generic; import org.apache.sqoop.common.Direction; import org.apache.sqoop.connector.hdfs.configuration.ToFormat; +import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MJob; import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.apache.sqoop.test.utils.ParametrizedUtils; @@ -59,8 +60,8 @@ public class PartitionerTest extends ConnectorTestCase { } private String partitionColumn; - private int extractors; private int maxOutputFiles; + private int extractors; public PartitionerTest(String partitionColumn, int expectedOutputFiles, int extractors) { this.partitionColumn = partitionColumn; @@ -74,7 +75,7 @@ public class PartitionerTest extends ConnectorTestCase { // RDBMS link MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkForm(rdbmsLink); + fillRdbmsLinkConfig(rdbmsLink); saveLink(rdbmsLink); // HDFS link @@ -84,13 +85,17 @@ public class PartitionerTest extends ConnectorTestCase { // Job creation MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); - // Connector values - MFormList forms = job.getConnectorPart(Direction.FROM); - forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn)); - fillToJobForm(job, ToFormat.TEXT_FILE); - forms = job.getFrameworkPart(); - forms.getIntegerInput("throttling.extractors").setValue(extractors); + // set the rdbms "FROM" config + MConfigList fromConfig = job.getJobConfig(Direction.FROM); + fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); + fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn)); + // fill hdfs "TO" config + fillHdfsToConfig(job, ToFormat.TEXT_FILE); + + // set driver config + MDriverConfig driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(extractors); + saveJob(job); executeJob(job); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java index 562a6a6..f42fa32 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java @@ -17,16 +17,16 @@ */ package org.apache.sqoop.integration.connector.jdbc.generic; +import static org.junit.Assert.assertEquals; + import org.apache.sqoop.common.Direction; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; import org.apache.sqoop.test.data.Cities; import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.junit.Test; -import static org.junit.Assert.assertEquals; - /** * */ @@ -46,7 +46,7 @@ public class TableStagedRDBMSTest extends ConnectorTestCase { // RDBMS link MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkForm(rdbmsLink); + fillRdbmsLinkConfig(rdbmsLink); saveLink(rdbmsLink); // HDFS link @@ -57,13 +57,19 @@ public class TableStagedRDBMSTest extends ConnectorTestCase { MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); - // Connector values - MFormList forms = job.getConnectorPart(Direction.TO); - forms.getStringInput("toJobConfig.tableName").setValue( + // fill HDFS "FROM" config + fillHdfsFromConfig(job); + + // fill rdbms "TO" config here + MConfigList configs = job.getJobConfig(Direction.TO); + configs.getStringInput("toJobConfig.tableName").setValue( provider.escapeTableName(getTableName())); - forms.getStringInput("toJobConfig.stageTableName").setValue( + configs.getStringInput("toJobConfig.stageTableName").setValue( provider.escapeTableName(stageTableName)); - fillFromJobForm(job); + // driver config + MConfigList driverConfig = job.getDriverConfig(); + driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3); + saveJob(job); executeJob(job); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java index 507ac53..055bc3d 100644 --- a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java @@ -23,7 +23,7 @@ import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.hdfs.configuration.ToFormat; import org.apache.sqoop.driver.DriverError; import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MJob; import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.junit.runner.RunWith; @@ -67,7 +67,7 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase { // RDBMS link MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); - fillRdbmsLinkForm(rdbmsLink); + fillRdbmsLinkConfig(rdbmsLink); saveLink(rdbmsLink); // HDFS link @@ -77,15 +77,21 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase { // Job creation MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); - // Connector values - MFormList forms = job.getConnectorPart(Direction.FROM); - forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); - forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); - // Framework values - fillToJobForm(job, ToFormat.TEXT_FILE); + // rdms "FROM" config + MConfigList fromConfig = job.getJobConfig(Direction.FROM); + fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName())); + fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + + // hdfs "TO" config + fillHdfsToConfig(job, ToFormat.TEXT_FILE); + + // driver config + MConfigList driverConfig = job.getDriverConfig(); + //driverConfig.getIntegerInput("throttlingConfig.extractors").setValue(3); + saveJob(job); - // Disable model entities as per parametrized run + // Disable model entities as per parameterized run getClient().enableLink(rdbmsLink.getPersistenceId(), enabledLink); getClient().enableJob(job.getPersistenceId(), enabledJob);