Repository: sqoop Updated Branches: refs/heads/SQOOP-1367 1a0e04ec4 -> 88eb766e5
SQOOP-1441: Sqoop2: Validations: Enforce defined validations Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/88eb766e Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/88eb766e Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/88eb766e Branch: refs/heads/SQOOP-1367 Commit: 88eb766e56311be7ee0495c2cf767fa33d9e6e0a Parents: 1a0e04e Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Aug 18 10:46:19 2014 -0700 Committer: Abraham Elmahrek <[email protected]> Committed: Mon Aug 18 11:04:08 2014 -0700 ---------------------------------------------------------------------- .../java/org/apache/sqoop/model/FormUtils.java | 59 +++++++ .../java/org/apache/sqoop/model/ModelError.java | 2 + .../sqoop/validation/ValidationResult.java | 82 ++++++++++ .../sqoop/validation/ValidationRunner.java | 145 +++++++++++++++++ .../sqoop/validation/validators/Validator.java | 13 +- .../sqoop/validation/TestValidationRunner.java | 154 +++++++++++++++++++ 6 files changed, 454 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/88eb766e/common/src/main/java/org/apache/sqoop/model/FormUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/FormUtils.java b/common/src/main/java/org/apache/sqoop/model/FormUtils.java index 27db8af..d9666c8 100644 --- a/common/src/main/java/org/apache/sqoop/model/FormUtils.java +++ b/common/src/main/java/org/apache/sqoop/model/FormUtils.java @@ -33,6 +33,8 @@ import java.util.Map; /** * Util class for transforming data from correctly annotated configuration * objects to different structures and vice-versa. + * + * TODO: This class should see some overhaul into more reusable code, especially expose and re-use the methods at the end. */ public class FormUtils { @@ -466,4 +468,61 @@ public class FormUtils { } } + public static String getName(Field input, Input annotation) { + return input.getName(); + } + + public static String getName(Field form, Form annotation) { + return form.getName(); + } + + public static ConfigurationClass getConfigurationClassAnnotation(Object object, boolean strict) { + ConfigurationClass annotation = object.getClass().getAnnotation(ConfigurationClass.class); + + if(strict && annotation == null) { + throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationClass on class " + object.getClass().getName()); + } + + return annotation; + } + + public static FormClass getFormClassAnnotation(Object object, boolean strict) { + FormClass annotation = object.getClass().getAnnotation(FormClass.class); + + if(strict && annotation == null) { + throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationClass on class " + object.getClass().getName()); + } + + return annotation; + } + + public static Form getFormAnnotation(Field field, boolean strict) { + Form annotation = field.getAnnotation(Form.class); + + if(strict && annotation == null) { + throw new SqoopException(ModelError.MODEL_003, "Missing annotation Form on Field " + field.getName() + " on class " + field.getDeclaringClass().getName()); + } + + return annotation; + } + + public static Input getInputAnnotation(Field field, boolean strict) { + Input annotation = field.getAnnotation(Input.class); + + if(strict && annotation == null) { + throw new SqoopException(ModelError.MODEL_003, "Missing annotation Input on Field " + field.getName() + " on class " + field.getDeclaringClass().getName()); + } + + return annotation; + } + + public static Object getFieldValue(Field field, Object object) { + try { + field.setAccessible(true); + return field.get(object); + } catch (IllegalAccessException e) { + throw new SqoopException(ModelError.MODEL_012, e); + } + } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/88eb766e/common/src/main/java/org/apache/sqoop/model/ModelError.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/ModelError.java b/common/src/main/java/org/apache/sqoop/model/ModelError.java index 1f466fe..470b61c 100644 --- a/common/src/main/java/org/apache/sqoop/model/ModelError.java +++ b/common/src/main/java/org/apache/sqoop/model/ModelError.java @@ -46,6 +46,8 @@ public enum ModelError implements ErrorCode { MODEL_011("Input do not exist"), + MODEL_012("Can't get value from object"), + ; private final String message; http://git-wip-us.apache.org/repos/asf/sqoop/blob/88eb766e/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java b/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java new file mode 100644 index 0000000..abe5b11 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.validation; + +import org.apache.sqoop.validation.validators.Validator; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Result of validation execution. + */ +public class ValidationResult { + + /** + * All messages for each named item. + */ + Map<String, List<Message>> messages; + + /** + * Overall status. + */ + Status status; + + public ValidationResult() { + messages = new HashMap<String, List<Message>>(); + status = Status.getDefault(); + } + + /** + * Add given validator result to this instance. + * + * @param name Full name of the validated object + * @param validator Executed validator + */ + public void addValidator(String name, Validator validator) { + if(validator.getStatus() == Status.getDefault()) { + return; + } + + status = Status.getWorstStatus(status, validator.getStatus()); + if(messages.containsKey(name)) { + messages.get(name).addAll(validator.getMessages()); + } else { + messages.put(name, validator.getMessages()); + } + } + + /** + * Merge results with another validation result. + * + * @param result Other validation result + */ + public void merge(ValidationResult result) { + messages.putAll(result.messages); + status = Status.getWorstStatus(status, result.status); + } + + public Status getStatus() { + return status; + } + + public Map<String, List<Message>> getMessages() { + return messages; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/88eb766e/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java new file mode 100644 index 0000000..46e2d56 --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.validation; + +import org.apache.sqoop.model.ConfigurationClass; +import org.apache.sqoop.model.Form; +import org.apache.sqoop.model.FormClass; +import org.apache.sqoop.model.FormUtils; +import org.apache.sqoop.model.Input; +import org.apache.sqoop.utils.ClassUtils; +import org.apache.sqoop.validation.validators.Validator; + +import java.lang.reflect.Field; + +/** + * Validation runner that will run validators associated with given configuration + * class or form object. + * + * Execution follows following rules: + * * Run children first (Inputs -> Form -> Class) + * * If any children is not suitable (canProceed = false), skip running parent + * + * Which means that form validator don't have to repeat it's input validators as it will + * be never called if the input's are not valid. Similarly Class validators won't be called + * unless all forms will pass validators. + * + * TODO: Cache the validators instances, so that we don't have create new instance every time + */ +public class ValidationRunner { + + /** + * Validate given configuration instance. + * + * @param config Configuration instance + * @return + */ + public ValidationResult validate(Object config) { + ValidationResult result = new ValidationResult(); + ConfigurationClass globalAnnotation = FormUtils.getConfigurationClassAnnotation(config, true); + + // Iterate over all declared form and call their validators + for (Field field : config.getClass().getDeclaredFields()) { + field.setAccessible(true); + + Form formAnnotation = FormUtils.getFormAnnotation(field, false); + if(formAnnotation == null) { + continue; + } + + String formName = FormUtils.getName(field, formAnnotation); + ValidationResult r = validateForm(formName, FormUtils.getFieldValue(field, config)); + result.merge(r); + } + + // Call class validator only as long as we are in suitable state + if(result.getStatus().canProceed()) { + ValidationResult r = validateArray("", config, globalAnnotation.validators()); + result.merge(r); + } + + return result; + } + + /** + * Validate given form instance. + * + * @param formName Form's name to build full name for all inputs. + * @param form Form instance + * @return + */ + public ValidationResult validateForm(String formName, Object form) { + ValidationResult result = new ValidationResult(); + FormClass formAnnotation = FormUtils.getFormClassAnnotation(form, true); + + // Iterate over all declared inputs and call their validators + for (Field field : form.getClass().getDeclaredFields()) { + Input inputAnnotation = FormUtils.getInputAnnotation(field, false); + if(inputAnnotation == null) { + continue; + } + + String name = formName + "." + FormUtils.getName(field, inputAnnotation); + + ValidationResult r = validateArray(name, FormUtils.getFieldValue(field, form), inputAnnotation.validators()); + result.merge(r); + } + + // Call form validator only as long as we are in suitable state + if(result.getStatus().canProceed()) { + ValidationResult r = validateArray(formName, form, formAnnotation.validators()); + result.merge(r); + } + + return result; + } + + /** + * Execute array of validators on given object (can be input/form/class). + * + * @param name Full name of the object + * @param object Input, Form or Class instance + * @param classes Validators array + * @return + */ + private ValidationResult validateArray(String name, Object object, Class<? extends Validator> []classes) { + ValidationResult result = new ValidationResult(); + + for (Class<? extends Validator> klass : classes) { + Validator v = executeValidator(object, klass); + result.addValidator(name, v); + } + + return result; + } + + /** + * Execute single validator. + * + * @param object Input, Form or Class instance + * @param klass Validator's clas + * @return + */ + private Validator executeValidator(Object object, Class<? extends Validator> klass) { + Validator instance = (Validator) ClassUtils.instantiate(klass); + instance.validate(object); + return instance; + } + + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/88eb766e/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java b/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java index 5c24b1a..bdb7c20 100644 --- a/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java +++ b/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java @@ -44,26 +44,37 @@ abstract public class Validator<T> { */ private List<Message> messages; + /** + * Overall status of the validation. + */ + private Status status; + public Validator() { reset(); } protected void addMessage(Message msg) { + status = Status.getWorstStatus(status, msg.getStatus()); messages.add(msg); } protected void addMessage(Status status, String msg) { - messages.add(new Message(status, msg)); + addMessage(new Message(status, msg)); } public List<Message> getMessages() { return messages; } + public Status getStatus() { + return status; + } + /** * Reset validator state (all previous messages). */ public void reset() { messages = new LinkedList<Message>(); + status = Status.getDefault(); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/88eb766e/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java b/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java new file mode 100644 index 0000000..1961425 --- /dev/null +++ b/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.validation; + +import org.apache.sqoop.model.ConfigurationClass; +import org.apache.sqoop.model.Form; +import org.apache.sqoop.model.FormClass; +import org.apache.sqoop.model.Input; +import org.apache.sqoop.validation.validators.NotEmpty; +import org.apache.sqoop.validation.validators.NotNull; +import org.apache.sqoop.validation.validators.Validator; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + */ +public class TestValidationRunner { + + @FormClass(validators = {FormA.FormValidator.class}) + public static class FormA { + @Input(validators = {NotNull.class}) + String notNull; + + public static class FormValidator extends Validator<FormA> { + @Override + public void validate(FormA form) { + if(form.notNull == null) { + addMessage(Status.UNACCEPTABLE, "null"); + } + if("error".equals(form.notNull)) { + addMessage(Status.UNACCEPTABLE, "error"); + } + } + } + } + + @Test + public void testValidateForm() { + FormA form = new FormA(); + ValidationRunner runner = new ValidationRunner(); + ValidationResult result; + + // Null string should fail on Input level and should not call form level validators + form.notNull = null; + result = runner.validateForm("formName", form); + assertEquals(Status.UNACCEPTABLE, result.getStatus()); + assertEquals(1, result.getMessages().size()); + assertTrue(result.getMessages().containsKey("formName.notNull")); + + // String "error" should trigger form level error, but not Input level + form.notNull = "error"; + result = runner.validateForm("formName", form); + assertEquals(Status.UNACCEPTABLE, result.getStatus()); + assertEquals(1, result.getMessages().size()); + assertTrue(result.getMessages().containsKey("formName")); + + // Acceptable state + form.notNull = "This is truly random string"; + result = runner.validateForm("formName", form); + assertEquals(Status.FINE, result.getStatus()); + assertEquals(0, result.getMessages().size()); + } + + @FormClass + public static class FormB { + @Input(validators = {NotNull.class, NotEmpty.class}) + String str; + } + + @Test + public void testMultipleValidatorsOnSingleInput() { + FormB form = new FormB(); + ValidationRunner runner = new ValidationRunner(); + ValidationResult result; + + form.str = null; + result = runner.validateForm("formName", form); + assertEquals(Status.UNACCEPTABLE, result.getStatus()); + assertEquals(1, result.getMessages().size()); + assertTrue(result.getMessages().containsKey("formName.str")); + assertEquals(2, result.getMessages().get("formName.str").size()); + } + + @ConfigurationClass(validators = {ConfigurationA.ClassValidator.class}) + public static class ConfigurationA { + @Form FormA formA; + public ConfigurationA() { + formA = new FormA(); + } + + public static class ClassValidator extends Validator<ConfigurationA> { + @Override + public void validate(ConfigurationA conf) { + if("error".equals(conf.formA.notNull)) { + addMessage(Status.UNACCEPTABLE, "error"); + } + if("conf-error".equals(conf.formA.notNull)) { + addMessage(Status.UNACCEPTABLE, "conf-error"); + } + } + } + } + + @Test + public void testValidate() { + ConfigurationA conf = new ConfigurationA(); + ValidationRunner runner = new ValidationRunner(); + ValidationResult result; + + // Null string should fail on Input level and should not call form nor class level validators + conf.formA.notNull = null; + result = runner.validate(conf); + assertEquals(Status.UNACCEPTABLE, result.getStatus()); + assertEquals(1, result.getMessages().size()); + assertTrue(result.getMessages().containsKey("formA.notNull")); + + // String "error" should trigger form level error, but not Input nor class level + conf.formA.notNull = "error"; + result = runner.validate(conf); + assertEquals(Status.UNACCEPTABLE, result.getStatus()); + assertEquals(1, result.getMessages().size()); + assertTrue(result.getMessages().containsKey("formA")); + + // String "conf-error" should trigger class level error, but not Input nor Form level + conf.formA.notNull = "conf-error"; + result = runner.validate(conf); + assertEquals(Status.UNACCEPTABLE, result.getStatus()); + assertEquals(1, result.getMessages().size()); + assertTrue(result.getMessages().containsKey("")); + + // Valid string + conf.formA.notNull = "Valid string"; + result = runner.validate(conf); + assertEquals(Status.FINE, result.getStatus()); + assertEquals(0, result.getMessages().size()); + } +}
