fapifta commented on code in PR #6932: URL: https://github.com/apache/ozone/pull/6932#discussion_r1807139806
########## hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java: ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.request.validation; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.Version; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Registry that loads and stores the request validators to be applied by + * a service. + */ +public class ValidatorRegistry<RequestType extends Enum<RequestType>> { + + private final Map<Class<? extends Version>, EnumMap<RequestType, + EnumMap<RequestProcessingPhase, IndexedItems<Method, Integer>>>> indexedValidatorMap; + + /** + * Creates a {@link ValidatorRegistry} instance that discovers validation + * methods in the provided package and the packages in the same resource. + * A validation method is recognized by all the annotations classes which + * are annotated by {@link RegisterValidator} annotation that contains + * important information about how and when to use the validator. + * @param validatorPackage the main package inside which validatiors should + * be discovered. + */ + public ValidatorRegistry(Class<RequestType> requestType, + String validatorPackage, + Set<Class<? extends Version>> allowedVersionTypes, + Set<RequestProcessingPhase> allowedProcessingPhases) { + this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedVersionTypes, allowedProcessingPhases); + } + + private Class<?> getReturnTypeOfAnnotationMethod(Class<? extends Annotation> clzz, String methodName) { + try { + return clzz.getMethod(methodName).getReturnType(); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Method " + methodName + " not found in class:" + clzz.getCanonicalName()); + } + } + + /** + * Creates a {@link ValidatorRegistry} instance that discovers validation + * methods under the provided URL. + * A validation method is recognized by all annotations having the annotated by {@link RegisterValidator} + * annotation that contains important information about how and when to use + * the validator. + * @param searchUrls the path in which the annotated methods are searched. + */ + public ValidatorRegistry(Class<RequestType> requestType, + Collection<URL> searchUrls, + Set<Class<? extends Version>> allowedVersionTypes, + Set<RequestProcessingPhase> allowedProcessingPhases) { + Class<RequestType[]> requestArrayClass = (Class<RequestType[]>) Array.newInstance(requestType, 0) + .getClass(); + Set<Class<? extends Annotation>> validatorsToBeRegistered = + new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("")) + .setScanners(Scanners.TypesAnnotated) + .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() + .filter(annotationClass -> getReturnTypeOfAnnotationMethod((Class<? extends Annotation>) annotationClass, + RegisterValidator.REQUEST_TYPE_METHOD_NAME) + .equals(requestArrayClass)) + .filter(annotationClass -> allowedVersionTypes.contains(getReturnTypeOfAnnotationMethod( + (Class<? extends Annotation>) annotationClass, + RegisterValidator.APPLY_UNTIL_METHOD_NAME))) + .map(annotationClass -> (Class<? extends Annotation>) annotationClass) + .collect(Collectors.toSet()); + this.indexedValidatorMap = allowedVersionTypes.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), + versionClass -> new EnumMap<>(requestType))); + Reflections reflections = new Reflections(new ConfigurationBuilder() + .setUrls(searchUrls) + .setScanners(Scanners.MethodsAnnotated) + .setParallel(true) + ); + initMaps(requestArrayClass, allowedProcessingPhases, validatorsToBeRegistered, reflections); + } + + /** + * Get the validators that has to be run in the given list of, + * for the given requestType and for the given request versions. + * {@link RequestProcessingPhase}. + * + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @param requestVersions different versions extracted from the request. + * @return the list of validation methods that has to run. + */ + public List<Method> validationsFor(RequestType requestType, + RequestProcessingPhase phase, + List<? extends Version> requestVersions) { + return requestVersions.stream() + .flatMap(requestVersion -> this.validationsFor(requestType, phase, requestVersion).stream()) + .distinct().collect(Collectors.toList()); + } + + /** + * Get the validators that has to be run in the given list of, + * for the given requestType and for the given request versions. + * {@link RequestProcessingPhase}. + * + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @param requestVersion version extracted corresponding to the request. + * @return the list of validation methods that has to run. + */ + public <V extends Version> List<Method> validationsFor(RequestType requestType, + RequestProcessingPhase phase, + V requestVersion) { + + return Optional.ofNullable(this.indexedValidatorMap.get(requestVersion.getClass())) + .map(requestTypeMap -> requestTypeMap.get(requestType)).map(phaseMap -> phaseMap.get(phase)) + .map(indexedMethods -> requestVersion.version() < 0 ? + indexedMethods.getItemsEqualToIdx(requestVersion.version()) : + indexedMethods.getItemsGreaterThanIdx(requestVersion.version())) + .orElse(Collections.emptyList()); + + } + + /** + * Calls a specified method on the validator. + * @Throws IllegalArgumentException when the specified method in the validator is invalid. + */ + private <ReturnValue, Validator extends Annotation> ReturnValue callAnnotationMethod( + Validator validator, String methodName, Class<ReturnValue> returnValueType) { + try { + return (ReturnValue) validator.getClass().getMethod(methodName).invoke(validator); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Method " + methodName + " not found in class:" + + validator.getClass().getCanonicalName(), e); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new IllegalArgumentException("Error while invoking Method " + methodName + " from " + + validator.getClass().getCanonicalName(), e); + } + } + + private <Validator extends Annotation> Version getApplyUntilVersion(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.APPLY_UNTIL_METHOD_NAME, Version.class); + } + + private <Validator extends Annotation> RequestProcessingPhase getRequestPhase(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, + RequestProcessingPhase.class); + } + + private <Validator extends Annotation> RequestType[] getRequestType(Validator validator, + Class<RequestType[]> requestType) { + return callAnnotationMethod(validator, RegisterValidator.REQUEST_TYPE_METHOD_NAME, requestType); + } + + + private <V> void checkAllowedAnnotationValues(Set<V> values, V value, String valueName, String methodName) { + if (!values.contains(value)) { + throw new IllegalArgumentException( + String.format("Invalid %1$s defined at annotation defined for method : %2$s, Annotation value : %3$s " + + "Allowed versionType: %4$s", valueName, methodName, value.toString(), values)); + } + } + + /** + * Initializes the internal request validator store. + * The requests are stored in the following structure: + * - An EnumMap with the RequestType as the key, and in which + * - values are an EnumMap with the request processing phase as the key, and in which + * - values is an {@link IndexedItems } containing the validation list + * @param validatorsToBeRegistered collection of the annotated validtors to process. + */ + private void initMaps(Class<RequestType[]> requestType, + Set<RequestProcessingPhase> allowedPhases, + Collection<Class<? extends Annotation>> validatorsToBeRegistered, + Reflections reflections) { + for (Class<? extends Annotation> validator : validatorsToBeRegistered) { + registerValidator(requestType, allowedPhases, validator, reflections); + } + } + + private void registerValidator(Class<RequestType[]> requestType, + Set<RequestProcessingPhase> allowedPhases, + Class<? extends Annotation> validatorToBeRegistered, + Reflections reflections) { + Collection<Method> methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); + Class<? extends Version> versionClass = (Class<? extends Version>) + this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.APPLY_UNTIL_METHOD_NAME); + List<Pair<? extends Annotation, Method>> sortedMethodsByApplyUntilVersion = methods.stream() + .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) + .sorted((validatorMethodPair1, validatorMethodPair2) -> + Integer.compare( + this.getApplyUntilVersion(validatorMethodPair1.getKey()).version(), + this.getApplyUntilVersion(validatorMethodPair2.getKey()).version())) + .collect(Collectors.toList()); + for (Pair<? extends Annotation, Method> validatorMethodPair : sortedMethodsByApplyUntilVersion) { + Annotation validator = validatorMethodPair.getKey(); + Method method = validatorMethodPair.getValue(); + Version applyUntilVersion = this.getApplyUntilVersion(validator); + RequestProcessingPhase phase = this.getRequestPhase(validator); + checkAllowedAnnotationValues(allowedPhases, phase, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, + method.getName()); + Set<RequestType> types = Sets.newHashSet(this.getRequestType(validator, requestType)); + method.setAccessible(true); + for (RequestType type : types) { + EnumMap<RequestType, EnumMap<RequestProcessingPhase, IndexedItems<Method, Integer>>> requestMap = + this.indexedValidatorMap.get(versionClass); + EnumMap<RequestProcessingPhase, IndexedItems<Method, Integer>> phaseMap = Review Comment: The indexedValidatorMap has different mappings for different version classes at the top level, that makes the distinction possible. The background of this is more interesting though... We have a proto definition that defines messages between a server and a client. In this case the server is the OM, while the client is the Ozone client. Both of these parties have a version associated with them, in this case the ClientVersion, and the OmLayoutVersion. Both sides can be aware of the corresponding version of the other side within its release version (though afaik the Ozone client does not care or may not know about the OmLayoutVersion as we did not handle compatibility from that side). Now... When an older client sends a request to a newer server, then the server has its client version hardcoded in its codebase, and compares that with the client version in the request. If the client version is different, it selects the validators that are required for the client version supplied in the request from the client version type related mappings. On top of this, the server knows is hardcoded server version, and its finalized server version, and if there is a difference between the two, then we are in a pre-finalized state, and it selects the validators that has to be applied for the finalized server version it has. When a newer client sends a request, the client version translates to FUTURE_VERSION, as the older server does not have the new client version enum value. In this case it does not apply any validators on the client version, but it still applies server version validators if there are any non-finalized server versions. (Like in a case where server has a version released in Ozone 3.2, the client is using code released with version 3.3, while the server side was just updated from 3.1 to 3.2.) Why we have validations for both the server and the client version? There may be changes in the metadata layout on the server side that does not affect the client side, so the client version is not bumped, but the server version is bumped. In this case we need the server side version to properly finalize things. But on the other hand after the finalization, we need to validate the client version, as older clients may not understand data returned in the response, as that may contain newly added things. In this case we either need to send an exception to the client that informs the client about the inability to serve the response to an old client, or we need to translate the data so that the old client understands. This part we can not spare, and after finalizing an upgrade old clients are affected. So that is why we have a different set of validators based on versions, as one case is to handle older clients, the other is to handle the pre finalized state. In the OM and Ozone client relation for which we have the system set up, OMClientVersionValidator is dealing with old clients, while the OMLayoutFeatureValidator deals with the pre-finalized state of OM. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
