smengcl commented on code in PR #3131: URL: https://github.com/apache/ozone/pull/3131#discussion_r874074643
########## hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/multitenant/OMRangerBGSyncService.java: ########## @@ -0,0 +1,761 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.om.multitenant; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import com.google.common.base.Preconditions; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult.EmptyTaskResult; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMultiTenantManager; +import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; +import org.apache.hadoop.ozone.om.helpers.OmDBTenantState; +import org.apache.hadoop.ozone.om.multitenant.CachedTenantState.CachedAccessIdInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerServiceVersionSyncRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.protocol.Message; +import org.apache.ratis.protocol.RaftClientRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.VOLUME_LOCK; + +/** + * Background Sync thread that reads Multi-Tenancy state from OM DB + * and applies it to Ranger. + */ +public class OMRangerBGSyncService extends BackgroundService { + + public static final Logger LOG = + LoggerFactory.getLogger(OMRangerBGSyncService.class); + private static final ClientId CLIENT_ID = ClientId.randomId(); + private static final long ONE_HOUR_IN_MILLIS = 3600 * 1000; + + private final OzoneManager ozoneManager; + private final OMMetadataManager metadataManager; + private final OMMultiTenantManager multiTenantManager; + private final MultiTenantAccessAuthorizer authorizer; + + // Maximum number of attempts for each sync run + private static final int MAX_ATTEMPT = 2; + private final AtomicLong runCount = new AtomicLong(0); + private int rangerOzoneServiceId; + + private boolean isServiceStarted = false; + + static class BGRole { + private final String name; + private String id; + private final HashSet<String> userSet; + + BGRole(String n) { + this.name = n; + userSet = new HashSet<>(); + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void addUserPrincipal(String userPrincipal) { + userSet.add(userPrincipal); + } + + public HashSet<String> getUserSet() { + return userSet; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + // TODO: Do we care about userSet + return this.hashCode() == o.hashCode(); + } + } + + // This map will be used to keep all the policies that are found in + // OM DB and should have been in Ranger. Currently, we are only printing such + // policyID. This can result if a tenant is deleted but the system + // crashed. Its an easy recovery to retry the "tenant delete" operation. + // + // Maps from policy name to policy ID in Ranger + private final HashMap<String, String> mtRangerPoliciesToBeCreated = + new HashMap<>(); + + // We will track all the policies in Ranger here. After we have + // processed all the policies from OM DB, this map will + // be left with policies that we need to delete. + // + // Maps from policy name to policy ID in Ranger + private final HashMap<String, String> mtRangerPoliciesToBeDeleted = + new HashMap<>(); + + // This map will keep all the Multi-Tenancy related roles from Ranger. + private final HashMap<String, BGRole> mtRangerRoles = new HashMap<>(); + + // Keep OM DB mapping of Roles -> list of user principals. + private final HashMap<String, HashSet<String>> mtOMDBRoles = new HashMap<>(); + + // Every BG ranger sync cycle we update this + private long lastRangerPolicyLoadTime; + + public OMRangerBGSyncService(OzoneManager ozoneManager, + MultiTenantAccessAuthorizer authorizer, long interval, + TimeUnit unit, long serviceTimeout) + throws IOException { + super("OMRangerBGSyncService", interval, unit, 1, serviceTimeout); + + this.ozoneManager = ozoneManager; + this.metadataManager = ozoneManager.getMetadataManager(); + this.multiTenantManager = ozoneManager.getMultiTenantManager(); + + this.authorizer = authorizer; + + if (authorizer != null) { + if (authorizer instanceof MultiTenantAccessAuthorizerRangerPlugin) { + MultiTenantAccessAuthorizerRangerPlugin rangerAuthorizer = + (MultiTenantAccessAuthorizerRangerPlugin) authorizer; + rangerOzoneServiceId = rangerAuthorizer.getRangerOzoneServiceId(); + } else if ( + !(authorizer instanceof MultiTenantAccessAuthorizerDummyPlugin)) { + throw new OMException("Unsupported MultiTenantAccessAuthorizer: " + + authorizer.getClass().getSimpleName(), + ResultCodes.INTERNAL_ERROR); + } + } else { + // authorizer can be null for unit tests Review Comment: done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
