[
https://issues.apache.org/jira/browse/GOBBLIN-2004?focusedWorklogId=908982&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-908982
]
ASF GitHub Bot logged work on GOBBLIN-2004:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 08/Mar/24 18:47
Start Date: 08/Mar/24 18:47
Worklog Time Spent: 10m
Work Description: phet commented on code in PR #3891:
URL: https://github.com/apache/gobblin/pull/3891#discussion_r1518062296
##########
gobblin-yarn/src/main/java/org/apache/gobblin/yarn/AbstractAppSecurityManager.java:
##########
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.yarn;
+
+import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.Credentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.util.ConfigUtils;
+import org.apache.gobblin.util.ExecutorsUtils;
+
+import static
org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
+
+
+/**
+ * <p>
+ * The super class for key management
+ * This class uses a scheduled task to do re-login to re-fetch token on a
+ * configurable schedule. It also uses a second scheduled task
+ * to renew the delegation token after each login. Both the re-login
interval and the token
+ * renewing interval are configurable.
+ * </p>
+ * @author Zihan Li
+ */
+public abstract class AbstractAppSecurityManager extends AbstractIdleService {
+
+ protected Logger LOGGER = LoggerFactory.getLogger(this.getClass().getName());
+
+ protected Config config;
+
+ protected final FileSystem fs;
+ protected final Path tokenFilePath;
+
+ protected Credentials credentials = new Credentials();
+ private final long loginIntervalInMinutes;
+ private final long tokenRenewIntervalInMinutes;
+ private final ScheduledExecutorService loginExecutor;
+ private final ScheduledExecutorService tokenRenewExecutor;
+
+ private Optional<ScheduledFuture<?>> scheduledTokenRenewTask =
Optional.absent();
+
+ // This flag is used to tell if this is the first login. If yes, no token
updated message will be
+ // sent to the controller and the participants as they may not be up running
yet. The first login
+ // happens after this class starts up so the token gets regularly refreshed
before the next login.
+ protected volatile boolean firstLogin = true;
+
+ public AbstractAppSecurityManager(Config config, FileSystem fs, Path
tokenFilePath) {
+ this.config = config;
+ this.fs = fs;
+ this.tokenFilePath = tokenFilePath;
+ this.fs.makeQualified(tokenFilePath);
+ this.loginIntervalInMinutes = ConfigUtils.getLong(config,
GobblinYarnConfigurationKeys.LOGIN_INTERVAL_IN_MINUTES,
+ GobblinYarnConfigurationKeys.DEFAULT_LOGIN_INTERVAL_IN_MINUTES);
+ this.tokenRenewIntervalInMinutes = ConfigUtils.getLong(config,
GobblinYarnConfigurationKeys.TOKEN_RENEW_INTERVAL_IN_MINUTES,
+ GobblinYarnConfigurationKeys.DEFAULT_TOKEN_RENEW_INTERVAL_IN_MINUTES);
+
+ this.loginExecutor = Executors.newSingleThreadScheduledExecutor(
+ ExecutorsUtils.newThreadFactory(Optional.of(LOGGER),
Optional.of("KeytabReLoginExecutor")));
+ this.tokenRenewExecutor = Executors.newSingleThreadScheduledExecutor(
+ ExecutorsUtils.newThreadFactory(Optional.of(LOGGER),
Optional.of("TokenRenewExecutor")));
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ LOGGER.info("Starting the " + this.getClass().getSimpleName());
+
+ LOGGER.info(
+ String.format("Scheduling the login task with an interval of %d
minute(s)", this.loginIntervalInMinutes));
+
+ // Schedule the Kerberos re-login task
+ this.loginExecutor.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ loginAndScheduleTokenRenewal();
+ }catch(Exception e){
+ LOGGER.error("Error during login, will continue the thread and try
next time.");
+ }
+ }
+ }, this.loginIntervalInMinutes, this.loginIntervalInMinutes,
TimeUnit.MINUTES);
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ LOGGER.info("Stopping the " + this.getClass().getSimpleName());
+
+ if (this.scheduledTokenRenewTask.isPresent()) {
Review Comment:
since this reads what's set in `scheduleTokenRenewalTask` and that is
clearly multi-threaded, let's synchronize both that method and this one (to
guarantee `shutdown()` is not called concurrently)
Issue Time Tracking
-------------------
Worklog Id: (was: 908982)
Time Spent: 1h 40m (was: 1.5h)
> Make a Gobblin yarn app launcher that does not depend on Helix
> --------------------------------------------------------------
>
> Key: GOBBLIN-2004
> URL: https://issues.apache.org/jira/browse/GOBBLIN-2004
> Project: Apache Gobblin
> Issue Type: Improvement
> Reporter: Matthew Ho
> Priority: Major
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
