yuqi1129 commented on code in PR #13724: URL: https://github.com/apache/gravitino/pull/13724#discussion_r4245571267
########## plugins/k8s-job-executor/src/main/java/org/apache/gravitino/job/k8s/K8sClientUtils.java: ########## @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.job.k8s; + +import com.google.common.base.Preconditions; +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.ConfigBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; +import io.fabric8.kubernetes.client.OAuthTokenProvider; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Creates the Kubernetes client of the job cluster from the job executor configurations. */ +public final class K8sClientUtils { + + /** + * An {@link OAuthTokenProvider} that reads the token from a file, and reads it again whenever the + * file changes, so that a rotated token is picked up without restarting the server. + */ + static final class FileTokenProvider implements OAuthTokenProvider { + + // How often a token file that stays empty or unreadable is reported. + private static final long FAILURE_LOG_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(5); + + private final Path tokenFile; + + private FileTime lastModified; + + private String token; + + private boolean failing; + + private long failureLoggedAtNanos; + + FileTokenProvider(Path tokenFile) { + this.tokenFile = tokenFile; + } + + @Override + public synchronized String getToken() { + try { + FileTime modified = Files.getLastModifiedTime(tokenFile); + if (token == null || !modified.equals(lastModified)) { + String content = new String(Files.readAllBytes(tokenFile), StandardCharsets.UTF_8).trim(); + if (StringUtils.isBlank(content)) { + if (token == null) { + throw new IllegalStateException("The token file " + tokenFile + " is empty"); + } + // The file is being rewritten. Its modification time isn't remembered: the new token + // may be written within the same tick of the file system's clock, and would then + // never be read. + return keepToken("is empty", null); + } + token = content; + lastModified = modified; + } + } catch (IOException e) { + if (token == null) { + throw new UncheckedIOException("Failed to read the token file " + tokenFile, e); + } + // For example, the file is being replaced. + return keepToken("can't be read", e); + } + failing = false; + return token; + } + + /** + * Keeps the token read before while the file has no token, checking the file again on the next + * request. The problem is reported when it starts, and then at an interval rather than on every + * request, so that a file that stays broken remains visible without flooding the log. + */ + private String keepToken(String problem, @Nullable IOException cause) { + long now = System.nanoTime(); + if (!failing || now - failureLoggedAtNanos >= FAILURE_LOG_INTERVAL_NANOS) { + LOG.warn( + "The token file {} {}, keeping the token read before. Requests will fail once that" + + " token expires", + tokenFile, + problem, + cause); + failing = true; + failureLoggedAtNanos = now; + } + return token; + } + } + + private static final Logger LOG = LoggerFactory.getLogger(K8sClientUtils.class); + + private static final Pattern SERVER_SCHEME = + Pattern.compile("^https?://", Pattern.CASE_INSENSITIVE); + + private K8sClientUtils() {} + + /** + * Creates the Kubernetes client of the job cluster. + * + * @param configs the job executor configurations + * @return the Kubernetes client + */ + public static KubernetesClient createClient(K8sJobExecutorConfigs configs) { + return new KubernetesClientBuilder().withConfig(createClientConfig(configs)).build(); Review Comment: [Question] Is the fabric8 default retry policy what part 3's callers should get? Every connection mode here keeps fabric8's defaults: 10 s connect and request timeouts, and `requestRetryBackoffLimit` 10 with a 100 ms exponential backoff. I measured one call, `client.pods().inNamespace("jobs").list()`, through `createClient` with the shaded jar on 67022ada24: - API server refusing connections (`https://127.0.0.1:1`): `KubernetesClientException` after 19.2 s. - Unreachable API server (`https://10.255.255.1:6443`): `KubernetesClientException` after 129 s. So while the job cluster is down, each status query or submission in part 3 holds its calling thread for up to about two minutes before failing. Whether that fits depends on what calls the executor in #13700, for example the job status poller, the REST request that submits a job, or shutdown. It is easy to bound here, for example with `withRequestRetryBackoffLimit(...)` and the timeouts set on the `Config` in all three modes, optionally behind configuration keys. Could you check this against #13700's callers, and either set limits here or say in the Javadoc that callers have to bound the wait themselves? Verified by: the probe above, run twice against the shaded jar. I did not check which failures fabric8 7.2.0 retries beyond connect failures. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
