[
https://issues.apache.org/jira/browse/HUDI-1138?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17392803#comment-17392803
]
ASF GitHub Bot commented on HUDI-1138:
--------------------------------------
yihua commented on a change in pull request #3233:
URL: https://github.com/apache/hudi/pull/3233#discussion_r682352714
##########
File path:
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java
##########
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.timeline.service.handlers;
+
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.metrics.Registry;
+import org.apache.hudi.common.model.IOType;
+import org.apache.hudi.common.table.view.FileSystemViewManager;
+import org.apache.hudi.timeline.service.TimelineService;
+import
org.apache.hudi.timeline.service.handlers.marker.MarkerCreationCompletableFuture;
+import
org.apache.hudi.timeline.service.handlers.marker.MarkerCreationDispatchingRunnable;
+import org.apache.hudi.timeline.service.handlers.marker.MarkerDirState;
+
+import io.javalin.Context;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+/**
+ * REST Handler servicing marker requests.
+ *
+ * The marker creation requests are handled asynchronous, while other types of
requests
+ * are handled synchronous.
+ *
+ * Marker creation requests are batch processed periodically by a thread.
Each batch
+ * processing thread adds new markers to a marker file. Given that marker
file operation
+ * can take time, multiple concurrent threads can run at the same, while they
operate
+ * on different marker files storing mutually exclusive marker entries. At
any given
+ * time, a marker file is touched by at most one thread to guarantee
consistency.
+ * Below is an example of running batch processing threads.
+ *
+ * |-----| batch interval
+ * Thread 1 |-------------------------->| writing to MARKERS1
+ * Thread 2 |-------------------------->| writing to MARKERS2
+ * Thread 3 |-------------------------->| writing to MARKERS3
+ */
+public class MarkerHandler extends Handler {
+ private static final Logger LOG = LogManager.getLogger(MarkerHandler.class);
+
+ private final Registry metricsRegistry;
+ // a scheduled executor service to schedule dispatching of marker creation
requests
+ private final ScheduledExecutorService dispatchingExecutorService;
+ // an executor service to schedule the worker threads of batch processing
marker creation requests
+ private final ExecutorService batchingExecutorService;
+ // Parallelism for reading and deleting marker files
+ private final int parallelism;
+ // Marker directory states, {markerDirPath -> MarkerDirState instance}
+ private final Map<String, MarkerDirState> markerDirStateMap = new
HashMap<>();
+ // A long-running thread to dispatch marker creation requests to batch
processing threads
+ private final MarkerCreationDispatchingRunnable
markerCreationDispatchingRunnable;
+ private final AtomicBoolean firstCreationRequestSeen;
+ private transient HoodieEngineContext hoodieEngineContext;
+ private ScheduledFuture<?> dispatchingScheduledFuture;
+
+ public MarkerHandler(Configuration conf, TimelineService.Config
timelineServiceConfig,
+ HoodieEngineContext hoodieEngineContext, FileSystem
fileSystem,
+ FileSystemViewManager viewManager, Registry
metricsRegistry) throws IOException {
+ super(conf, timelineServiceConfig, fileSystem, viewManager);
+ LOG.debug("MarkerHandler FileSystem: " + this.fileSystem.getScheme());
+ LOG.debug("MarkerHandler batching params: batchNumThreads=" +
timelineServiceConfig.markerBatchNumThreads
+ + " batchIntervalMs=" + timelineServiceConfig.markerBatchIntervalMs +
"ms");
+ this.hoodieEngineContext = hoodieEngineContext;
+ this.metricsRegistry = metricsRegistry;
+ this.parallelism = timelineServiceConfig.markerParallelism;
+ this.dispatchingExecutorService =
Executors.newSingleThreadScheduledExecutor();
+ this.batchingExecutorService =
Executors.newFixedThreadPool(timelineServiceConfig.markerBatchNumThreads);
+ this.markerCreationDispatchingRunnable =
+ new MarkerCreationDispatchingRunnable(markerDirStateMap,
batchingExecutorService);
+ this.firstCreationRequestSeen = new AtomicBoolean(false);
+ }
+
+ /**
+ * Stops the dispatching of marker creation requests.
+ */
+ public void stop() {
+ if (dispatchingScheduledFuture != null) {
+ dispatchingScheduledFuture.cancel(true);
+ }
+ dispatchingExecutorService.shutdown();
+ batchingExecutorService.shutdown();
+ }
+
+ /**
+ * @param markerDir marker directory path
+ * @return all marker paths in the marker directory
+ */
+ public Set<String> getAllMarkers(String markerDir) {
+ MarkerDirState markerDirState = getMarkerDirState(markerDir);
+ return markerDirState.getAllMarkers();
+ }
+
+ /**
+ * @param markerDir marker directory path
+ * @return all marker paths of write IO type "CREATE" and "MERGE"
+ */
+ public Set<String> getCreateAndMergeMarkers(String markerDir) {
+ return getAllMarkers(markerDir).stream()
+ .filter(markerName -> !markerName.endsWith(IOType.APPEND.name()))
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * @param markerDir marker directory path
+ * @return {@code true} if the marker directory exists; {@code false}
otherwise.
+ */
+ public boolean doesMarkerDirExist(String markerDir) {
+ MarkerDirState markerDirState = getMarkerDirState(markerDir);
+ return markerDirState.exists();
+ }
+
+ /**
+ * Generates a future for an async marker creation request
+ *
+ * The future is added to the marker creation future list and waits for the
next batch processing
+ * of marker creation requests.
+ *
+ * @param context Javalin app context
+ * @param markerDir marker directory path
+ * @param markerName marker name
+ * @return the {@code CompletableFuture} instance for the request
+ */
+ public CompletableFuture<String> createMarker(Context context, String
markerDir, String markerName) {
+ LOG.info("Request: create marker " + markerDir + " " + markerName);
+ MarkerCreationCompletableFuture future = new
MarkerCreationCompletableFuture(context, markerDir, markerName);
+ // Add the future to the list
+ MarkerDirState markerDirState = getMarkerDirState(markerDir);
+ markerDirState.addMarkerCreationFuture(future);
+ if (firstCreationRequestSeen.getAndSet(true)) {
+ dispatchingScheduledFuture =
dispatchingExecutorService.scheduleAtFixedRate(markerCreationDispatchingRunnable,
+ timelineServiceConfig.markerBatchIntervalMs,
timelineServiceConfig.markerBatchIntervalMs,
+ TimeUnit.MILLISECONDS);
+ }
+ return future;
+ }
+
+ /**
+ * Deletes markers in the directory.
+ *
+ * @param markerDir marker directory path
+ * @return {@code true} if successful; {@code false} otherwise.
+ */
+ public Boolean deleteMarkers(String markerDir) {
+ boolean result = getMarkerDirState(markerDir).deleteAllMarkers();
+ markerDirStateMap.remove(markerDir);
+ return result;
+ }
+
+ private MarkerDirState getMarkerDirState(String markerDir) {
+ MarkerDirState markerDirState = markerDirStateMap.get(markerDir);
+ if (markerDirState == null) {
+ synchronized (markerDirStateMap) {
+ markerDirState = new MarkerDirState(markerDir,
timelineServiceConfig.markerBatchNumThreads,
Review comment:
Right, to avoid concurrent instantiation. Fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Re-implement marker files via timeline server
> ---------------------------------------------
>
> Key: HUDI-1138
> URL: https://issues.apache.org/jira/browse/HUDI-1138
> Project: Apache Hudi
> Issue Type: Improvement
> Components: Writer Core
> Affects Versions: 0.9.0
> Reporter: Vinoth Chandar
> Assignee: Ethan Guo
> Priority: Blocker
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> Even as you can argue that RFC-15/consolidated metadata, removes the need for
> deleting partial files written due to spark task failures/stage retries. It
> will still leave extra files inside the table (and users will pay for it
> every month) and we need the marker mechanism to be able to delete these
> partial files.
> Here we explore if we can improve the current marker file mechanism, that
> creates one marker file per data file written, by
> Delegating the createMarker() call to the driver/timeline server, and have it
> create marker metadata into a single file handle, that is flushed for
> durability guarantees
>
> P.S: I was tempted to think Spark listener mechanism can help us deal with
> failed tasks, but it has no guarantees. the writer job could die without
> deleting a partial file. i.e it can improve things, but cant provide
> guarantees
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
