yihua commented on a change in pull request #3233: URL: https://github.com/apache/hudi/pull/3233#discussion_r682348725
########## File path: hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java ########## @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.handlers; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.model.IOType; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.timeline.service.TimelineService; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationCompletableFuture; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationDispatchingRunnable; +import org.apache.hudi.timeline.service.handlers.marker.MarkerDirState; + +import io.javalin.Context; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +/** + * REST Handler servicing marker requests. + * + * The marker creation requests are handled asynchronous, while other types of requests + * are handled synchronous. + * + * Marker creation requests are batch processed periodically by a thread. Each batch + * processing thread adds new markers to a marker file. Given that marker file operation + * can take time, multiple concurrent threads can run at the same, while they operate + * on different marker files storing mutually exclusive marker entries. At any given + * time, a marker file is touched by at most one thread to guarantee consistency. + * Below is an example of running batch processing threads. + * + * |-----| batch interval + * Thread 1 |-------------------------->| writing to MARKERS1 + * Thread 2 |-------------------------->| writing to MARKERS2 + * Thread 3 |-------------------------->| writing to MARKERS3 + */ +public class MarkerHandler extends Handler { + private static final Logger LOG = LogManager.getLogger(MarkerHandler.class); + + private final Registry metricsRegistry; + // a scheduled executor service to schedule dispatching of marker creation requests + private final ScheduledExecutorService dispatchingExecutorService; + // an executor service to schedule the worker threads of batch processing marker creation requests + private final ExecutorService batchingExecutorService; + // Parallelism for reading and deleting marker files + private final int parallelism; + // Marker directory states, {markerDirPath -> MarkerDirState instance} + private final Map<String, MarkerDirState> markerDirStateMap = new HashMap<>(); + // A long-running thread to dispatch marker creation requests to batch processing threads Review comment: Fixed. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
