nsivabalan commented on a change in pull request #3233: URL: https://github.com/apache/hudi/pull/3233#discussion_r682045334
########## File path: hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java ########## @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.handlers; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.model.IOType; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.timeline.service.TimelineService; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationCompletableFuture; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationDispatchingRunnable; +import org.apache.hudi.timeline.service.handlers.marker.MarkerDirState; + +import io.javalin.Context; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +/** + * REST Handler servicing marker requests. + * + * The marker creation requests are handled asynchronous, while other types of requests + * are handled synchronous. + * + * Marker creation requests are batch processed periodically by a thread. Each batch + * processing thread adds new markers to a marker file. Given that marker file operation + * can take time, multiple concurrent threads can run at the same, while they operate + * on different marker files storing mutually exclusive marker entries. At any given + * time, a marker file is touched by at most one thread to guarantee consistency. + * Below is an example of running batch processing threads. + * + * |-----| batch interval + * Thread 1 |-------------------------->| writing to MARKERS1 + * Thread 2 |-------------------------->| writing to MARKERS2 + * Thread 3 |-------------------------->| writing to MARKERS3 + */ +public class MarkerHandler extends Handler { + private static final Logger LOG = LogManager.getLogger(MarkerHandler.class); + + private final Registry metricsRegistry; + // a scheduled executor service to schedule dispatching of marker creation requests + private final ScheduledExecutorService dispatchingExecutorService; + // an executor service to schedule the worker threads of batch processing marker creation requests + private final ExecutorService batchingExecutorService; + // Parallelism for reading and deleting marker files + private final int parallelism; + // Marker directory states, {markerDirPath -> MarkerDirState instance} + private final Map<String, MarkerDirState> markerDirStateMap = new HashMap<>(); + // A long-running thread to dispatch marker creation requests to batch processing threads + private final MarkerCreationDispatchingRunnable markerCreationDispatchingRunnable; + private final AtomicBoolean firstCreationRequestSeen; + private transient HoodieEngineContext hoodieEngineContext; + private ScheduledFuture<?> dispatchingScheduledFuture; + + public MarkerHandler(Configuration conf, TimelineService.Config timelineServiceConfig, + HoodieEngineContext hoodieEngineContext, FileSystem fileSystem, + FileSystemViewManager viewManager, Registry metricsRegistry) throws IOException { + super(conf, timelineServiceConfig, fileSystem, viewManager); + LOG.debug("MarkerHandler FileSystem: " + this.fileSystem.getScheme()); + LOG.debug("MarkerHandler batching params: batchNumThreads=" + timelineServiceConfig.markerBatchNumThreads + + " batchIntervalMs=" + timelineServiceConfig.markerBatchIntervalMs + "ms"); + this.hoodieEngineContext = hoodieEngineContext; + this.metricsRegistry = metricsRegistry; + this.parallelism = timelineServiceConfig.markerParallelism; + this.dispatchingExecutorService = Executors.newSingleThreadScheduledExecutor(); + this.batchingExecutorService = Executors.newFixedThreadPool(timelineServiceConfig.markerBatchNumThreads); + this.markerCreationDispatchingRunnable = + new MarkerCreationDispatchingRunnable(markerDirStateMap, batchingExecutorService); + this.firstCreationRequestSeen = new AtomicBoolean(false); + } + + /** + * Stops the dispatching of marker creation requests. + */ + public void stop() { + if (dispatchingScheduledFuture != null) { + dispatchingScheduledFuture.cancel(true); + } + dispatchingExecutorService.shutdown(); + batchingExecutorService.shutdown(); + } + + /** + * @param markerDir marker directory path + * @return all marker paths in the marker directory + */ + public Set<String> getAllMarkers(String markerDir) { + MarkerDirState markerDirState = getMarkerDirState(markerDir); + return markerDirState.getAllMarkers(); + } + + /** + * @param markerDir marker directory path + * @return all marker paths of write IO type "CREATE" and "MERGE" + */ + public Set<String> getCreateAndMergeMarkers(String markerDir) { + return getAllMarkers(markerDir).stream() + .filter(markerName -> !markerName.endsWith(IOType.APPEND.name())) + .collect(Collectors.toSet()); + } + + /** + * @param markerDir marker directory path + * @return {@code true} if the marker directory exists; {@code false} otherwise. + */ + public boolean doesMarkerDirExist(String markerDir) { + MarkerDirState markerDirState = getMarkerDirState(markerDir); + return markerDirState.exists(); + } + + /** + * Generates a future for an async marker creation request + * + * The future is added to the marker creation future list and waits for the next batch processing + * of marker creation requests. + * + * @param context Javalin app context + * @param markerDir marker directory path + * @param markerName marker name + * @return the {@code CompletableFuture} instance for the request + */ + public CompletableFuture<String> createMarker(Context context, String markerDir, String markerName) { + LOG.info("Request: create marker " + markerDir + " " + markerName); + MarkerCreationCompletableFuture future = new MarkerCreationCompletableFuture(context, markerDir, markerName); + // Add the future to the list + MarkerDirState markerDirState = getMarkerDirState(markerDir); + markerDirState.addMarkerCreationFuture(future); + if (firstCreationRequestSeen.getAndSet(true)) { Review comment: there should be a "!" in the condition. ########## File path: hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java ########## @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.handlers.marker; + +import org.apache.hudi.common.config.SerializableConfiguration; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.collection.ImmutablePair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.StringUtils; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.hudi.common.util.FileIOUtils.closeQuietly; +import static org.apache.hudi.timeline.service.RequestHandler.jsonifyResult; + +/** + * Stores the state of a marker directory. + * + * The operations inside this class is designed to be thread-safe. + */ +public class MarkerDirState implements Serializable { + public static final String MARKERS_FILENAME_PREFIX = "MARKERS"; + private static final Logger LOG = LogManager.getLogger(MarkerDirState.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + // Marker directory + private final String markerDirPath; + private final FileSystem fileSystem; + private final Registry metricsRegistry; + // A cached copy of all markers in memory + private final Set<String> allMarkers = new HashSet<>(); + // A cached copy of marker entries in each marker file, stored in StringBuilder + // for efficient appending + // Mapping: {markerFileIndex -> markers} + private final Map<Integer, StringBuilder> fileMarkersMap = new HashMap<>(); + // A list of use status of underlying files storing markers by a thread. + // {@code true} means the file is in use by a {@code BatchCreateMarkerRunnable}. + // Index of the list is used for the filename, i.e., "1" -> "MARKERS1" + private final List<Boolean> threadUseStatus; + // A list of pending futures from async marker creation requests + private final List<MarkerCreationCompletableFuture> markerCreationFutures = new ArrayList<>(); + private final int parallelism; + private final Object lazyInitLock = new Object(); + private final Object markerCreationProcessingLock = new Object(); + private transient HoodieEngineContext hoodieEngineContext; + // Last underlying file index used, for finding the next file index + // in a round-robin fashion + private int lastFileIndex = 0; Review comment: lastFileIndexUsed ########## File path: hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java ########## @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.handlers; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.model.IOType; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.timeline.service.TimelineService; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationCompletableFuture; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationDispatchingRunnable; +import org.apache.hudi.timeline.service.handlers.marker.MarkerDirState; + +import io.javalin.Context; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +/** + * REST Handler servicing marker requests. + * + * The marker creation requests are handled asynchronous, while other types of requests + * are handled synchronous. + * + * Marker creation requests are batch processed periodically by a thread. Each batch + * processing thread adds new markers to a marker file. Given that marker file operation + * can take time, multiple concurrent threads can run at the same, while they operate + * on different marker files storing mutually exclusive marker entries. At any given + * time, a marker file is touched by at most one thread to guarantee consistency. + * Below is an example of running batch processing threads. + * + * |-----| batch interval + * Thread 1 |-------------------------->| writing to MARKERS1 + * Thread 2 |-------------------------->| writing to MARKERS2 + * Thread 3 |-------------------------->| writing to MARKERS3 + */ +public class MarkerHandler extends Handler { + private static final Logger LOG = LogManager.getLogger(MarkerHandler.class); + + private final Registry metricsRegistry; + // a scheduled executor service to schedule dispatching of marker creation requests + private final ScheduledExecutorService dispatchingExecutorService; + // an executor service to schedule the worker threads of batch processing marker creation requests + private final ExecutorService batchingExecutorService; + // Parallelism for reading and deleting marker files + private final int parallelism; + // Marker directory states, {markerDirPath -> MarkerDirState instance} + private final Map<String, MarkerDirState> markerDirStateMap = new HashMap<>(); + // A long-running thread to dispatch marker creation requests to batch processing threads Review comment: fix comments. its not a long running thread anymore ########## File path: hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java ########## @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.timeline.service.handlers; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.model.IOType; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.timeline.service.TimelineService; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationCompletableFuture; +import org.apache.hudi.timeline.service.handlers.marker.MarkerCreationDispatchingRunnable; +import org.apache.hudi.timeline.service.handlers.marker.MarkerDirState; + +import io.javalin.Context; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +/** + * REST Handler servicing marker requests. + * + * The marker creation requests are handled asynchronous, while other types of requests + * are handled synchronous. + * + * Marker creation requests are batch processed periodically by a thread. Each batch + * processing thread adds new markers to a marker file. Given that marker file operation + * can take time, multiple concurrent threads can run at the same, while they operate + * on different marker files storing mutually exclusive marker entries. At any given + * time, a marker file is touched by at most one thread to guarantee consistency. + * Below is an example of running batch processing threads. + * + * |-----| batch interval + * Thread 1 |-------------------------->| writing to MARKERS1 + * Thread 2 |-------------------------->| writing to MARKERS2 + * Thread 3 |-------------------------->| writing to MARKERS3 + */ +public class MarkerHandler extends Handler { + private static final Logger LOG = LogManager.getLogger(MarkerHandler.class); + + private final Registry metricsRegistry; + // a scheduled executor service to schedule dispatching of marker creation requests + private final ScheduledExecutorService dispatchingExecutorService; + // an executor service to schedule the worker threads of batch processing marker creation requests + private final ExecutorService batchingExecutorService; + // Parallelism for reading and deleting marker files + private final int parallelism; + // Marker directory states, {markerDirPath -> MarkerDirState instance} + private final Map<String, MarkerDirState> markerDirStateMap = new HashMap<>(); + // A long-running thread to dispatch marker creation requests to batch processing threads + private final MarkerCreationDispatchingRunnable markerCreationDispatchingRunnable; + private final AtomicBoolean firstCreationRequestSeen; + private transient HoodieEngineContext hoodieEngineContext; + private ScheduledFuture<?> dispatchingScheduledFuture; + + public MarkerHandler(Configuration conf, TimelineService.Config timelineServiceConfig, + HoodieEngineContext hoodieEngineContext, FileSystem fileSystem, + FileSystemViewManager viewManager, Registry metricsRegistry) throws IOException { + super(conf, timelineServiceConfig, fileSystem, viewManager); + LOG.debug("MarkerHandler FileSystem: " + this.fileSystem.getScheme()); + LOG.debug("MarkerHandler batching params: batchNumThreads=" + timelineServiceConfig.markerBatchNumThreads + + " batchIntervalMs=" + timelineServiceConfig.markerBatchIntervalMs + "ms"); + this.hoodieEngineContext = hoodieEngineContext; + this.metricsRegistry = metricsRegistry; + this.parallelism = timelineServiceConfig.markerParallelism; + this.dispatchingExecutorService = Executors.newSingleThreadScheduledExecutor(); + this.batchingExecutorService = Executors.newFixedThreadPool(timelineServiceConfig.markerBatchNumThreads); + this.markerCreationDispatchingRunnable = + new MarkerCreationDispatchingRunnable(markerDirStateMap, batchingExecutorService); + this.firstCreationRequestSeen = new AtomicBoolean(false); + } + + /** + * Stops the dispatching of marker creation requests. + */ + public void stop() { + if (dispatchingScheduledFuture != null) { + dispatchingScheduledFuture.cancel(true); + } + dispatchingExecutorService.shutdown(); + batchingExecutorService.shutdown(); + } + + /** + * @param markerDir marker directory path + * @return all marker paths in the marker directory + */ + public Set<String> getAllMarkers(String markerDir) { + MarkerDirState markerDirState = getMarkerDirState(markerDir); + return markerDirState.getAllMarkers(); + } + + /** + * @param markerDir marker directory path + * @return all marker paths of write IO type "CREATE" and "MERGE" + */ + public Set<String> getCreateAndMergeMarkers(String markerDir) { + return getAllMarkers(markerDir).stream() + .filter(markerName -> !markerName.endsWith(IOType.APPEND.name())) + .collect(Collectors.toSet()); + } + + /** + * @param markerDir marker directory path + * @return {@code true} if the marker directory exists; {@code false} otherwise. + */ + public boolean doesMarkerDirExist(String markerDir) { + MarkerDirState markerDirState = getMarkerDirState(markerDir); + return markerDirState.exists(); + } + + /** + * Generates a future for an async marker creation request + * + * The future is added to the marker creation future list and waits for the next batch processing + * of marker creation requests. + * + * @param context Javalin app context + * @param markerDir marker directory path + * @param markerName marker name + * @return the {@code CompletableFuture} instance for the request + */ + public CompletableFuture<String> createMarker(Context context, String markerDir, String markerName) { + LOG.info("Request: create marker " + markerDir + " " + markerName); + MarkerCreationCompletableFuture future = new MarkerCreationCompletableFuture(context, markerDir, markerName); + // Add the future to the list + MarkerDirState markerDirState = getMarkerDirState(markerDir); + markerDirState.addMarkerCreationFuture(future); + if (firstCreationRequestSeen.getAndSet(true)) { + dispatchingScheduledFuture = dispatchingExecutorService.scheduleAtFixedRate(markerCreationDispatchingRunnable, + timelineServiceConfig.markerBatchIntervalMs, timelineServiceConfig.markerBatchIntervalMs, + TimeUnit.MILLISECONDS); + } + return future; + } + + /** + * Deletes markers in the directory. + * + * @param markerDir marker directory path + * @return {@code true} if successful; {@code false} otherwise. + */ + public Boolean deleteMarkers(String markerDir) { + boolean result = getMarkerDirState(markerDir).deleteAllMarkers(); + markerDirStateMap.remove(markerDir); + return result; + } + + private MarkerDirState getMarkerDirState(String markerDir) { + MarkerDirState markerDirState = markerDirStateMap.get(markerDir); + if (markerDirState == null) { + synchronized (markerDirStateMap) { + markerDirState = new MarkerDirState(markerDir, timelineServiceConfig.markerBatchNumThreads, Review comment: we should have another if (markerDirState == null) within the sync block. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
