showuon commented on code in PR #13837: URL: https://github.com/apache/kafka/pull/13837#discussion_r1250396683
########## storage/src/test/java/org/apache/kafka/server/log/remote/storage/LocalTieredStorageListener.java: ########## @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static java.util.Objects.requireNonNull; + +/** + * Used to subscribe to the {@link LocalTieredStorageEvent} generated by a {@link LocalTieredStorage}. + * + * Such events model the interaction between a broker and the second-tier storage system, here set to + * be the {@link LocalTieredStorage}. + * + * Unlike {@link LocalTieredStorageTraverser}, the intent is to generate instant notification, + * rather than walking the directory structure of the storage at a given point in time. + */ +public interface LocalTieredStorageListener { + + /** + * Called by the {@link LocalTieredStorage} subscribed to when the given {@link LocalTieredStorageEvent} + * is generated. + * + * @param event The event which characterizes the interaction between Kafka and the second-tier storage + * system. It includes the nature of the interaction, e.g. when Kafka initiates the upload + * of a log segment or fetches the offset index of a segment stored in the second-tier + * storage. + */ + void onStorageEvent(LocalTieredStorageEvent event); + + /** + * Delegates to a list of listeners in insertion order. + * Failures (escaped non-error exceptions) of one listener does not prevent execution of the next ones in the list. + */ + final class LocalTieredStorageListeners implements LocalTieredStorageListener { + private static final Logger LOGGER = LoggerFactory.getLogger(LocalTieredStorageListener.class); + private final List<LocalTieredStorageListener> listeners = new CopyOnWriteArrayList<>(); + + public void add(final LocalTieredStorageListener listener) { + listeners.add(requireNonNull(listener)); + } + + @Override + public void onStorageEvent(final LocalTieredStorageEvent event) { + for (final LocalTieredStorageListener listener: listeners) { + try { + listener.onStorageEvent(event); + Review Comment: nit: extra line ########## storage/src/test/java/org/apache/kafka/server/log/remote/storage/LocalTieredStorageHistory.java: ########## @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.server.log.remote.storage.LocalTieredStorageEvent.EventType; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; +import static java.util.Collections.unmodifiableMap; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.slf4j.LoggerFactory.getLogger; + +/** + * Accumulates and retains the interactions between brokers and {@link LocalTieredStorage} instances. + * These interactions are modelled via events of type {@link LocalTieredStorageEvent}. + * + * Events from an instance of storage are captured by the {@link LocalTieredStorageHistory} after + * {@link LocalTieredStorageHistory#listenTo(LocalTieredStorage)} is called. + */ +/* @ThreadSafe */ +public final class LocalTieredStorageHistory { + private static final int HARD_EVENT_COUNT_LIMIT = 1_000_000; + + private static final Logger LOGGER = getLogger(LocalTieredStorageHistory.class); + + private final Map<EventType, List<LocalTieredStorageEvent>> history; + + LocalTieredStorageHistory() { + this.history = unmodifiableMap(stream(EventType.values()).collect(toMap(identity(), t -> new ArrayList<>()))); + } + + /** + * Returns the list of events accumulated by this instance of history from the {@link LocalTieredStorage} + * it captures events from. + * + * @param type The type of the events to retrieve ((e.g. offload or fetch a segment, fetch a time index, etc.) Review Comment: nit: double `((` ########## storage/src/test/java/org/apache/kafka/server/log/remote/storage/LocalTieredStorageHistory.java: ########## @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.server.log.remote.storage.LocalTieredStorageEvent.EventType; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; +import static java.util.Collections.unmodifiableMap; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.slf4j.LoggerFactory.getLogger; + +/** + * Accumulates and retains the interactions between brokers and {@link LocalTieredStorage} instances. + * These interactions are modelled via events of type {@link LocalTieredStorageEvent}. + * + * Events from an instance of storage are captured by the {@link LocalTieredStorageHistory} after + * {@link LocalTieredStorageHistory#listenTo(LocalTieredStorage)} is called. + */ +/* @ThreadSafe */ +public final class LocalTieredStorageHistory { + private static final int HARD_EVENT_COUNT_LIMIT = 1_000_000; + + private static final Logger LOGGER = getLogger(LocalTieredStorageHistory.class); + + private final Map<EventType, List<LocalTieredStorageEvent>> history; + + LocalTieredStorageHistory() { + this.history = unmodifiableMap(stream(EventType.values()).collect(toMap(identity(), t -> new ArrayList<>()))); + } + + /** + * Returns the list of events accumulated by this instance of history from the {@link LocalTieredStorage} + * it captures events from. + * + * @param type The type of the events to retrieve ((e.g. offload or fetch a segment, fetch a time index, etc.) + * @param topicPartition The topic-partition which the events relate to. + * @return The list of events accumulated in this instance. + */ + public List<LocalTieredStorageEvent> getEvents(final EventType type, final TopicPartition topicPartition) { + List<LocalTieredStorageEvent> matchingTypeEvents = history.get(type); + + synchronized (matchingTypeEvents) { + matchingTypeEvents = new ArrayList<>(matchingTypeEvents); + } + + return matchingTypeEvents.stream().filter(matches(topicPartition)).collect(Collectors.toList()); + } + + /** + * Returns the latest event captured so far of the given type and relating to the given topic-partition. + * + * @param type The type of the events to retrieve ((e.g. offload or fetch a segment, fetch a time index, etc.) Review Comment: nit: double `((` ########## storage/src/test/java/org/apache/kafka/server/log/remote/storage/LocalTieredStorageListener.java: ########## @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static java.util.Objects.requireNonNull; + +/** + * Used to subscribe to the {@link LocalTieredStorageEvent} generated by a {@link LocalTieredStorage}. + * + * Such events model the interaction between a broker and the second-tier storage system, here set to + * be the {@link LocalTieredStorage}. + * + * Unlike {@link LocalTieredStorageTraverser}, the intent is to generate instant notification, + * rather than walking the directory structure of the storage at a given point in time. + */ +public interface LocalTieredStorageListener { + + /** + * Called by the {@link LocalTieredStorage} subscribed to when the given {@link LocalTieredStorageEvent} + * is generated. + * + * @param event The event which characterizes the interaction between Kafka and the second-tier storage + * system. It includes the nature of the interaction, e.g. when Kafka initiates the upload + * of a log segment or fetches the offset index of a segment stored in the second-tier + * storage. + */ + void onStorageEvent(LocalTieredStorageEvent event); + + /** + * Delegates to a list of listeners in insertion order. + * Failures (escaped non-error exceptions) of one listener does not prevent execution of the next ones in the list. + */ + final class LocalTieredStorageListeners implements LocalTieredStorageListener { + private static final Logger LOGGER = LoggerFactory.getLogger(LocalTieredStorageListener.class); + private final List<LocalTieredStorageListener> listeners = new CopyOnWriteArrayList<>(); + + public void add(final LocalTieredStorageListener listener) { + listeners.add(requireNonNull(listener)); + } + + @Override + public void onStorageEvent(final LocalTieredStorageEvent event) { + for (final LocalTieredStorageListener listener: listeners) { + try { + listener.onStorageEvent(event); + + } catch (Exception e) { + LOGGER.error("Caught failure from listener", e); Review Comment: Should we log which listener it is? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
