nbalajee commented on a change in pull request #2607: URL: https://github.com/apache/hudi/pull/2607#discussion_r597098674
########## File path: hudi-common/src/main/java/org/apache/hudi/common/model/HoodieObservabilityStat.java ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.model; + +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.util.Option; + +import java.io.Serializable; + +/** + * Observability related metrics collection operations. + */ +public class HoodieObservabilityStat implements Serializable { + public static final String OBSERVABILITY_REGISTRY_NAME = "Observability"; + + // define a unique metric name string for each metric to be collected. + public static final String PARQUET_NORMALIZED_WRITE_TIME = "writeTimePerRecordInUSec"; + public static final String PARQUET_CUMULATIVE_WRITE_TIME = "cumulativeParquetWriteTimeInUSec"; + public static final String PARQUET_WRITE_TIME_PER_MB_IN_USEC = "writeTimePerMBInUSec"; + public static final String PARQUET_WRITE_THROUGHPUT_MBPS = "writeThroughputMBps"; + public static final String TOTAL_RECORDS_WRITTEN = "totalRecordsWritten"; + + public enum WriteType { + INSERT, + UPSERT, + UPDATE, + } + + public static long ONE_MB = 1024 * 1024; + public static long USEC_PER_SEC = 1000 * 1000; + Option<Registry> observabilityRegistry; + String tableName; + WriteType writeType; + String hostName; + Long stageId; + Long partitionId; + + public HoodieObservabilityStat(Option<Registry> registry, String tableName, WriteType type, String host, + long stageId, long partitionId) { + this.observabilityRegistry = registry; + this.tableName = tableName; + this.writeType = type; + this.hostName = host; + this.stageId = stageId; + this.partitionId = partitionId; + } + + private String getWriteMetricWithTypeHostNameAndPartitionId(String tableName, String metric, String type, + String host, long partitionId) { + return String.format("%s.%s.%s.%s.%d", tableName, metric, type, host, partitionId); + } + + private String getConsolidatedWriteMetricKey(String tableName, String metric, String type) { + return String.format("%s.consolidated.%s.%s", tableName, metric, type); + } + + public void recordWriteStats(long totalRecs, long cumulativeWriteTimeInMsec, long fileSizeInBytes) { Review comment: Looked into this. Yes, by overriding the add/merge functions in HoodieObservabilityMetrics we can merge the executor level stats into consolidated stats. However, doing it this way provides flexibility and more control over granularity. For example, aggregating over <tablename>.consolidated.<metrics>.INSERT or UPSERT operations. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
