n3nash commented on a change in pull request #991: Hudi Test Suite (Refactor) URL: https://github.com/apache/incubator-hudi/pull/991#discussion_r344360967
########## File path: hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DeltaConfig.java ########## @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.bench.configuration; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.bench.DeltaInputFormat; +import org.apache.hudi.bench.DeltaSinkType; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.util.collection.Pair; + +/** + * Configuration to hold the delta sink type and delta format + */ +public class DeltaConfig implements Serializable { + + private final DeltaSinkType deltaSinkType; + private final DeltaInputFormat deltaInputFormat; + private final SerializableConfiguration configuration; + + public DeltaConfig(DeltaSinkType deltaSinkType, DeltaInputFormat deltaInputFormat, + SerializableConfiguration configuration) { + this.deltaSinkType = deltaSinkType; + this.deltaInputFormat = deltaInputFormat; + this.configuration = configuration; + } + + public DeltaSinkType getDeltaSinkType() { + return deltaSinkType; + } + + public DeltaInputFormat getDeltaInputFormat() { + return deltaInputFormat; + } + + public Configuration getConfiguration() { + return configuration.get(); + } + + /** + * Represents any kind of workload operation for new data. Each workload also contains a set of Option sequence of + * actions that can be executed in parallel. + */ + public static class Config { + + public static final String CONFIG_NAME = "config"; + public static final String TYPE = "type"; + public static final String NODE_NAME = "name"; + public static final String DEPENDENCIES = "deps"; + public static final String CHILDREN = "children"; + public static final String HIVE_QUERIES = "hive_queries"; + private static String NUM_RECORDS_INSERT = "num_records_insert"; + private static String NUM_RECORDS_UPSERT = "num_records_upsert"; + private static String REPEAT_COUNT = "repeat_count"; + private static String RECORD_SIZE = "record_size"; + private static String NUM_PARTITIONS_INSERT = "num_partitions_insert"; + private static String NUM_PARTITIONS_UPSERT = "num_partitions_upsert"; + private static String NUM_FILES_UPSERT = "num_files_upsert"; + private static String FRACTION_UPSERT_PER_FILE = "fraction_upsert_per_file"; + private static String DISABLE_GENERATE = "disable_generate"; + private static String DISABLE_INGEST = "disable_ingest"; + private static String HIVE_LOCAL = "hive_local"; + private static String HIVE_QUEUE_NAME = "queue_name"; + private static String HIVE_EXECUTION_ENGINE = "query_engine"; + + private Map<String, Object> configsMap; + + @VisibleForTesting + public Config(Map<String, Object> configsMap) { + this.configsMap = configsMap; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public long getNumRecordsInsert() { + return Long.valueOf(configsMap.getOrDefault(NUM_RECORDS_INSERT, 0).toString()); + } + + public long getNumRecordsUpsert() { + return Long.valueOf(configsMap.getOrDefault(NUM_RECORDS_UPSERT, 0).toString()); + } + + public int getRecordSize() { + return Integer.valueOf(configsMap.getOrDefault(RECORD_SIZE, 1024).toString()); + } + + public int getNumInsertPartitions() { + return Integer.valueOf(configsMap.getOrDefault(NUM_PARTITIONS_INSERT, 1).toString()); + } + + public int getRepeatCount() { + return Integer.valueOf(configsMap.getOrDefault(REPEAT_COUNT, 1).toString()); + } + + public int getNumUpsertPartitions() { + return Integer.valueOf(configsMap.getOrDefault(NUM_PARTITIONS_UPSERT, 0).toString()); + } + + public int getNumUpsertFiles() { + return Integer.valueOf(configsMap.getOrDefault(NUM_FILES_UPSERT, 1).toString()); + } + + public double getFractionUpsertPerFile() { + return Double.valueOf(configsMap.getOrDefault(FRACTION_UPSERT_PER_FILE, 0.0).toString()); + } + + public boolean isDisableGenerate() { + return Boolean.valueOf(configsMap.getOrDefault(DISABLE_GENERATE, false).toString()); + } + + public boolean isDisableIngest() { + return Boolean.valueOf(configsMap.getOrDefault(DISABLE_INGEST, false).toString()); + } + + public Map<String, Object> getOtherConfigs() { + if (configsMap == null) { + return new HashMap<>(); + } + return configsMap; + } + + public List<Pair<String, Integer>> getHiveQueries() { + try { + System.out.println(this.configsMap); Review comment: same here ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
