leesf commented on a change in pull request #1239: [HUDI-551] Abstract a test case class for DFS Source to make it extensible URL: https://github.com/apache/incubator-hudi/pull/1239#discussion_r368264818
########## File path: hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/AbstractDFSSourceTestBase.java ########## @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.sources; + +import org.apache.hudi.AvroConversionUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.utilities.UtilitiesTestBase; +import org.apache.hudi.utilities.deltastreamer.SourceFormatAdapter; +import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * An abstract test base for {@link Source} using DFS as the file system. + */ +public abstract class AbstractDFSSourceTestBase extends UtilitiesTestBase { + + FilebasedSchemaProvider schemaProvider; + String dfsRoot; + String fileSuffix; + int fileCount = 1; + HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator(); + + @BeforeClass + public static void initClass() throws Exception { + UtilitiesTestBase.initClass(); + } + + @AfterClass + public static void cleanupClass() throws Exception { + UtilitiesTestBase.cleanupClass(); + } + + @Before + public void setup() throws Exception { + super.setup(); + schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS(), jsc); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + /** + * Prepares the specific {@link Source} to test, by passing in necessary configurations. + * + * @return A {@link Source} using DFS as the file system. + */ + abstract Source prepareDFSSource(); + + /** + * Writes test data, i.e., a {@link List} of {@link HoodieRecord}, to a file on DFS. + * + * @param records Test data. + * @param path The path in {@link Path} of the file to write. + * @throws IOException + */ + abstract void writeNewDataToFile(List<HoodieRecord> records, Path path) throws IOException; + + /** + * Generates a batch of test data and writes the data to a file. This can be called multiple times to generate multiple files. + * + * @return The {@link Path} of the file. + * @throws IOException + */ + Path generateOneFile() throws IOException { + Path path = new Path(dfsRoot, fileCount + fileSuffix); + switch (fileCount) { + case 1: + writeNewDataToFile(dataGenerator.generateInserts("000", 100), path); + fileCount++; + return path; + case 2: + writeNewDataToFile(dataGenerator.generateInserts("001", 10000), path); + fileCount++; + return path; + default: + return null; + } + } Review comment: Would not get the point of this method, a bit of tricky. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
