This is an automated email from the ASF dual-hosted git repository.
xbli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f4a4bf389a Custom configuration property reader for segment metadata
files (#12440)
f4a4bf389a is described below
commit f4a4bf389a5a81d116e38ea73280277a23080b43
Author: Abhishek Sharma <[email protected]>
AuthorDate: Wed Jun 5 17:30:56 2024 -0400
Custom configuration property reader for segment metadata files (#12440)
* Added versioned configuration reader/writer
* Added methods to read/write segment metadata with versioned reader/writer
---
.../pinot/spi/env/CommonsConfigurationUtils.java | 179 +++++++++++++----
...ctory.java => ConfigFilePropertyIOFactory.java} | 2 +-
.../apache/pinot/spi/env/PinotConfiguration.java | 6 +-
...aderFactory.java => PropertyIOFactoryKind.java} | 40 +++-
...yReaderFactory.java => VersionedIOFactory.java} | 19 +-
.../pinot/spi/env/VersionedPropertyReader.java | 61 ++++++
...erFactory.java => VersionedPropertyWriter.java} | 23 ++-
.../spi/env/CommonsConfigurationUtilsTest.java | 48 ++++-
.../pinot/spi/env/PinotConfigurationTest.java | 4 +-
.../pinot/spi/env/VersionedPropertyConfigTest.java | 223 +++++++++++++++++++++
...segment-metadata-with-version-header.properties | 125 ++++++++++++
...ment-metadata-without-version-header.properties | 123 ++++++++++++
pom.xml | 6 +
13 files changed, 790 insertions(+), 69 deletions(-)
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java
index 0613230e09..10c9f7151d 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/CommonsConfigurationUtils.java
@@ -18,8 +18,10 @@
*/
package org.apache.pinot.spi.env;
-import com.google.common.base.Preconditions;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Iterator;
@@ -42,29 +44,21 @@ import org.apache.commons.lang3.StringUtils;
* Provide utility functions to manipulate Apache Commons {@link
Configuration} instances.
*/
public class CommonsConfigurationUtils {
+ // value to be used to set the global separator for versioned properties
configuration.
+ // the value is same of PropertiesConfiguration `DEFAULT_SEPARATOR` constant.
+ public static final String VERSIONED_CONFIG_SEPARATOR = " = ";
private static final Character DEFAULT_LIST_DELIMITER = ',';
+ public static final String VERSION_HEADER_IDENTIFIER = "version";
- private CommonsConfigurationUtils() {
- }
+ // usage: default header version of all configurations.
+ // if properties configuration doesn't contain header version, it will be
considered as 1
+ public static final String DEFAULT_PROPERTIES_CONFIGURATION_HEADER_VERSION =
"1";
- /**
- * Instantiate a {@link PropertiesConfiguration} from a {@link File}.
- * @param file containing properties
- * @return a {@link PropertiesConfiguration} instance. Empty if file does
not exist.
- */
- public static PropertiesConfiguration fromFile(File file)
- throws ConfigurationException {
- return fromFile(file, false, true);
- }
+ // usage: used in reading segment metadata or other properties
configurations with 'VersionedIOFactory' IO Factory.
+ // version signifies that segment metadata or other properties
configurations contains keys with no special character.
+ public static final String PROPERTIES_CONFIGURATION_HEADER_VERSION_2 = "2";
- /**
- * Instantiate a {@link PropertiesConfiguration} from an {@link InputStream}.
- * @param stream containing properties
- * @return a {@link PropertiesConfiguration} instance.
- */
- public static PropertiesConfiguration fromInputStream(InputStream stream)
- throws ConfigurationException {
- return fromInputStream(stream, false, true);
+ private CommonsConfigurationUtils() {
}
/**
@@ -74,20 +68,20 @@ public class CommonsConfigurationUtils {
*/
public static PropertiesConfiguration fromPath(String path)
throws ConfigurationException {
- return fromPath(path, false, true);
+ return fromPath(path, true, null);
}
/**
* Instantiate a {@link PropertiesConfiguration} from an {@link String}.
* @param path representing the path of file
- * @param setIOFactory representing to set the IOFactory or not
* @param setDefaultDelimiter representing to set the default list delimiter.
+ * @param ioFactoryKind representing to set IOFactory. It can be null.
* @return a {@link PropertiesConfiguration} instance.
*/
- public static PropertiesConfiguration fromPath(@Nullable String path,
boolean setIOFactory,
- boolean setDefaultDelimiter)
+ public static PropertiesConfiguration fromPath(@Nullable String path,
boolean setDefaultDelimiter,
+ @Nullable PropertyIOFactoryKind ioFactoryKind)
throws ConfigurationException {
- PropertiesConfiguration config =
createPropertiesConfiguration(setIOFactory, setDefaultDelimiter);
+ PropertiesConfiguration config =
createPropertiesConfiguration(setDefaultDelimiter, ioFactoryKind);
// if provided path is non-empty, load the existing properties from
provided file path
if (StringUtils.isNotEmpty(path)) {
FileHandler fileHandler = new FileHandler(config);
@@ -99,14 +93,24 @@ public class CommonsConfigurationUtils {
/**
* Instantiate a {@link PropertiesConfiguration} from an {@link InputStream}.
* @param stream containing properties
- * @param setIOFactory representing to set the IOFactory or not
+ * @return a {@link PropertiesConfiguration} instance.
+ */
+ public static PropertiesConfiguration fromInputStream(InputStream stream)
+ throws ConfigurationException {
+ return fromInputStream(stream, true,
PropertyIOFactoryKind.DefaultIOFactory);
+ }
+
+ /**
+ * Instantiate a {@link PropertiesConfiguration} from an {@link InputStream}.
+ * @param stream containing properties
* @param setDefaultDelimiter representing to set the default list delimiter.
+ * @param ioFactoryKind representing to set IOFactory. It can be null.
* @return a {@link PropertiesConfiguration} instance.
*/
- public static PropertiesConfiguration fromInputStream(@Nullable InputStream
stream, boolean setIOFactory,
- boolean setDefaultDelimiter)
+ public static PropertiesConfiguration fromInputStream(@Nullable InputStream
stream,
+ boolean setDefaultDelimiter, @Nullable PropertyIOFactoryKind
ioFactoryKind)
throws ConfigurationException {
- PropertiesConfiguration config =
createPropertiesConfiguration(setIOFactory, setDefaultDelimiter);
+ PropertiesConfiguration config =
createPropertiesConfiguration(setDefaultDelimiter, ioFactoryKind);
// if provided stream is not null, load the existing properties from
provided input stream.
if (stream != null) {
FileHandler fileHandler = new FileHandler(config);
@@ -115,16 +119,45 @@ public class CommonsConfigurationUtils {
return config;
}
+ /**
+ * Instantiate a Segment Metadata {@link PropertiesConfiguration} from a
{@link File}.
+ * @param file containing properties
+ * @param setDefaultDelimiter representing to set the default list delimiter.
+ * @return a {@link PropertiesConfiguration} instance.
+ */
+ public static PropertiesConfiguration getSegmentMetadataFromFile(File file,
boolean setDefaultDelimiter)
+ throws ConfigurationException {
+ PropertyIOFactoryKind ioFactoryKind =
PropertyIOFactoryKind.DefaultIOFactory;
+
+ // if segment metadata contains version header with value '2', set
VersionedIOFactory as IO factory.
+ if
(PROPERTIES_CONFIGURATION_HEADER_VERSION_2.equals(getConfigurationHeaderVersion(file)))
{
+ ioFactoryKind = PropertyIOFactoryKind.VersionedIOFactory;
+ }
+
+ return fromFile(file, setDefaultDelimiter, ioFactoryKind);
+ }
+
+ /**
+ * Instantiate a {@link PropertiesConfiguration} from a {@link File}.
+ * @param file containing properties
+ * @return a {@link PropertiesConfiguration} instance. Empty if file does
not exist.
+ */
+ public static PropertiesConfiguration fromFile(File file)
+ throws ConfigurationException {
+ return fromFile(file, true, PropertyIOFactoryKind.DefaultIOFactory);
+ }
+
/**
* Instantiate a {@link PropertiesConfiguration} from a {@link File}.
* @param file containing properties
- * @param setIOFactory representing to set the IOFactory or not
* @param setDefaultDelimiter representing to set the default list delimiter.
+ * @param ioFactoryKind representing to set IOFactory. It can be null.
* @return a {@link PropertiesConfiguration} instance.
*/
- public static PropertiesConfiguration fromFile(@Nullable File file, boolean
setIOFactory, boolean setDefaultDelimiter)
+ public static PropertiesConfiguration fromFile(@Nullable File file,
+ boolean setDefaultDelimiter, @Nullable PropertyIOFactoryKind
ioFactoryKind)
throws ConfigurationException {
- PropertiesConfiguration config =
createPropertiesConfiguration(setIOFactory, setDefaultDelimiter);
+ PropertiesConfiguration config =
createPropertiesConfiguration(setDefaultDelimiter, ioFactoryKind);
// check if file exists, load the existing properties.
if (file != null && file.exists()) {
FileHandler fileHandler = new FileHandler(config);
@@ -133,13 +166,37 @@ public class CommonsConfigurationUtils {
return config;
}
+ /**
+ * save the segment metadata configuration content into the provided file
based on the version header.
+ * @param propertiesConfiguration a {@link PropertiesConfiguration} instance.
+ * @param file a {@link File} instance.
+ * @param versionHeader a Nullable {@link String} instance.
+ */
+ public static void saveSegmentMetadataToFile(PropertiesConfiguration
propertiesConfiguration, File file,
+ @Nullable String versionHeader) {
+ if (StringUtils.isNotEmpty(versionHeader)) {
+ String header = getVersionHeaderString(versionHeader);
+ propertiesConfiguration.setHeader(header);
+
+ // checks whether the provided versionHeader equals to
VersionedIOFactory kind.
+ // if true, set IO factory as VersionedIOFactory
+ if (PROPERTIES_CONFIGURATION_HEADER_VERSION_2.equals(versionHeader)) {
+ // set versioned IOFactory
+
propertiesConfiguration.setIOFactory(PropertyIOFactoryKind.VersionedIOFactory.getInstance());
+ // setting the global separator makes sure the configurations gets
written with ' = ' separator.
+ // global separator overrides the separator set at the key level as
well.
+
propertiesConfiguration.getLayout().setGlobalSeparator(VERSIONED_CONFIG_SEPARATOR);
+ }
+ }
+ saveToFile(propertiesConfiguration, file);
+ }
+
/**
* Save the propertiesConfiguration content into the provided file.
* @param propertiesConfiguration a {@link PropertiesConfiguration} instance.
* @param file a {@link File} instance.
*/
public static void saveToFile(PropertiesConfiguration
propertiesConfiguration, File file) {
- Preconditions.checkNotNull(file, "File object can not be null for saving
configurations");
FileHandler fileHandler = new FileHandler(propertiesConfiguration);
fileHandler.setFile(file);
try {
@@ -276,20 +333,64 @@ public class CommonsConfigurationUtils {
return value.replace("\0\0", ",");
}
- private static PropertiesConfiguration createPropertiesConfiguration(boolean
setIOFactory,
- boolean setDefaultDelimiter) {
+ /**
+ * creates the instance of the {@link
org.apache.commons.configuration2.PropertiesConfiguration}
+ * with custom IO factory based on kind {@link
org.apache.commons.configuration2.PropertiesConfiguration.IOFactory}
+ * and legacy list delimiter {@link
org.apache.commons.configuration2.convert.LegacyListDelimiterHandler}
+ *
+ * @param setDefaultDelimiter sets the default list delimiter.
+ * @param ioFactoryKind IOFactory kind, can be null.
+ * @return PropertiesConfiguration
+ */
+ private static PropertiesConfiguration createPropertiesConfiguration(boolean
setDefaultDelimiter,
+ @Nullable PropertyIOFactoryKind ioFactoryKind) {
PropertiesConfiguration config = new PropertiesConfiguration();
- // setting IO Reader Factory
- if (setIOFactory) {
- config.setIOFactory(new ConfigFilePropertyReaderFactory());
+ // setting IO Reader Factory of the configuration.
+ if (ioFactoryKind != null) {
+ config.setIOFactory(ioFactoryKind.getInstance());
}
- // setting DEFAULT_LIST_DELIMITER
+ // setting the DEFAULT_LIST_DELIMITER
if (setDefaultDelimiter) {
config.setListDelimiterHandler(new
LegacyListDelimiterHandler(DEFAULT_LIST_DELIMITER));
}
return config;
}
+
+ /**
+ * checks whether the configuration file first line is version header or not.
+ * @param file configuration file
+ * @return String
+ * @throws ConfigurationException exception.
+ */
+ private static String getConfigurationHeaderVersion(File file)
+ throws ConfigurationException {
+ String versionValue = DEFAULT_PROPERTIES_CONFIGURATION_HEADER_VERSION;
+ if (file.exists()) {
+ try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
+ String fileFirstLine = reader.readLine();
+ // header version is written as a comment and start with '# '
+ String versionHeaderCommentPrefix = String.format("# %s",
VERSION_HEADER_IDENTIFIER);
+ // check whether the file has the version header or not
+ if (StringUtils.startsWith(fileFirstLine, versionHeaderCommentPrefix))
{
+ String[] headerKeyValue =
StringUtils.splitByWholeSeparator(fileFirstLine, VERSIONED_CONFIG_SEPARATOR, 2);
+ if (headerKeyValue.length == 2) {
+ versionValue = headerKeyValue[1];
+ }
+ }
+ } catch (IOException exception) {
+ throw new ConfigurationException(
+ "Error occurred while reading configuration file " +
file.getName(), exception);
+ }
+ }
+ return versionValue;
+ }
+
+ // Returns the version header string based on the version header value
provided.
+ // The return statement follow the pattern 'version = <value>'
+ static String getVersionHeaderString(String versionHeaderValue) {
+ return String.format("%s%s%s", VERSION_HEADER_IDENTIFIER,
VERSIONED_CONFIG_SEPARATOR, versionHeaderValue);
+ }
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyIOFactory.java
similarity index 94%
copy from
pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
copy to
pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyIOFactory.java
index 6de66bad7c..6dfa2b87da 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyIOFactory.java
@@ -23,7 +23,7 @@ import
org.apache.commons.configuration2.PropertiesConfiguration.DefaultIOFactor
import
org.apache.commons.configuration2.PropertiesConfiguration.PropertiesReader;
-public class ConfigFilePropertyReaderFactory extends DefaultIOFactory {
+public class ConfigFilePropertyIOFactory extends DefaultIOFactory {
@Override
public PropertiesReader createPropertiesReader(Reader in) {
return new ConfigFilePropertyReader(in);
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
index 0516ee6b41..72be975544 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
@@ -213,9 +213,11 @@ public class PinotConfiguration {
PropertiesConfiguration propertiesConfiguration;
if (configPath.startsWith("classpath:")) {
propertiesConfiguration = CommonsConfigurationUtils.fromInputStream(
-
PinotConfiguration.class.getResourceAsStream(configPath.substring("classpath:".length())),
true, true);
+
PinotConfiguration.class.getResourceAsStream(configPath.substring("classpath:".length())),
true,
+ PropertyIOFactoryKind.ConfigFileIOFactory);
} else {
- propertiesConfiguration =
CommonsConfigurationUtils.fromPath(configPath, true, true);
+ propertiesConfiguration =
CommonsConfigurationUtils.fromPath(configPath, true,
+ PropertyIOFactoryKind.ConfigFileIOFactory);
}
return propertiesConfiguration;
} catch (ConfigurationException e) {
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PropertyIOFactoryKind.java
similarity index 53%
copy from
pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
copy to
pinot-spi/src/main/java/org/apache/pinot/spi/env/PropertyIOFactoryKind.java
index 6de66bad7c..bef441de3b 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PropertyIOFactoryKind.java
@@ -18,14 +18,38 @@
*/
package org.apache.pinot.spi.env;
-import java.io.Reader;
-import
org.apache.commons.configuration2.PropertiesConfiguration.DefaultIOFactory;
-import
org.apache.commons.configuration2.PropertiesConfiguration.PropertiesReader;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+public enum PropertyIOFactoryKind {
+ ConfigFileIOFactory {
+ public String toString() {
+ return "ConfigFile";
+ }
-public class ConfigFilePropertyReaderFactory extends DefaultIOFactory {
- @Override
- public PropertiesReader createPropertiesReader(Reader in) {
- return new ConfigFilePropertyReader(in);
- }
+ @Override
+ public ConfigFilePropertyIOFactory getInstance() {
+ return new ConfigFilePropertyIOFactory();
+ }
+ },
+ VersionedIOFactory {
+ public String toString() {
+ return "Versioned";
+ }
+ @Override
+ public VersionedIOFactory getInstance() {
+ return new VersionedIOFactory();
+ }
+ },
+ DefaultIOFactory {
+ public String toString() {
+ return "Default";
+ }
+ @Override
+ public PropertiesConfiguration.DefaultIOFactory getInstance() {
+ return new PropertiesConfiguration.DefaultIOFactory();
+ }
+ };
+
+ // get the instance of the IO factory.
+ public abstract PropertiesConfiguration.DefaultIOFactory getInstance();
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedIOFactory.java
similarity index 60%
copy from
pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
copy to pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedIOFactory.java
index 6de66bad7c..3ba7105357 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedIOFactory.java
@@ -19,13 +19,26 @@
package org.apache.pinot.spi.env;
import java.io.Reader;
+import java.io.Writer;
import
org.apache.commons.configuration2.PropertiesConfiguration.DefaultIOFactory;
import
org.apache.commons.configuration2.PropertiesConfiguration.PropertiesReader;
+import
org.apache.commons.configuration2.PropertiesConfiguration.PropertiesWriter;
+import org.apache.commons.configuration2.convert.ListDelimiterHandler;
-public class ConfigFilePropertyReaderFactory extends DefaultIOFactory {
+/**
+ * VersionedIOFactory extends the DefaultIOFactory
+ * <p>
+ * Purpose: factory class for creating the custom versioned property
configuration reader and writer.
+ */
+class VersionedIOFactory extends DefaultIOFactory {
+ @Override
+ public PropertiesReader createPropertiesReader(Reader reader) {
+ return new VersionedPropertyReader(reader);
+ }
+
@Override
- public PropertiesReader createPropertiesReader(Reader in) {
- return new ConfigFilePropertyReader(in);
+ public PropertiesWriter createPropertiesWriter(Writer out,
ListDelimiterHandler handler) {
+ return new VersionedPropertyWriter(out, handler);
}
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedPropertyReader.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedPropertyReader.java
new file mode 100644
index 0000000000..c108bfebcc
--- /dev/null
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedPropertyReader.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.env;
+
+import com.google.common.base.Preconditions;
+import java.io.Reader;
+import
org.apache.commons.configuration2.PropertiesConfiguration.PropertiesReader;
+import org.apache.commons.lang3.StringUtils;
+
+
+/**
+ * VersionedPropertyReader extends the PropertiesReader
+ * <p>
+ * Purpose: loads the segment metadata faster
+ * - by skipping the unescaping of key and
+ * - parsing the line by splitting based on first occurrence of separator
+ */
+class VersionedPropertyReader extends PropertiesReader {
+
+ public VersionedPropertyReader(Reader reader) {
+ super(reader);
+ }
+
+ @Override
+ protected void parseProperty(final String line) {
+ // skip the regex based parsing of the line content and splitting the
content based on first occurrence of separator
+ // getPropertySeparator(), in general returns the PropertiesConfiguration
`DEFAULT_SEPARATOR` value i.e. ' = '.
+ String separator = getPropertySeparator();
+
Preconditions.checkArgument(CommonsConfigurationUtils.VERSIONED_CONFIG_SEPARATOR.equals(separator),
+ String.format("Versioned property configuration separator '%s' should
be equal to '%s'",
+ separator, CommonsConfigurationUtils.VERSIONED_CONFIG_SEPARATOR));
+
+ String[] keyValue = StringUtils.splitByWholeSeparator(line, separator, 2);
+ Preconditions.checkArgument(keyValue.length == 2, "property content split
should result in key and value");
+ initPropertyName(keyValue[0]);
+ initPropertyValue(keyValue[1]);
+ initPropertySeparator(separator);
+ }
+
+ @Override
+ protected String unescapePropertyName(final String name) {
+ // skip the unescaping of the propertyName(key)
+ return name;
+ }
+}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedPropertyWriter.java
similarity index 61%
rename from
pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
rename to
pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedPropertyWriter.java
index 6de66bad7c..86dbbb95f1 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/env/ConfigFilePropertyReaderFactory.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/env/VersionedPropertyWriter.java
@@ -18,14 +18,25 @@
*/
package org.apache.pinot.spi.env;
-import java.io.Reader;
-import
org.apache.commons.configuration2.PropertiesConfiguration.DefaultIOFactory;
-import
org.apache.commons.configuration2.PropertiesConfiguration.PropertiesReader;
+import java.io.Writer;
+import
org.apache.commons.configuration2.PropertiesConfiguration.PropertiesWriter;
+import org.apache.commons.configuration2.convert.ListDelimiterHandler;
-public class ConfigFilePropertyReaderFactory extends DefaultIOFactory {
+/**
+ * SegmentMetadataPropertyWriter extends the PropertiesWriter
+ * <p>
+ * Purpose: custom property writer for writing the segment metadata faster by
skipping the escaping of key.
+ */
+public class VersionedPropertyWriter extends PropertiesWriter {
+
+ public VersionedPropertyWriter(final Writer writer, ListDelimiterHandler
handler) {
+ super(writer, handler);
+ }
+
@Override
- public PropertiesReader createPropertiesReader(Reader in) {
- return new ConfigFilePropertyReader(in);
+ protected String escapeKey(final String key) {
+ // skip the escapeKey functionality,
+ return key;
}
}
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java
index 06174ed212..19a7e83b2f 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/env/CommonsConfigurationUtilsTest.java
@@ -27,6 +27,7 @@ import
org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
+import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -39,6 +40,7 @@ import static org.testng.Assert.assertTrue;
public class CommonsConfigurationUtilsTest {
private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(),
"CommonsConfigurationUtilsTest");
private static final File CONFIG_FILE = new File(TEMP_DIR, "config");
+ private static final File SEGMENT_METADATA_CONFIG_FILE = new File(TEMP_DIR,
"segmentMetadataConfig");
private static final String PROPERTY_KEY = "testKey";
private static final int NUM_ROUNDS = 10000;
@@ -54,6 +56,34 @@ public class CommonsConfigurationUtilsTest {
FileUtils.deleteDirectory(TEMP_DIR);
}
+ @Test
+ public void testSegmentMetadataFromFile() {
+ // load the existing config and check the properties config instance
+ try {
+ PropertiesConfiguration config = CommonsConfigurationUtils
+ .getSegmentMetadataFromFile(SEGMENT_METADATA_CONFIG_FILE, true);
+ assertNotNull(config);
+
+ config.setProperty("testKey", "testValue");
+
+ // add the segment version header to the file and read it again
+ CommonsConfigurationUtils.saveSegmentMetadataToFile(config, CONFIG_FILE,
+ CommonsConfigurationUtils.PROPERTIES_CONFIGURATION_HEADER_VERSION_2);
+
+ // reading the property with header.
+ config =
CommonsConfigurationUtils.getSegmentMetadataFromFile(CONFIG_FILE, true);
+ assertNotNull(config);
+ assertEquals(config.getHeader(), "# version = 2");
+ } catch (Exception ex) {
+ Assert.fail(String.format("should not throw ConfigurationException
exception with valid file, %s",
+ ex.getMessage()));
+ }
+
+ // load the non-existing file and expect the exception
+ Assert.expectThrows(NullPointerException.class,
+ () -> CommonsConfigurationUtils.getSegmentMetadataFromFile(null,
true));
+ }
+
@Test
public void testPropertyValueWithSpecialCharacters()
throws Exception {
@@ -111,14 +141,16 @@ public class CommonsConfigurationUtilsTest {
return;
}
- PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromFile(CONFIG_FILE, false, true);
+ PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromFile(CONFIG_FILE, true,
+ PropertyIOFactoryKind.DefaultIOFactory);
configuration.setProperty(PROPERTY_KEY, replacedValue);
String recoveredValue =
CommonsConfigurationUtils.recoverSpecialCharacterInPropertyValue(
(String) configuration.getProperty(PROPERTY_KEY));
assertEquals(recoveredValue, value);
CommonsConfigurationUtils.saveToFile(configuration, CONFIG_FILE);
- configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE, false,
true);
+ configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE, true,
+ PropertyIOFactoryKind.DefaultIOFactory);
recoveredValue =
CommonsConfigurationUtils.recoverSpecialCharacterInPropertyValue(
(String) configuration.getProperty(PROPERTY_KEY));
assertEquals(recoveredValue, value);
@@ -127,12 +159,12 @@ public class CommonsConfigurationUtilsTest {
@Test
public void testPropertiesConfigurationFromFile()
throws ConfigurationException {
- PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromFile(null, false, true);
+ PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromFile(null, false, null);
assertNotNull(configuration);
configuration.setProperty("Test Key", "Test Value");
CommonsConfigurationUtils.saveToFile(configuration, CONFIG_FILE);
- configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE, false,
true);
+ configuration = CommonsConfigurationUtils.fromFile(CONFIG_FILE, false,
null);
assertNotNull(configuration);
assertEquals(configuration.getProperty("Test Key"), "Test Value");
}
@@ -140,12 +172,12 @@ public class CommonsConfigurationUtilsTest {
@Test
public void testPropertiesConfigurationFromPath()
throws ConfigurationException {
- PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromPath(null, false, true);
+ PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromPath(null, false, null);
assertNotNull(configuration);
configuration.setProperty("Test Key", "Test Value");
CommonsConfigurationUtils.saveToFile(configuration, CONFIG_FILE);
- configuration = CommonsConfigurationUtils.fromPath(CONFIG_FILE.getPath(),
false, true);
+ configuration = CommonsConfigurationUtils.fromPath(CONFIG_FILE.getPath(),
false, null);
assertNotNull(configuration);
assertEquals(configuration.getProperty("Test Key"), "Test Value");
}
@@ -153,13 +185,13 @@ public class CommonsConfigurationUtilsTest {
@Test
public void testPropertiesConfigurationFromInputStream()
throws ConfigurationException, FileNotFoundException {
- PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromInputStream(null, false, true);
+ PropertiesConfiguration configuration =
CommonsConfigurationUtils.fromInputStream(null, false, null);
assertNotNull(configuration);
configuration.setProperty("Test Key", "Test Value");
CommonsConfigurationUtils.saveToFile(configuration, CONFIG_FILE);
FileInputStream inputStream = new FileInputStream(CONFIG_FILE);
- configuration = CommonsConfigurationUtils.fromInputStream(inputStream,
false, true);
+ configuration = CommonsConfigurationUtils.fromInputStream(inputStream,
false, null);
assertNotNull(configuration);
assertEquals(configuration.getProperty("Test Key"), "Test Value");
}
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java
index a7c1f0783e..1c810dc6ef 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/env/PinotConfigurationTest.java
@@ -200,8 +200,8 @@ public class PinotConfigurationTest {
public void assertPropertiesFromBaseConfiguration()
throws ConfigurationException {
PropertiesConfiguration propertiesConfiguration =
CommonsConfigurationUtils.fromPath(
-
PropertiesConfiguration.class.getClassLoader().getResource("pinot-configuration-1.properties").getFile(),
true,
- true);
+
PropertiesConfiguration.class.getClassLoader().getResource("pinot-configuration-1.properties").getFile(),
+ true, PropertyIOFactoryKind.ConfigFileIOFactory);
PinotConfiguration config = new
PinotConfiguration(propertiesConfiguration);
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/env/VersionedPropertyConfigTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/env/VersionedPropertyConfigTest.java
new file mode 100644
index 0000000000..8bf07f3f46
--- /dev/null
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/env/VersionedPropertyConfigTest.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.env;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
+
+public class VersionedPropertyConfigTest {
+
+ private static final String COLON_SEPARATOR = ":";
+ private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(),
"VersionedPropertyConfigTest");
+ private static final File CONFIG_FILE = new File(TEMP_DIR, "config");
+ private static final String[] TEST_PROPERTY_KEY = { "test1", "test2_key",
"test3_key_",
+ "test4_key_1234", "test-1", "test.1" };
+ private static final String[] TEST_PROPERTY_KEY_WITH_SPECIAL_CHAR = {
"test:1", "test2=key",
+ "test3,key_", "test4:=._key_1234", "test5-1=" };
+
+ @BeforeClass
+ public void setUp()
+ throws IOException {
+ FileUtils.deleteDirectory(TEMP_DIR);
+ }
+
+ @AfterClass
+ @AfterMethod
+ public void tearDown()
+ throws IOException {
+ FileUtils.deleteDirectory(TEMP_DIR);
+ }
+
+ @Test
+ public void testVersionedPropertyConfiguration()
+ throws ConfigurationException {
+ testVersionedPropertiesConfiguration(null, TEST_PROPERTY_KEY, false);
+ }
+
+ @Test
+ public void testVersionedPropertyConfigurationWithDefaultHeaderVersion()
+ throws ConfigurationException {
+ testVersionedPropertiesConfiguration(
+
CommonsConfigurationUtils.DEFAULT_PROPERTIES_CONFIGURATION_HEADER_VERSION,
TEST_PROPERTY_KEY, false);
+ }
+
+ @Test
+ public void testVersionedPropertyConfigurationWithHeaderVersion2()
+ throws ConfigurationException {
+
testVersionedPropertiesConfiguration(CommonsConfigurationUtils.PROPERTIES_CONFIGURATION_HEADER_VERSION_2,
+ TEST_PROPERTY_KEY, false);
+ }
+
+ @Test
+ public void testVersionedReaderWithSpecialCharsPropertyKeys()
+ throws ConfigurationException {
+ testVersionedPropertiesConfiguration(null,
TEST_PROPERTY_KEY_WITH_SPECIAL_CHAR, false);
+ }
+
+ @Test
+ public void
testVersionedReaderWithSpecialCharsPropertyKeysWithDefaultHeader()
+ throws ConfigurationException {
+
testVersionedPropertiesConfiguration(CommonsConfigurationUtils.DEFAULT_PROPERTIES_CONFIGURATION_HEADER_VERSION,
+ TEST_PROPERTY_KEY_WITH_SPECIAL_CHAR, false);
+ }
+
+ @Test
+ public void
testVersionedReaderWithSpecialCharsPropertyKeysWithHeaderVersion2()
+ throws ConfigurationException {
+
testVersionedPropertiesConfiguration(CommonsConfigurationUtils.PROPERTIES_CONFIGURATION_HEADER_VERSION_2,
+ TEST_PROPERTY_KEY_WITH_SPECIAL_CHAR, false);
+ }
+
+ @Test
+ public void testVersionedReaderWithDifferentSeparator() throws
ConfigurationException {
+
testVersionedPropertiesConfiguration(CommonsConfigurationUtils.PROPERTIES_CONFIGURATION_HEADER_VERSION_2,
+ TEST_PROPERTY_KEY, true);
+ }
+
+ @Test
+ //Test requires 'segment-metadata-without-version-header.properties' sample
segment metadata file in resources folder
+ public void testOldSegmentMetadataBackwardCompatability()
+ throws ConfigurationException {
+ File oldSegmentProperties = new File(
+ Objects.requireNonNull(
+ PropertiesConfiguration.class.getClassLoader()
+
.getResource("segment-metadata-without-version-header.properties")).getFile());
+ PropertiesConfiguration configuration =
+
CommonsConfigurationUtils.getSegmentMetadataFromFile(oldSegmentProperties,
true);
+
+ // assert that Header is null for the config.
+ assertNull(configuration.getHeader());
+
+ // assert that configuration has DefaultIOFactory
+ assertEquals(configuration.getIOFactory().getClass(),
PropertiesConfiguration.DefaultIOFactory.class);
+
+ testSegmentMetadataContent(configuration);
+
+ // asserting escaped value ('column-ProductId-maxValue')
+ String productIDMaxValue =
configuration.getString("column-ProductId-maxValue");
+ assertNotNull(productIDMaxValue);
+ assertEquals(productIDMaxValue, "B009,WVB40S");
+ }
+
+ @Test
+ //Test requires 'segment-metadata-with-version-header.properties' sample
segment metadata file in resources folder
+ public void testSegmentMetadataWithVersionHeader()
+ throws ConfigurationException {
+ File oldSegmentProperties = new File(
+ Objects.requireNonNull(
+ PropertiesConfiguration.class.getClassLoader()
+
.getResource("segment-metadata-with-version-header.properties")).getFile());
+ PropertiesConfiguration configuration =
+
CommonsConfigurationUtils.getSegmentMetadataFromFile(oldSegmentProperties,
true);
+
+ // assert that Header is equals to '# version = 2'
+ assertEquals(configuration.getHeader(), "# version = 2");
+
+ // assert that configuration has SegmentMetadataPropertyIOFactory
+ assertEquals(configuration.getIOFactory().getClass(),
VersionedIOFactory.class);
+
+ testSegmentMetadataContent(configuration);
+ }
+
+ private static void testSegmentMetadataContent(PropertiesConfiguration
configuration) {
+ // getting all the keys, length of the list should be equal to the number
of lines in the segment metadata
+ List<String> keys = CommonsConfigurationUtils.getKeys(configuration);
+ assertEquals(keys.size(), 123);
+
+ // asserting table name property from segment metadata
+ String tableName = configuration.getString("segment.table.name");
+ assertEquals(tableName, "fineFoodReviews");
+
+ // asserting table name property from segment metadata
+ String segmentName = configuration.getString("segment.name");
+ assertEquals(segmentName, "fineFoodReviews_OFFLINE_0");
+
+ // asserting segment dimension column names from segment metadata
+ String[] segmentDimensionColumnNames =
configuration.getStringArray("segment.dimension.column.names");
+ assertEquals(segmentDimensionColumnNames.length, 8);
+ assertEquals(String.join(",", segmentDimensionColumnNames),
+ "ProductId,Score,Summary,Text,UserId,combined,embedding,n_tokens");
+
+ // asserting segment.index.version
+ String segmentIndexVersion =
configuration.getString("segment.index.version");
+ assertEquals(segmentIndexVersion, "v3");
+ }
+
+ private static void testVersionedPropertiesConfiguration(String
versionHeader, String[] keysArray,
+ boolean setDifferentSeparator)
+ throws ConfigurationException {
+ PropertiesConfiguration configuration =
+ CommonsConfigurationUtils.getSegmentMetadataFromFile(CONFIG_FILE,
true);
+
+ // setting the random value of the test keys
+ for (String key: keysArray) {
+ configuration.setProperty(key, RandomStringUtils.randomAscii(5));
+
+ // setting it at the key level as well for testing
+ if (setDifferentSeparator) {
+ configuration.getLayout().setSeparator(key, COLON_SEPARATOR);
+ }
+ }
+
+ // set the different separator, other than '='
+ if (setDifferentSeparator) {
+ configuration.getLayout().setGlobalSeparator(COLON_SEPARATOR);
+ }
+
+ // recovered keys from the configuration.
+ List<String> recoveredKeys =
CommonsConfigurationUtils.getKeys(configuration);
+ testPropertyKeys(recoveredKeys, keysArray);
+
+ // save the configuration.
+ CommonsConfigurationUtils.saveSegmentMetadataToFile(configuration,
CONFIG_FILE, versionHeader);
+
+ if (versionHeader != null) {
+ String expectedHeader =
CommonsConfigurationUtils.getVersionHeaderString(versionHeader);
+ assertEquals(configuration.getHeader(), expectedHeader);
+ }
+
+ // reading the configuration from saved file.
+ configuration =
CommonsConfigurationUtils.getSegmentMetadataFromFile(CONFIG_FILE, true);
+ recoveredKeys = CommonsConfigurationUtils.getKeys(configuration);
+ testPropertyKeys(recoveredKeys, keysArray);
+ }
+
+ private static void testPropertyKeys(List<String> recoveredKeys, String[]
actualKeys) {
+ assertEquals(recoveredKeys.size(), actualKeys.length);
+ for (int i = 0; i < recoveredKeys.size(); i++) {
+ String recoveredValue = recoveredKeys.get(i);
+ assertEquals(recoveredValue, actualKeys[i]);
+ }
+ }
+}
diff --git
a/pinot-spi/src/test/resources/segment-metadata-with-version-header.properties
b/pinot-spi/src/test/resources/segment-metadata-with-version-header.properties
new file mode 100644
index 0000000000..b01085a673
--- /dev/null
+++
b/pinot-spi/src/test/resources/segment-metadata-with-version-header.properties
@@ -0,0 +1,125 @@
+# version = 2
+
+segment.padding.character = \u0000
+segment.name = fineFoodReviews_OFFLINE_0
+segment.table.name = fineFoodReviews
+segment.dimension.column.names =
ProductId,Score,Summary,Text,UserId,combined,embedding,n_tokens
+segment.total.docs = 1000
+custom.input.data.file.uri =
~/examples/batch/fineFoodReviews/rawdata/fine_food_reviews_with_embeddings_1k.parquet.gzip
+column.ProductId.cardinality = 868
+column.ProductId.totalDocs = 1000
+column.ProductId.dataType = STRING
+column.ProductId.bitsPerElement = 10
+column.ProductId.lengthOfEachEntry = 10
+column.ProductId.columnType = DIMENSION
+column.ProductId.isSorted = false
+column.ProductId.hasDictionary = true
+column.ProductId.isSingleValues = true
+column.ProductId.maxNumberOfMultiValues = 0
+column.ProductId.totalNumberOfEntries = 1000
+column.ProductId.isAutoGenerated = false
+column.ProductId.minValue = 7310172001
+column.ProductId.maxValue = B009WVB40S
+column.ProductId.defaultNullValue = null
+column.Score.cardinality = 5
+column.Score.totalDocs = 1000
+column.Score.dataType = INT
+column.Score.bitsPerElement = 3
+column.Score.lengthOfEachEntry = 0
+column.Score.columnType = DIMENSION
+column.Score.isSorted = false
+column.Score.hasDictionary = true
+column.Score.isSingleValues = true
+column.Score.maxNumberOfMultiValues = 0
+column.Score.totalNumberOfEntries = 1000
+column.Score.isAutoGenerated = false
+column.Score.minValue = 1
+column.Score.maxValue = 5
+column.Score.defaultNullValue = -2147483648
+column.Summary.cardinality = 721
+column.Summary.totalDocs = 1000
+column.Summary.dataType = STRING
+column.Summary.bitsPerElement = 10
+column.Summary.lengthOfEachEntry = 125
+column.Summary.columnType = DIMENSION
+column.Summary.isSorted = false
+column.Summary.hasDictionary = true
+column.Summary.isSingleValues = true
+column.Summary.maxNumberOfMultiValues = 0
+column.Summary.totalNumberOfEntries = 1000
+column.Summary.isAutoGenerated = false
+column.Summary.minValue = 34 unique and one duplicate
+column.Summary.maxValue = yummy low fat snacks
+column.Summary.defaultNullValue = null
+column.Text.cardinality = 761
+column.Text.totalDocs = 1000
+column.Text.dataType = STRING
+column.Text.bitsPerElement = 10
+column.Text.lengthOfEachEntry = 0
+column.Text.columnType = DIMENSION
+column.Text.isSorted = false
+column.Text.hasDictionary = false
+column.Text.isSingleValues = true
+column.Text.maxNumberOfMultiValues = 0
+column.Text.totalNumberOfEntries = 1000
+column.Text.isAutoGenerated = false
+column.Text.defaultNullValue = null
+column.UserId.cardinality = 708
+column.UserId.totalDocs = 1000
+column.UserId.dataType = STRING
+column.UserId.bitsPerElement = 10
+column.UserId.lengthOfEachEntry = 21
+column.UserId.columnType = DIMENSION
+column.UserId.isSorted = false
+column.UserId.hasDictionary = true
+column.UserId.isSingleValues = true
+column.UserId.maxNumberOfMultiValues = 0
+column.UserId.totalNumberOfEntries = 1000
+column.UserId.isAutoGenerated = false
+column.UserId.minValue = A06364072LBY1F3ING9XN
+column.UserId.maxValue = AZUCLRMHEBUG0
+column.UserId.defaultNullValue = null
+column.combined.cardinality = 762
+column.combined.totalDocs = 1000
+column.combined.dataType = STRING
+column.combined.bitsPerElement = 10
+column.combined.lengthOfEachEntry = 512
+column.combined.columnType = DIMENSION
+column.combined.isSorted = false
+column.combined.hasDictionary = true
+column.combined.isSingleValues = true
+column.combined.maxNumberOfMultiValues = 0
+column.combined.totalNumberOfEntries = 1000
+column.combined.isAutoGenerated = false
+column.combined.defaultNullValue = null
+column.embedding.cardinality = 1158749
+column.embedding.totalDocs = 1000
+column.embedding.dataType = FLOAT
+column.embedding.bitsPerElement = 21
+column.embedding.lengthOfEachEntry = 0
+column.embedding.columnType = DIMENSION
+column.embedding.isSorted = false
+column.embedding.hasDictionary = true
+column.embedding.isSingleValues = false
+column.embedding.maxNumberOfMultiValues = 1536
+column.embedding.totalNumberOfEntries = 1536000
+column.embedding.isAutoGenerated = false
+column.embedding.minValue = -0.66914773
+column.embedding.maxValue = 0.23317827
+column.embedding.defaultNullValue = -Infinity
+column.n_tokens.cardinality = 201
+column.n_tokens.totalDocs = 1000
+column.n_tokens.dataType = INT
+column.n_tokens.bitsPerElement = 8
+column.n_tokens.lengthOfEachEntry = 0
+column.n_tokens.columnType = DIMENSION
+column.n_tokens.isSorted = false
+column.n_tokens.hasDictionary = true
+column.n_tokens.isSingleValues = true
+column.n_tokens.maxNumberOfMultiValues = 0
+column.n_tokens.totalNumberOfEntries = 1000
+column.n_tokens.isAutoGenerated = false
+column.n_tokens.minValue = 28
+column.n_tokens.maxValue = 645
+column.n_tokens.defaultNullValue = -2147483648
+segment.index.version = v3
diff --git
a/pinot-spi/src/test/resources/segment-metadata-without-version-header.properties
b/pinot-spi/src/test/resources/segment-metadata-without-version-header.properties
new file mode 100644
index 0000000000..09d0334582
--- /dev/null
+++
b/pinot-spi/src/test/resources/segment-metadata-without-version-header.properties
@@ -0,0 +1,123 @@
+segment.padding.character = \u0000
+segment.name = fineFoodReviews_OFFLINE_0
+segment.table.name = fineFoodReviews
+segment.dimension.column.names =
ProductId,Score,Summary,Text,UserId,combined,embedding,n_tokens
+segment.total.docs = 1000
+custom.input.data.file.uri =
~/examples/batch/fineFoodReviews/rawdata/fine_food_reviews_with_embeddings_1k.parquet.gzip
+column.ProductId.cardinality = 868
+column.ProductId.totalDocs = 1000
+column.ProductId.dataType = STRING
+column.ProductId.bitsPerElement = 10
+column.ProductId.lengthOfEachEntry = 10
+column.ProductId.columnType = DIMENSION
+column.ProductId.isSorted = false
+column.ProductId.hasDictionary = true
+column.ProductId.isSingleValues = true
+column.ProductId.maxNumberOfMultiValues = 0
+column.ProductId.totalNumberOfEntries = 1000
+column.ProductId.isAutoGenerated = false
+column.ProductId.minValue = 7310172001
+column-ProductId-maxValue = B009\\,WVB40S
+column.ProductId.defaultNullValue = null
+column.Score.cardinality = 5
+column.Score.totalDocs = 1000
+column.Score.dataType = INT
+column.Score.bitsPerElement = 3
+column.Score.lengthOfEachEntry = 0
+column.Score.columnType = DIMENSION
+column.Score.isSorted = false
+column.Score.hasDictionary = true
+column.Score.isSingleValues = true
+column.Score.maxNumberOfMultiValues = 0
+column.Score.totalNumberOfEntries = 1000
+column.Score.isAutoGenerated = false
+column.Score.minValue = 1
+column.Score.maxValue = 5
+column.Score.defaultNullValue = -2147483648
+column.Summary.cardinality = 721
+column.Summary.totalDocs = 1000
+column.Summary.dataType = STRING
+column.Summary.bitsPerElement = 10
+column.Summary.lengthOfEachEntry = 125
+column.Summary.columnType = DIMENSION
+column.Summary.isSorted = false
+column.Summary.hasDictionary = true
+column.Summary.isSingleValues = true
+column.Summary.maxNumberOfMultiValues = 0
+column.Summary.totalNumberOfEntries = 1000
+column.Summary.isAutoGenerated = false
+column.Summary.minValue = 34 unique and one duplicate
+column.Summary.maxValue = yummy low fat snacks
+column.Summary.defaultNullValue = null
+column.Text.cardinality = 761
+column.Text.totalDocs = 1000
+column.Text.dataType = STRING
+column.Text.bitsPerElement = 10
+column.Text.lengthOfEachEntry = 0
+column.Text.columnType = DIMENSION
+column.Text.isSorted = false
+column.Text.hasDictionary = false
+column.Text.isSingleValues = true
+column.Text.maxNumberOfMultiValues = 0
+column.Text.totalNumberOfEntries = 1000
+column.Text.isAutoGenerated = false
+column.Text.defaultNullValue = null
+column.UserId.cardinality = 708
+column.UserId.totalDocs = 1000
+column.UserId.dataType = STRING
+column.UserId.bitsPerElement = 10
+column.UserId.lengthOfEachEntry = 21
+column.UserId.columnType = DIMENSION
+column.UserId.isSorted = false
+column.UserId.hasDictionary = true
+column.UserId.isSingleValues = true
+column.UserId.maxNumberOfMultiValues = 0
+column.UserId.totalNumberOfEntries = 1000
+column.UserId.isAutoGenerated = false
+column.UserId.minValue = A06364072LBY1F3ING9XN
+column.UserId.maxValue = AZUCLRMHEBUG0
+column.UserId.defaultNullValue = null
+column.combined.cardinality = 762
+column.combined.totalDocs = 1000
+column.combined.dataType = STRING
+column.combined.bitsPerElement = 10
+column.combined.lengthOfEachEntry = 512
+column.combined.columnType = DIMENSION
+column.combined.isSorted = false
+column.combined.hasDictionary = true
+column.combined.isSingleValues = true
+column.combined.maxNumberOfMultiValues = 0
+column.combined.totalNumberOfEntries = 1000
+column.combined.isAutoGenerated = false
+column.combined.defaultNullValue = null
+column.embedding.cardinality = 1158749
+column.embedding.totalDocs = 1000
+column.embedding.dataType = FLOAT
+column.embedding.bitsPerElement = 21
+column.embedding.lengthOfEachEntry = 0
+column.embedding.columnType = DIMENSION
+column.embedding.isSorted = false
+column.embedding.hasDictionary = true
+column.embedding.isSingleValues = false
+column.embedding.maxNumberOfMultiValues = 1536
+column.embedding.totalNumberOfEntries = 1536000
+column.embedding.isAutoGenerated = false
+column.embedding.minValue = -0.66914773
+column.embedding.maxValue = 0.23317827
+column.embedding.defaultNullValue = -Infinity
+column.n_tokens.cardinality = 201
+column.n_tokens.totalDocs = 1000
+column.n_tokens.dataType = INT
+column.n_tokens.bitsPerElement = 8
+column.n_tokens.lengthOfEachEntry = 0
+column.n_tokens.columnType = DIMENSION
+column.n_tokens.isSorted = false
+column.n_tokens.hasDictionary = true
+column.n_tokens.isSingleValues = true
+column.n_tokens.maxNumberOfMultiValues = 0
+column.n_tokens.totalNumberOfEntries = 1000
+column.n_tokens.isAutoGenerated = false
+column.n_tokens.minValue = 28
+column.n_tokens.maxValue = 645
+column.n_tokens.defaultNullValue = -2147483648
+segment.index.version = v3
diff --git a/pom.xml b/pom.xml
index aaac77a9dc..6e71442f6b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2115,6 +2115,9 @@
<exclude>**/.project</exclude>
<exclude>**/.classpath</exclude>
<exclude>.externalToolBuilders/**</exclude>
+
+ <!-- specific files -->
+
<exclude>**/src/test/resources/*version-header.properties</exclude>
</excludes>
</licenseSet>
</licenseSets>
@@ -2240,6 +2243,9 @@
<!-- MISC -->
<exclude>**/.factorypath</exclude>
+
+ <!-- specific files -->
+ <exclude>**/src/test/resources/*version-header.properties</exclude>
</excludes>
</configuration>
</plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
