This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c0cd3f51d3e272b6abdcff0fb2d48cd508c7c64b Author: Michael Blow <[email protected]> AuthorDate: Wed Aug 20 19:26:17 2025 -0400 [NO ISSUE][HYR] Update Jackson to 2.19.2 to address CVEs - user model changes: yes - storage format changes: no - interface changes: no Adds new common properties to allow users to customize limits intro'd in Jackson 2.15: • JSON_MAX_DEPTH - The maximum nesting depth for JSON objects. The depth is a count of objects and arrays that have not been closed, { and [ respectively (default: 1000) • JSON_MAX_DOC_LENGTH - The maximum length of a JSON document in bytes (<=0 is no limit) (default: -1) • JSON_MAX_TOKEN_COUNT - The maximum number of JSON tokens in a JSON object (<=0 is no limit). A token is a single unit of input, such as a number, a string, an object start or end, or an array start or end (default: -1) • JSON_MAX_NUMBER_LENGTH - The maximum length of a JSON number in bytes (default: 1000) • JSON_MAX_STRING_LENGTH - The maximum length of a JSON string in bytes (default: Integer.MAX_VALUE (2147483647)) • JSON_MAX_NAME_LENGTH - The maximum length of a JSON name in bytes (default: 50000) - update Azure libraries, since they also include Jackson - update Netty libraries, for CVEs Ext-ref: MB-68123 Change-Id: Ic0b744711dd5097fbc3bff581f49e6fce857a409 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20251 Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Michael Blow <[email protected]> Tested-by: Michael Blow <[email protected]> --- .../asterix/common/config/ConfigUsageTest.java | 2 +- .../asterix/common/api/IPropertiesFactory.java | 3 + .../asterix/common/config/AsterixProperties.java | 10 +- .../asterix/common/config/JacksonProperties.java | 138 +++++++++++++++++++++ .../asterix/common/config/PropertiesFactory.java | 5 + asterixdb/pom.xml | 56 +++++++-- .../org/apache/hyracks/control/cc/CCDriver.java | 5 +- .../control/common/config/ConfigManager.java | 7 +- .../control/common/controllers/NCConfig.java | 3 +- .../java/org/apache/hyracks/util/StorageUtil.java | 23 ++-- .../org/apache/hyracks/util/StorageUnitTest.java | 17 ++- hyracks-fullstack/pom.xml | 96 +++++++++++++- 12 files changed, 331 insertions(+), 34 deletions(-) diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java index b1b0d1d41d..843bce8a9f 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java @@ -197,7 +197,7 @@ public class ConfigUsageTest { return maxWidth; } - private String extractValue(Column column, IOption option) { + protected String extractValue(Column column, IOption option) { switch (column) { case SECTION: return getSectionDisplayFunction().apply(option.section()); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java index 7857f18fc3..0520fe68f5 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java @@ -22,6 +22,7 @@ import org.apache.asterix.common.config.ActiveProperties; import org.apache.asterix.common.config.BuildProperties; import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.ExternalProperties; +import org.apache.asterix.common.config.JacksonProperties; import org.apache.asterix.common.config.MessagingProperties; import org.apache.asterix.common.config.MetadataProperties; import org.apache.asterix.common.config.NodeProperties; @@ -100,4 +101,6 @@ public interface IPropertiesFactory { * @return new node properties */ NodeProperties newNodeProperties(); + + JacksonProperties newJacksonProperties(); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java index d192b495e0..fab1d1ae61 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java @@ -37,7 +37,7 @@ public class AsterixProperties { configManager.register(NodeProperties.Option.class, CompilerProperties.Option.class, MetadataProperties.Option.class, ExternalProperties.Option.class, ActiveProperties.Option.class, MessagingProperties.Option.class, ReplicationProperties.Option.class, StorageProperties.Option.class, - TransactionProperties.Option.class); + TransactionProperties.Option.class, JacksonProperties.Option.class); // we need to process the old-style asterix config before we apply defaults! configManager.addConfigurator(IConfigManager.ConfiguratorMetric.APPLY_DEFAULTS.metric() - 1, () -> { @@ -47,5 +47,13 @@ public class AsterixProperties { throw HyracksDataException.create(e); } }); + configManager.addConfigurator(IConfigManager.ConfiguratorMetric.APPLY_DEFAULTS.metric() + 1, () -> { + try { + PropertiesAccessor accessor = PropertiesAccessor.getInstance(configManager.getAppConfig()); + JacksonProperties.configureJackson(accessor); + } catch (AsterixException e) { + throw HyracksDataException.create(e); + } + }); } } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/JacksonProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/JacksonProperties.java new file mode 100644 index 0000000000..ced27f732b --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/JacksonProperties.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import static org.apache.hyracks.control.common.config.OptionTypes.LONG; +import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; +import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER; +import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER_BYTE_UNIT; + +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.hyracks.api.config.IOption; +import org.apache.hyracks.api.config.IOptionType; +import org.apache.hyracks.api.config.Section; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamWriteConstraints; + +public class JacksonProperties extends AbstractProperties { + + public enum Option implements IOption { + JSON_MAX_DEPTH(POSITIVE_INTEGER, StreamReadConstraints.DEFAULT_MAX_DEPTH), + JSON_MAX_DOC_LENGTH(LONG_BYTE_UNIT, StreamReadConstraints.DEFAULT_MAX_DOC_LEN), + JSON_MAX_TOKEN_COUNT(LONG, StreamReadConstraints.DEFAULT_MAX_TOKEN_COUNT), + JSON_MAX_NUMBER_LENGTH(POSITIVE_INTEGER_BYTE_UNIT, StreamReadConstraints.DEFAULT_MAX_NUM_LEN), + JSON_MAX_STRING_LENGTH(POSITIVE_INTEGER_BYTE_UNIT, Integer.MAX_VALUE), + JSON_MAX_NAME_LENGTH(POSITIVE_INTEGER_BYTE_UNIT, StreamReadConstraints.DEFAULT_MAX_NAME_LEN); + + private final IOptionType type; + private final Object defaultValue; + + Option(IOptionType type, Object defaultValue) { + this.type = type; + this.defaultValue = defaultValue; + } + + @Override + public Section section() { + return Section.COMMON; + } + + @Override + public String description() { + switch (this) { + case JSON_MAX_DEPTH: + return "The maximum nesting depth for JSON objects. The depth is a count of objects and arrays that have not been closed, `{` and `[` respectively"; + case JSON_MAX_DOC_LENGTH: + return "The maximum length of a JSON document in bytes"; + case JSON_MAX_TOKEN_COUNT: + return "The maximum number of JSON tokens in a JSON object (<=0 is no limit). A token is a single unit of input, such as a number, a string, an object start or end, or an array start or end"; + case JSON_MAX_NUMBER_LENGTH: + return "The maximum length of a JSON number in bytes (<=0 is no limit)"; + case JSON_MAX_STRING_LENGTH: + return "The maximum length of a JSON string in bytes (<=0 is no limit)"; + case JSON_MAX_NAME_LENGTH: + return "The maximum length of a JSON name in bytes"; + default: + throw new IllegalStateException("NYI: " + this); + } + } + + @Override + public IOptionType type() { + return type; + } + + @Override + public Object defaultValue() { + return defaultValue; + } + } + + private static final Logger LOGGER = LogManager.getLogger(); + + static void configureJackson(PropertiesAccessor accessor) { + StreamWriteConstraints writeConstraints = + StreamWriteConstraints.builder().maxNestingDepth(accessor.getInt(Option.JSON_MAX_DEPTH)).build(); + StreamWriteConstraints.overrideDefaultStreamWriteConstraints(writeConstraints); + StreamReadConstraints readConstraints = + StreamReadConstraints.builder().maxNestingDepth(accessor.getInt(Option.JSON_MAX_DEPTH)) + .maxDocumentLength(accessor.getLong(Option.JSON_MAX_DOC_LENGTH)) + .maxTokenCount(accessor.getLong(Option.JSON_MAX_TOKEN_COUNT)) + .maxNameLength(accessor.getInt(Option.JSON_MAX_NAME_LENGTH)) + .maxStringLength(accessor.getInt(Option.JSON_MAX_STRING_LENGTH)) + .maxNumberLength(accessor.getInt(Option.JSON_MAX_NUMBER_LENGTH)).build(); + StreamReadConstraints.overrideDefaultStreamReadConstraints(readConstraints); + LOGGER.info("Configured Jackson read & write constraints: {{}}", Stream.of(Option.values()) + .map(option -> option.camelCase() + "=" + accessor.get(option)).collect(Collectors.joining(", "))); + } + + public JacksonProperties(PropertiesAccessor accessor) { + super(accessor); + } + + public int getJsonMaxDepth() { + return accessor.getInt(Option.JSON_MAX_DEPTH); + } + + public long getJsonMaxDocLength() { + return accessor.getLong(Option.JSON_MAX_DOC_LENGTH); + } + + public long getJsonMaxTokenCount() { + return accessor.getLong(Option.JSON_MAX_TOKEN_COUNT); + } + + public int getJsonMaxNumberLength() { + return accessor.getInt(Option.JSON_MAX_NUMBER_LENGTH); + } + + public int getJsonMaxStringLength() { + return accessor.getInt(Option.JSON_MAX_STRING_LENGTH); + } + + public int getJsonMaxNameLength() { + return accessor.getInt(Option.JSON_MAX_NAME_LENGTH); + } + +} diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java index 8f75397c62..88bf908fd5 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java @@ -77,4 +77,9 @@ public class PropertiesFactory implements IPropertiesFactory { public NodeProperties newNodeProperties() { return new NodeProperties(propertiesAccessor); } + + @Override + public JacksonProperties newJacksonProperties() { + return new JacksonProperties(propertiesAccessor); + } } diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index 0edbe0f732..c5c278c84a 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -90,10 +90,14 @@ <awsjavasdk.version>2.29.27</awsjavasdk.version> <parquet.version>1.15.2</parquet.version> <!-- NOTICE: please update transitives from parquet below on any change --> <hadoop-awsjavasdk.version>1.12.779</hadoop-awsjavasdk.version> - <azureblobjavasdk.version>12.25.1</azureblobjavasdk.version> - <azurecommonjavasdk.version>12.24.1</azurecommonjavasdk.version> - <azureidentity.version>1.13.3</azureidentity.version> - <azuredatalakejavasdk.version>12.18.1</azuredatalakejavasdk.version> + + <azureblobjavasdk.version>12.31.1</azureblobjavasdk.version> + <azurecommonjavasdk.version>12.30.1</azurecommonjavasdk.version> + <azureidentity.version>1.17.0</azureidentity.version> + <azuredatalakejavasdk.version>12.24.1</azuredatalakejavasdk.version> + <azurecore.version>1.56.0</azurecore.version> + <azurecorehttpnetty.version>1.16.0</azurecorehttpnetty.version> + <gcsjavasdk.version>2.45.0</gcsjavasdk.version> <hadoop-azuresdk.version>8.6.6</hadoop-azuresdk.version> @@ -1052,10 +1056,6 @@ <groupId>ch.qos.reload4j</groupId> <artifactId>reload4j</artifactId> </exclusion> - <exclusion> - <groupId>com.fasterxml</groupId> - <artifactId>woodstox-core</artifactId> - </exclusion> <exclusion> <groupId>dnsjava</groupId> <artifactId>dnsjava</artifactId> @@ -1214,6 +1214,16 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.codehaus.woodstox</groupId> + <artifactId>stax2-api</artifactId> + <version>4.2.2</version> + </dependency> + <dependency> + <groupId>com.fasterxml.woodstox</groupId> + <artifactId>woodstox-core</artifactId> + <version>7.1.1</version> + </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> @@ -1619,6 +1629,34 @@ <artifactId>azure-storage-common</artifactId> <version>${azurecommonjavasdk.version}</version> </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-core</artifactId> + <version>${azurecore.version}</version> + </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-core-http-netty</artifactId> + <version>${azurecorehttpnetty.version}</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-epoll</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-unix-common</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-kqueue</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-resolver-dns-native-macos</artifactId> + </exclusion> + </exclusions> + </dependency> <!-- Azure Blob Storage end --> <!-- Google Cloud Storage start --> <dependency> @@ -1802,7 +1840,7 @@ <dependency> <groupId>io.netty</groupId> <artifactId>netty-tcnative-boringssl-static</artifactId> - <version>2.0.71.Final</version> + <version>2.0.72.Final</version> </dependency> </dependencies> </dependencyManagement> diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java index 8b5804697d..80ce8ceb9e 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java @@ -22,6 +22,7 @@ import static org.apache.hyracks.control.common.controllers.CCConfig.Option.APP_ import java.io.IOException; import java.util.Arrays; +import java.util.concurrent.Semaphore; import org.apache.hyracks.api.application.ICCApplication; import org.apache.hyracks.control.common.config.ConfigManager; @@ -57,9 +58,7 @@ public class CCDriver { ctx.start(logCfgFactory.getConfiguration(ctx, ConfigurationSource.NULL_SOURCE)); ClusterControllerService ccService = new ClusterControllerService(ccConfig, application); ccService.start(); - while (true) { - Thread.sleep(100000); - } + new Semaphore(0).acquire(); } catch (CmdLineException e) { LOGGER.log(Level.DEBUG, "Exception parsing command line: " + Arrays.toString(args), e); System.exit(2); diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java index 9c5a9fa144..9ebb14263a 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java @@ -585,12 +585,17 @@ public class ConfigManager implements IConfigManager, Serializable { } public String defaultTextForUsage(IOption option, Function<IOption, String> optionPrinter) { + return defaultTextForUsage(option, optionPrinter, IOption::defaultValue); + } + + public String defaultTextForUsage(IOption option, Function<IOption, String> optionPrinter, + Function<IOption, Object> defaultValueFunction) { StringBuilder buf = new StringBuilder(); String override = option.usageDefaultOverride(appConfig, optionPrinter); if (override != null) { buf.append(override); } else { - final Object value = option.defaultValue(); + final Object value = defaultValueFunction.apply(option); if (value instanceof IOption) { buf.append("same as ").append(optionPrinter.apply((IOption) value)); } else if (value instanceof Function) { diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java index 01cb9bfc9d..44cb7e61a1 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java @@ -101,8 +101,7 @@ public class NCConfig extends ControllerConfig { PYTHON_ENV(STRING_ARRAY, (String[]) null), CREDENTIAL_FILE( OptionTypes.STRING, - (Function<IApplicationConfig, String>) appConfig -> FileUtil - .joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "passwd"), + appConfig -> FileUtil.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "passwd"), ControllerConfig.Option.DEFAULT_DIR.cmdline() + "/passwd"); private final IOptionType parser; diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java index 0456dc7304..e7b6ec7a32 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java @@ -26,15 +26,15 @@ import java.util.regex.Pattern; public class StorageUtil { public static final int BASE = 1024; - private static final Pattern PATTERN = Pattern.compile("^(-?[.0-9]+)([A-Z]{0,2})$"); + private static final Pattern PATTERN = Pattern.compile("^(-?[.0-9]+)([A-Z]{0,3})$"); public enum StorageUnit { BYTE("B", "b", 1), - KILOBYTE("KB", "kb", BASE), - MEGABYTE("MB", "m", KILOBYTE.multiplier * BASE), - GIGABYTE("GB", "g", MEGABYTE.multiplier * BASE), - TERABYTE("TB", "t", GIGABYTE.multiplier * BASE), - PETABYTE("PB", "p", TERABYTE.multiplier * BASE); + KILOBYTE("KiB", "kb", BASE), + MEGABYTE("MiB", "m", KILOBYTE.multiplier * BASE), + GIGABYTE("GiB", "g", MEGABYTE.multiplier * BASE), + TERABYTE("TiB", "t", GIGABYTE.multiplier * BASE), + PETABYTE("PiB", "p", TERABYTE.multiplier * BASE); private final String unitTypeInLetter; private final String linuxUnitTypeInLetter; @@ -43,7 +43,8 @@ public class StorageUtil { static { for (StorageUnit unit : values()) { - SUFFIX_TO_UNIT_MAP.put(unit.unitTypeInLetter, unit); + SUFFIX_TO_UNIT_MAP.put(unit.unitTypeInLetter.toUpperCase(), unit); + SUFFIX_TO_UNIT_MAP.put(unit.unitTypeInLetter.replace("i", ""), unit); } } @@ -124,11 +125,11 @@ public class StorageUtil { private static IllegalArgumentException invalidFormatException(String s) { return new IllegalArgumentException( - "The given string: " + s + " is not a byte unit string (e.g., 320KB or 1024)."); + "The given string: " + s + " is not a byte unit string (e.g., 320KiB or 1024)."); } /** - * Return byte value for the given string (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...) + * Return byte value for the given string (e.g., 0.1KiB, 100kb, 1mb, 3MiB, 8.5GiB ...) * * @throws IllegalArgumentException */ @@ -143,7 +144,7 @@ public class StorageUtil { /** * Returns a human readable value in storage units rounded up to two decimal places. - * e.g. toHumanReadableSize(1024L * 1024L * 1024l * 10L *) + 1024l * 1024l * 59) returns 1.06 GB + * e.g. toHumanReadableSize(1024L * 1024L * 1024l * 10L *) + 1024l * 1024l * 59) returns 1.06 GiB * * @param bytes * @return Value in storage units. @@ -153,7 +154,7 @@ public class StorageUtil { return bytes + " B"; } final int baseValue = (63 - Long.numberOfLeadingZeros(bytes)) / 10; - final char bytePrefix = " kMGTPE".charAt(baseValue); + final String bytePrefix = new String[] { " ", "Ki", "Mi", "Gi", "Ti", "Pi" }[baseValue]; final long divisor = 1L << (baseValue * 10); if (bytes % divisor == 0) { return String.format("%d %sB", bytes / divisor, bytePrefix); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java index 445d15f6f9..c15ee80b2a 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/StorageUnitTest.java @@ -24,7 +24,7 @@ import org.junit.Test; public class StorageUnitTest { @Test - public void test() { + public void testParse() { // Valid cases double result1NoUnit = StorageUtil.getSizeInBytes("1"); // Defaults to bytes Assert.assertEquals(1.0, result1NoUnit, 0); @@ -38,6 +38,9 @@ public class StorageUnitTest { double result1Kb = StorageUtil.getSizeInBytes("1KB"); Assert.assertEquals(1024.0, result1Kb, 0); + double result1Kb2 = StorageUtil.getSizeInBytes("1KiB"); + Assert.assertEquals(1024.0, result1Kb2, 0); + double result1KbWithSpaces = StorageUtil.getSizeInBytes(" 1 K B "); Assert.assertEquals(1024.0, result1KbWithSpaces, 0); @@ -68,6 +71,17 @@ public class StorageUnitTest { invalidCase("123MBB"); } + @Test + public void testToString() { + Assert.assertEquals("1 KiB", StorageUtil.toHumanReadableSize(1024)); + Assert.assertEquals("1.02 KiB", StorageUtil.toHumanReadableSize(1048)); + Assert.assertEquals("1 MiB", StorageUtil.toHumanReadableSize(1024 * 1024)); + Assert.assertEquals("1.50 MiB", StorageUtil.toHumanReadableSize(1024 * 1536)); + Assert.assertEquals("1.75 MiB", StorageUtil.toHumanReadableSize(StorageUtil.getByteValue("1.75 MiB"))); + Assert.assertEquals("1.75 MiB", StorageUtil.toHumanReadableSize(StorageUtil.getByteValue("1.75 MB"))); + Assert.assertEquals("11.77 TiB", StorageUtil.toHumanReadableSize(StorageUtil.getByteValue("12345678 MB"))); + } + private void invalidCase(String value) { try { StorageUtil.getSizeInBytes(value); @@ -76,4 +90,5 @@ public class StorageUnitTest { .contains("IllegalArgumentException: The given string: " + value + " is not a byte unit string")); } } + } diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml index 19bffbbe95..a72968d200 100644 --- a/hyracks-fullstack/pom.xml +++ b/hyracks-fullstack/pom.xml @@ -74,9 +74,9 @@ <jacoco.version>0.7.6.201602180812</jacoco.version> <log4j.version>2.19.0</log4j.version> <snappy.version>1.1.10.5</snappy.version> - <jackson.version>2.14.3</jackson.version> + <jackson.version>2.19.2</jackson.version> <jackson-databind.version>${jackson.version}</jackson-databind.version> - <netty.version>4.1.121.Final</netty.version> + <netty.version>4.1.124.Final</netty.version> <implementation.title>Apache Hyracks and Algebricks - ${project.name}</implementation.title> <implementation.url>https://asterixdb.apache.org/</implementation.url> @@ -106,6 +106,10 @@ <groupId>io.netty</groupId> <artifactId>netty-transport-classes-kqueue</artifactId> </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-resolver-dns</artifactId> + </exclusion> <exclusion> <groupId>io.netty</groupId> <artifactId>netty-resolver-dns-classes-macos</artifactId> @@ -171,6 +175,24 @@ <groupId>io.netty</groupId> <artifactId>netty-transport</artifactId> <version>${netty.version}</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-epoll</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-classes-epoll</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-kqueue</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-classes-kqueue</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>io.netty</groupId> @@ -205,19 +227,46 @@ </dependency> <dependency> <groupId>io.netty</groupId> - <artifactId>netty-transport-classes-epoll</artifactId> + <artifactId>netty-transport-native-epoll</artifactId> <version>${netty.version}</version> </dependency> <dependency> <groupId>io.netty</groupId> - <artifactId>netty-transport-native-epoll</artifactId> + <artifactId>netty-resolver</artifactId> <version>${netty.version}</version> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> + <!-- <dependency> <groupId>io.netty</groupId> - <artifactId>netty-resolver</artifactId> + <artifactId>netty-resolver-dns-native-macos</artifactId> <version>${netty.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-resolver-dns-classes-macos</artifactId> + <version>${netty.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + --> <dependency> <groupId>io.netty</groupId> <artifactId>netty-transport-native-epoll</artifactId> @@ -240,11 +289,33 @@ <artifactId>netty-resolver-dns</artifactId> <version>${netty.version}</version> </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-resolver-dns-native-macos</artifactId> + <version>${netty.version}</version> + <classifier>osx-x86_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-resolver-dns-native-macos</artifactId> + <version>${netty.version}</version> + <classifier>osx-aarch_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-resolver-dns-classes-macos</artifactId> + <version>${netty.version}</version> + </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-codec-http2</artifactId> <version>${netty.version}</version> </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-unix-common</artifactId> + <version>${netty.version}</version> + </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> @@ -381,6 +452,21 @@ <artifactId>jackson-annotations</artifactId> <version>${jackson.version}</version> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-cbor</artifactId> + <version>${jackson.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-xml</artifactId> + <version>${jackson.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jsr310</artifactId> + <version>${jackson.version}</version> + </dependency> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId>
