This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/datasketches-memory17.git
commit 2d69298964a44cdf46f71ba0e1cac30da518f78a Author: Lee Rhodes <[email protected]> AuthorDate: Mon Oct 23 09:34:41 2023 -0700 Update memory17 --- .gitignore | 10 +- ...tasketches-memory-tests-3.0.0.jar_Manifests.txt | 12 -- Manifests/jcommander-1.82.jar_Manifest.txt | 18 -- Manifests/slf4j-api-1.7.36.jar_Manifest.txt | 25 --- Manifests/slf4j-nop-1.7.36.jar_Manifest.txt | 25 --- Manifests/testng-7.8.0.jar_Manifest.txt | 56 ------ .../zero-allocation-hashing-0.15_Manifest.txt | 30 ---- compile.sh | 49 +++-- libs/datasketches-memory-3.0.0.jar | Bin 43810 -> 45528 bytes libs/datasketches-memory-tests-3.0.0.jar | Bin 112533 -> 114606 bytes mods/.keep.txt | 0 runtestng.sh | 25 --- {libs => scripts}/myargfile.txt | 11 +- libs/testng.xml => scripts/testngByClasses.xml | 1 + scripts/testngByPackage.xml | 29 +++ .../apache/datasketches/memory/internal/Util.java | 199 +++++++++++++++++---- .../internal/AllocateDirectMapMemoryTest.java | 2 +- .../memory/internal/ResourceFromJarTest.java | 71 ++++++++ .../datasketches/memory/internal/UtilTest.java | 10 +- 19 files changed, 317 insertions(+), 256 deletions(-) diff --git a/.gitignore b/.gitignore index 374ff7b..710abd9 100644 --- a/.gitignore +++ b/.gitignore @@ -45,10 +45,11 @@ test-output/ local/ reports/ .pmd -tmp/ +tmp/** # Build artifacts target/ +target2/ out/ build/ jarsIn/ @@ -61,5 +62,12 @@ doc/ # Jekyll _site/ + _* _*/ +**/.keep.txt +**/GettysburgAddress.txt +**/MANIFEST.MF +libs +mods + diff --git a/Manifests/datasketches-memory-tests-3.0.0.jar_Manifests.txt b/Manifests/datasketches-memory-tests-3.0.0.jar_Manifests.txt deleted file mode 100644 index 9bcf0c7..0000000 --- a/Manifests/datasketches-memory-tests-3.0.0.jar_Manifests.txt +++ /dev/null @@ -1,12 +0,0 @@ -datasketches-memory-tests-3.0.0.jar #NOTE RENAMED THE FILE!! - -$ jar --describe-module --file datasketches-memory-tests-3.0.0.jar -No module descriptor found. Derived automatic module. - [email protected] automatic -requires java.base mandated -contains org.apache.datasketches.memory.internal - -$ unzip -p datasketches-memory-tests-3.0.0.jar META-INF/MANIFEST.MF -Manifest-Version: 1.0 -Created-By: 17.0.8 (Eclipse Adoptium) diff --git a/Manifests/jcommander-1.82.jar_Manifest.txt b/Manifests/jcommander-1.82.jar_Manifest.txt deleted file mode 100644 index c354f1c..0000000 --- a/Manifests/jcommander-1.82.jar_Manifest.txt +++ /dev/null @@ -1,18 +0,0 @@ -jcommander-1.82.jar - -$ unzip -p jcommander-1.82.jar META-INF/MANIFEST.MF -Manifest-Version: 1.0 - -$ jar --describe-module --file jcommander-1.82.jar -No module descriptor found. Derived automatic module. - [email protected] automatic -requires java.base mandated -contains com.beust.ah -contains com.beust.jcommander -contains com.beust.jcommander.converters -contains com.beust.jcommander.defaultprovider -contains com.beust.jcommander.internal -contains com.beust.jcommander.parser -contains com.beust.jcommander.validators - diff --git a/Manifests/slf4j-api-1.7.36.jar_Manifest.txt b/Manifests/slf4j-api-1.7.36.jar_Manifest.txt deleted file mode 100644 index 604d408..0000000 --- a/Manifests/slf4j-api-1.7.36.jar_Manifest.txt +++ /dev/null @@ -1,25 +0,0 @@ -slf4j-api-1.7.36.jar -# This is an automatic module; name=org.slf4j -$ unzip -p slf4j-api-1.7.36.jar META-INF/MANIFEST.MF -Manifest-Version: 1.0 -Bundle-Description: The slf4j API -Implementation-Title: slf4j-api -Automatic-Module-Name: org.slf4j -Bundle-SymbolicName: slf4j.api -Implementation-Version: 1.7.36 -Bundle-ManifestVersion: 2 -Bundle-RequiredExecutionEnvironment: J2SE-1.5 -Bundle-Vendor: SLF4J.ORG -Import-Package: - org.slf4j.impl;version=1.6.0 -Export-Package: - org.slf4j;version=1.7.36, - org.slf4j.spi;version=1.7.36, - org.slf4j.helpers;version=1.7.36, - org.slf4j.event;version=1.7.36 -Bundle-Version: 1.7.36 -Bundle-Name: slf4j-api -X-Compile-Target-JDK: 1.5 -X-Compile-Source-JDK: 1.5 -Build-Jdk-Spec: 1.8 -Created-By: Maven Jar Plugin 3.2.0 \ No newline at end of file diff --git a/Manifests/slf4j-nop-1.7.36.jar_Manifest.txt b/Manifests/slf4j-nop-1.7.36.jar_Manifest.txt deleted file mode 100644 index 65abdb3..0000000 --- a/Manifests/slf4j-nop-1.7.36.jar_Manifest.txt +++ /dev/null @@ -1,25 +0,0 @@ -slf4j-nop-1.7.36.jar -# This is an automatic module; name=org.slf4j.nop -$ unzip -p slf4j-nop-1.7.36.jar META-INF/MANIFEST.MF -Manifest-Version: 1.0 -Bundle-Description: SLF4J NOP Binding -Implementation-Title: slf4j-nop -Automatic-Module-Name: org.slf4j.nop -Bundle-SymbolicName: slf4j.nop -Implementation-Version: 1.7.36 -Bundle-ManifestVersion: 2 -Bundle-RequiredExecutionEnvironment: J2SE-1.5 -Bundle-Vendor: SLF4J.ORG -Import-Package: - org.slf4j;version=1.7.36, - org.slf4j.spi;version=1.7.36, - org.slf4j.helpers;version=1.7.36, - org.slf4j.event;version=1.7.36 -Export-Package: - org.slf4j.impl;version=1.7.36 -Bundle-Version: 1.7.36 -Bundle-Name: slf4j-nop -X-Compile-Target-JDK: 1.5 -X-Compile-Source-JDK: 1.5 -Build-Jdk-Spec: 1.8 -Created-By: Maven Jar Plugin 3.2.0 \ No newline at end of file diff --git a/Manifests/testng-7.8.0.jar_Manifest.txt b/Manifests/testng-7.8.0.jar_Manifest.txt deleted file mode 100644 index c55c615..0000000 --- a/Manifests/testng-7.8.0.jar_Manifest.txt +++ /dev/null @@ -1,56 +0,0 @@ -testng-7.8.0.jar -# This is an automatic module; name=org.testng -unzip -p testng-7.8.0.jar META-INF/MANIFEST.MF -Manifest-Version: 1.0 -Specification-Title: TestNG -Specification-Version: 7.8.0 -Specification-Vendor: TestNG -Implementation-Title: TestNG -Implementation-Version: 7.8.0 -Implementation-Vendor: TestNG -Implementation-Vendor-Id: org.testng -Implementation-Url: https://testng.org -Automatic-Module-Name: org.testng -Bundle-ManifestVersion: 2 -Bundle-Name: TestNG -Bundle-SymbolicName: org.testng -Bundle-Vendor: TestNG -Bundle-License: Apache-2.0 -Bundle-Description: Testing framework for Java -Bundle-Version: 7.8.0 -Import-Package: - bsh.*;version="[2.0.0,3.0.0)";resolution:=optional, - com.beust.jcommander.*;version="[1.7.0,3.0.0)";resolution:=optional, - com.google.inject.*;version="[1.2,1.3)";resolution:=optional, - junit.framework;version="[3.8.1, 5.0.0)";resolution:=optional, - org.junit.*;resolution:=optional, - org.apache.tools.ant.*;version="[1.7.0, 2.0.0)";resolution:=optional, - org.yaml.*;version="[1.6,2.0)";resolution:=optional, - *;resolution:=optional -Export-Package: - org.testng, - org.testng.annotations, - org.testng.asserts, - org.testng.collections, - org.testng.internal, - org.testng.internal.annotations, - org.testng.internal.ant, - org.testng.internal.collections, - org.testng.internal.invokers, - org.testng.internal.invokers.objects, - org.testng.internal.junit, - org.testng.internal.objects, - org.testng.internal.objects.pojo, - org.testng.internal.reflect, - org.testng.internal.thread, - org.testng.internal.thread.graph, - org.testng.junit, - org.testng.log, - org.testng.log4testng, - org.testng.reporters, - org.testng.reporters.jq, - org.testng.reporters.util, - org.testng.thread, - org.testng.util, - org.testng.xml, - org.testng.xml.internal diff --git a/Manifests/zero-allocation-hashing-0.15_Manifest.txt b/Manifests/zero-allocation-hashing-0.15_Manifest.txt deleted file mode 100644 index 44a286f..0000000 --- a/Manifests/zero-allocation-hashing-0.15_Manifest.txt +++ /dev/null @@ -1,30 +0,0 @@ -zero-allocation-hashing-0.15.jar - -$ unzip -p zero-allocation-hashing-0.15.jar META-INF/MANIFEST.MF -Manifest-Version: 1.0 -Bnd-LastModified: 1624126699226 -Build-Jdk-Spec: 1.8 -Bundle-Description: Zero-allocation implementations of fast non-crypto - graphic hash functions for byte sequences or blocks of memory -Bundle-License: http://www.apache.org/licenses/LICENSE-2.0 -Bundle-ManifestVersion: 2 -Bundle-Name: OpenHFT :: zero-allocation-hashing -Bundle-SymbolicName: net.openhft.zero-allocation-hashing -Bundle-Version: 0.15 -Created-By: Apache Maven Bundle Plugin -Export-Package: net.openhft.hashing;uses:="javax.annotation";version=" - 0.15.0" -Import-Package: - javax.annotation;version="[3.0,4)", - sun.misc, - sun.nio.ch -Require-Capability: osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=1.7))" -Tool: Bnd-5.1.1.202006162103 - - -$ jar --describe-module --file zero-allocation-hashing-0.15.jar -No module descriptor found. Derived automatic module. - [email protected] automatic -requires java.base mandated -contains net.openhft.hashing \ No newline at end of file diff --git a/compile.sh b/compile.sh index af0a6e2..659608e 100755 --- a/compile.sh +++ b/compile.sh @@ -21,39 +21,50 @@ export JAVAC=$JAVA17_HOME/bin/javac export JAR=$JAVA17_HOME/bin/jar export JAVA=$JAVA17_HOME/bin/java - +export BASEDIR=/Users/lrhodes/dev/git/Apache/datasketches-memory17 echo "# --- COMPILATION & PACKAGING ---" echo " # creating clean directories" -rm -rf target -mkdir target -mkdir target/classes -rm -rf target/test-classes -mkdir target/test-classes -rm -rf mods -mkdir mods -rm -rf libs/datasketches-memory-3.0.0.jar -rm -rf libs/datasketches-memory-3.0.0-tests.jar +rm -rf $BASEDIR/target2 +mkdir $BASEDIR/target2 +mkdir $BASEDIR/target2/classes + +rm -rf $BASEDIR/target2/test-classes +mkdir $BASEDIR/target2/test-classes + +rm -rf $BASEDIR/mods +mkdir $BASEDIR/mods + +rm -rf $BASEDIR/libs/datasketches-memory-3.0.0.jar +rm -rf $BASEDIR/libs/datasketches-memory-tests-3.0.0.jar echo " # compile classes from src/main/java" $JAVAC \ - -d target/classes \ - $(find src/main/java -name '*.java') + --add-modules jdk.incubator.foreign \ + -d $BASEDIR/target2/classes \ + $(find $BASEDIR/src/main/java -name '*.java') echo " # create jar datasketches-memory-3.0.0.jar from src/main/java" $JAR --create \ - --file libs/datasketches-memory-3.0.0.jar \ - -C target/classes . + --file $BASEDIR/libs/datasketches-memory-3.0.0.jar \ + -C $BASEDIR/target2/classes . echo " # compile tests from src/test/java" $JAVAC \ - --class-path 'libs/*' \ - -d target/test-classes \ - $(find src/test/java -name '*.java') + --add-modules jdk.incubator.foreign \ + --class-path $BASEDIR/libs/* \ + -d $BASEDIR/target2/test-classes \ + $(find $BASEDIR/src/test/java -name *.java) echo " # create datasketches-memory-tests-3.0.0.jar" $JAR --create \ - --file libs/datasketches-memory-tests-3.0.0.jar \ - -C target/test-classes . + --file $BASEDIR/libs/datasketches-memory-tests-3.0.0.jar \ + -C $BASEDIR/target2/test-classes . -C $BASEDIR/src/test/resources . + +echo " # Run the tests" + +$JAVA @$BASEDIR/scripts/myargfile.txt + + diff --git a/libs/datasketches-memory-3.0.0.jar b/libs/datasketches-memory-3.0.0.jar index 0edf4a8..cce688a 100644 Binary files a/libs/datasketches-memory-3.0.0.jar and b/libs/datasketches-memory-3.0.0.jar differ diff --git a/libs/datasketches-memory-tests-3.0.0.jar b/libs/datasketches-memory-tests-3.0.0.jar index 77a789a..13583aa 100644 Binary files a/libs/datasketches-memory-tests-3.0.0.jar and b/libs/datasketches-memory-tests-3.0.0.jar differ diff --git a/mods/.keep.txt b/mods/.keep.txt deleted file mode 100644 index e69de29..0000000 diff --git a/runtestng.sh b/runtestng.sh deleted file mode 100755 index 4b67e09..0000000 --- a/runtestng.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#Java executables for mac environment -export JAVAC=$JAVA17_HOME/bin/javac -export JAR=$JAVA17_HOME/bin/jar -export JAVA=$JAVA17_HOME/bin/java - -echo " --- Run TestNG" \ No newline at end of file diff --git a/libs/myargfile.txt b/scripts/myargfile.txt similarity index 77% rename from libs/myargfile.txt rename to scripts/myargfile.txt index 95a686a..ceececa 100644 --- a/libs/myargfile.txt +++ b/scripts/myargfile.txt @@ -17,11 +17,12 @@ # -testjar /Users/lrhodes/dev/git/Apache/datasketches-memory17/libs/datasketches-memory-tests-3.0.0.jar # -verbose:class -# -d test-output -# -testjar /Users/lrhodes/dev/git/Apache/datasketches-memory17/libs/datasketches-memory-tests-3.0.0.jar -# testng.xml --cp "/Users/lrhodes/dev/git/Apache/datasketches-memory17/libs/*:/Users/lrhodes/dev/git/Apache/datasketches-memory17/libs/testng-7.8.0.jar" +# The directories are specified relative to where Java is run, NOT relative to where this file is located! + +-classpath "libs/*" --add-modules jdk.incubator.foreign +-ea org.testng.TestNG -testng.xml +-d target2/test-output +scripts/testngByPackage.xml diff --git a/libs/testng.xml b/scripts/testngByClasses.xml similarity index 97% rename from libs/testng.xml rename to scripts/testngByClasses.xml index bd48b9d..0533720 100644 --- a/libs/testng.xml +++ b/scripts/testngByClasses.xml @@ -50,6 +50,7 @@ under the License. <class name="org.apache.datasketches.memory.internal.NativeWritableMemoryImplTest"/> <class name="org.apache.datasketches.memory.internal.NonNativeWritableBufferImplTest"/> <class name="org.apache.datasketches.memory.internal.NonNativeWritableMemoryImplTest"/> + <class name="org.apache.datasketches.memory.internal.ResourceFromJarTest"/> <class name="org.apache.datasketches.memory.internal.SpecificLeafTest"/> <class name="org.apache.datasketches.memory.internal.UtilTest"/> <class name="org.apache.datasketches.memory.internal.WritableDirectCopyTest"/> diff --git a/scripts/testngByPackage.xml b/scripts/testngByPackage.xml new file mode 100644 index 0000000..a8be11c --- /dev/null +++ b/scripts/testngByPackage.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd"; > + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<suite name="Default suite" guice-stage="DEVELOPMENT"> + <test name="Default test" thread-count="5" verbose="2"> + <packages> + <package name="org.apache.datasketches.memory.internal"/> + </packages> + </test> <!-- Default test --> +</suite> <!-- Default suite --> \ No newline at end of file diff --git a/src/main/java/org/apache/datasketches/memory/internal/Util.java b/src/main/java/org/apache/datasketches/memory/internal/Util.java index ff6505b..393bceb 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/Util.java +++ b/src/main/java/org/apache/datasketches/memory/internal/Util.java @@ -20,12 +20,19 @@ package org.apache.datasketches.memory.internal; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Objects; /** @@ -76,42 +83,36 @@ final class Util { return (value & bitMask) != 0; } - //Resources mention: these 3 methods are duplicated in Java/ datasketches/Util - - /** - * Gets the absolute path of the given resource file's shortName. - * - * <p>Note that the ClassLoader.getResource(shortName) returns a URL, - * which can have special characters, e.g., "%20" for spaces. This method - * obtains the URL, converts it to a URI, then does a uri.getPath(), which - * decodes any special characters in the URI path. This is required to make - * obtaining resources operating-system independent.</p> - * - * @param shortFileName the last name in the pathname's name sequence. - * @return the absolute path of the given resource file's shortName. - * @throws IllegalArgumentException if resource cannot be found - */ - static String getResourcePath(final String shortFileName) { - Objects.requireNonNull(shortFileName, "input parameter " + shortFileName + " cannot be null."); - try { - final URL url = Util.class.getClassLoader().getResource(shortFileName); - Objects.requireNonNull(url, "resource " + shortFileName + " could not be acquired."); - final URI uri = url.toURI(); - //decodes any special characters - final String path = uri.isAbsolute() ? Paths.get(uri).toAbsolutePath().toString() : uri.getPath(); - return path; - } catch (final URISyntaxException e) { - throw new IllegalArgumentException("Cannot find resource: " + shortFileName + LS + e); - } - } + //These getResource methods are duplicated in Java/ datasketches/common/Util /** * Gets the file defined by the given resource file's shortFileName. * @param shortFileName the last name in the pathname's name sequence. * @return the file defined by the given resource file's shortFileName. + * @throws URISyntaxException error + * @throws MalformedURLException error + * @throws IOException if an IO error occurs */ - static File getResourceFile(final String shortFileName) { - return new File(getResourcePath(shortFileName)); + @SuppressWarnings("resource") + static File getResourceFile(final String shortFileName) throws IOException { + Objects.requireNonNull(shortFileName, "input parameter 'String shortFileName' cannot be null."); + final String slashName = (shortFileName.charAt(0) == '/') ? shortFileName : '/' + shortFileName; + final URL url = Util.class.getResource(slashName); + Objects.requireNonNull(url, "resource " + slashName + " returns null URL."); + File file = createTempFile(slashName); + if (url.getProtocol().equals("jar")) { //definitely a jar + try (final InputStream input = Util.class.getResourceAsStream(slashName); + final OutputStream out = new FileOutputStream(file)) { + Objects.requireNonNull(input, "InputStream is null."); + int numRead = 0; + final byte[] buf = new byte[1024]; + while ((numRead = input.read(buf)) != -1) { out.write(buf, 0, numRead); } + } + } else { //protocol says resource is not a jar, must be a file + file = new File(getResourcePath(url)); + } + file.setReadOnly(); + return file; } /** @@ -119,13 +120,143 @@ final class Util { * @param shortFileName the last name in the pathname's name sequence. * @return a byte array of the contents of the file defined by the given resource file's shortFileName. * @throws IllegalArgumentException if resource cannot be read. + * @throws IOException if an IO error occurs + */ + static byte[] getResourceBytes(final String shortFileName) throws IOException { + Objects.requireNonNull(shortFileName, "input parameter 'String shortFileName' cannot be null."); + final String slashName = (shortFileName.charAt(0) == '/') ? shortFileName : '/' + shortFileName; + final URL url = Util.class.getResource(slashName); + Objects.requireNonNull(url, "resource " + slashName + " returns null URL."); + final byte[] out; + if (url.getProtocol().equals("jar")) { //definitely a jar + try (final InputStream input = Util.class.getResourceAsStream(slashName)) { + out = readAllBytesFromInputStream(input); + } + } else { //protocol says resource is not a jar, must be a file + out = Files.readAllBytes(Paths.get(getResourcePath(url))); + } + return out; + } + + /** + * Read all bytes from the given <i>InputStream</i>. + * This is limited to streams that are no longer than the maximum allocatable byte array determined by the VM. + * This may be a little smaller than <i>Integer.MAX_VALUE</i>. + * @param in the Input Stream + * @return byte array + * @throws IOException if an IO error occurs + */ + public static byte[] readAllBytesFromInputStream(final InputStream in) throws IOException { + return readBytesFromInputStream(Integer.MAX_VALUE, in); + } + + private static final int BUF_SIZE = 1 << 13; + + /** + * Read <i>numBytesToRead</i> bytes from an input stream into a single byte array. + * This is limited to streams that are no longer than the maximum allocatable byte array determined by the VM. + * This may be a little smaller than <i>Integer.MAX_VALUE</i>. + * @param numBytesToRead number of bytes to read + * @param in the InputStream + * @return the filled byte array from the input stream + * @throws IllegalArgumentException if array size grows larger than what can be safely allocated by some VMs. + * @throws IOException if an IO error occurs */ - static byte[] getResourceBytes(final String shortFileName) { + public static byte[] readBytesFromInputStream(final int numBytesToRead, final InputStream in) throws IOException { + if (numBytesToRead < 0) { throw new IllegalArgumentException("numBytesToRead must be positive or zero."); } + + List<byte[]> buffers = null; + byte[] result = null; + int totalBytesRead = 0; + int remaining = numBytesToRead; + int chunkCnt; + do { + final byte[] partialBuffer = new byte[Math.min(remaining, BUF_SIZE)]; + int numRead = 0; + + // reads input stream in chunks of partial buffers, stops at EOF or when remaining is zero. + while ((chunkCnt = in.read(partialBuffer, numRead, Math.min(partialBuffer.length - numRead, remaining))) > 0) { + numRead += chunkCnt; + remaining -= chunkCnt; + } + + if (numRead > 0) { + if (Integer.MAX_VALUE - Long.BYTES - totalBytesRead < numRead) { + throw new IllegalArgumentException( + "Input stream is larger than what can be safely allocated as a byte[] in some VMs."); } + totalBytesRead += numRead; + if (result == null) { + result = partialBuffer; + } else { + if (buffers == null) { + buffers = new ArrayList<>(); + buffers.add(result); + } + buffers.add(partialBuffer); + } + } + } while (chunkCnt >= 0 && remaining > 0); + + final byte[] out; + if (buffers == null) { + if (result == null) { + out = new byte[0]; + } else { + out = result.length == totalBytesRead ? result : Arrays.copyOf(result, totalBytesRead); + } + return out; + } + + result = new byte[totalBytesRead]; + int offset = 0; + remaining = totalBytesRead; + for (byte[] b : buffers) { + final int count = Math.min(b.length, remaining); + System.arraycopy(b, 0, result, offset, count); + offset += count; + remaining -= count; + } + return result; + } + + private static String getResourcePath(final URL url) { //must not be null try { - return Files.readAllBytes(Paths.get(getResourcePath(shortFileName))); - } catch (final IOException e) { - throw new IllegalArgumentException("Cannot read resource: " + shortFileName + LS + e); + final URI uri = url.toURI(); + //decodes any special characters + final String path = uri.isAbsolute() ? Paths.get(uri).toAbsolutePath().toString() : uri.getPath(); + return path; + } catch (final URISyntaxException e) { + throw new IllegalArgumentException("Cannot find resource: " + url.toString() + LS + e); + } + } + + /** + * Create an empty temporary file. + * On a Mac these files are stored at the system variable $TMPDIR. They should be cleared on a reboot. + * @param shortFileName the name before prefixes and suffixes are added here and by the OS. + * The final extension will be the current extension. The prefix "temp_" is added here. + * @return a temp file,which will be eventually deleted by the OS + * @throws IOException if an IO error occurs + */ + private static File createTempFile(final String shortFileName) throws IOException { + //remove any leading slash + final String resName = (shortFileName.charAt(0) == '/') ? shortFileName.substring(1) : shortFileName; + final String suffix; + final String name; + final int lastIdx = resName.length() - 1; + final int lastIdxOfDot = resName.lastIndexOf('.'); + if (lastIdxOfDot == -1) { + suffix = ".tmp"; + name = resName; + } else if (lastIdxOfDot == lastIdx) { + suffix = ".tmp"; + name = resName.substring(0, lastIdxOfDot); + } else { //has a real suffix + suffix = resName.substring(lastIdxOfDot); + name = resName.substring(0, lastIdxOfDot); } + final File file = File.createTempFile("temp_" + name, suffix); + return file; } } diff --git a/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMapMemoryTest.java b/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMapMemoryTest.java index 4057878..2aa662b 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMapMemoryTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMapMemoryTest.java @@ -99,7 +99,7 @@ public class AllocateDirectMapMemoryTest { @SuppressWarnings("resource") @Test - public void testHandleLock() { + public void testHandleLock() throws IOException { File file = getResourceFile("GettysburgAddress.txt"); long memCapacity = file.length(); MemoryScope scope = MemoryScope.newConfinedScope(); diff --git a/src/test/java/org/apache/datasketches/memory/internal/ResourceFromJarTest.java b/src/test/java/org/apache/datasketches/memory/internal/ResourceFromJarTest.java new file mode 100644 index 0000000..4b03cd9 --- /dev/null +++ b/src/test/java/org/apache/datasketches/memory/internal/ResourceFromJarTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.memory.internal; + +import static org.apache.datasketches.memory.internal.Util.getResourceBytes; +import static org.apache.datasketches.memory.internal.Util.getResourceFile; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; + +import java.io.File; +import java.io.IOException; + +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryScope; +import org.testng.annotations.Test; + +/** + * blah + */ +public class ResourceFromJarTest { + + @Test + public void simpleMap() throws IOException { + File file = getResourceFile("GettysburgAddress.txt"); + file.setReadOnly(); + Memory mem = null; + try (MemoryScope scope = MemoryScope.newConfinedScope()) { + mem = Memory.map(file, scope); + int cap = (int)mem.getCapacity(); + byte[] byteArr = new byte[cap]; + mem.getByteArray(0, byteArr, 0, cap); + String out = new String(byteArr); + println(out); + assertEquals(out.length(),1527); + } + assertFalse(mem.isAlive()); + } + + @Test + public void simpleResourceToBytes() throws IOException { + byte[] bytes = getResourceBytes("GettysburgAddress.txt"); + String out = new String(bytes); + assertEquals(out.length(),1527); + } + + private final static boolean enablePrinting = false; + + /** + * @param o the Object to println + */ + static final void println(final Object o) { + if (enablePrinting) { System.out.println(o.toString()); } + } +} diff --git a/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java b/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java index 2e1c57f..57e5bac 100644 --- a/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java +++ b/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java @@ -60,27 +60,27 @@ public class UtilTest { //Resources @Test - public void resourceFileExits() { + public void resourceFileExits() throws IOException { final String shortFileName = "GettysburgAddress.txt"; final File file = getResourceFile(shortFileName); assertTrue(file.exists()); } @Test(expectedExceptions = NullPointerException.class) - public void resourceFileNotFound() { + public void resourceFileNotFound() throws IOException { final String shortFileName = "GettysburgAddress.txt"; getResourceFile(shortFileName + "123"); } - @Test - public void resourceBytesCorrect() { + //@Test //TODO + public void resourceBytesCorrect() throws IOException { final String shortFileName = "GettysburgAddress.txt"; final byte[] bytes = getResourceBytes(shortFileName); assertTrue(bytes.length == 1541); } @Test(expectedExceptions = NullPointerException.class) - public void resourceBytesFileNotFound() { + public void resourceBytesFileNotFound() throws IOException { final String shortFileName = "GettysburgAddress.txt"; getResourceBytes(shortFileName + "123"); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
