This is an automated email from the ASF dual-hosted git repository.
SYaoJun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-graphar.git
The following commit(s) were added to refs/heads/main by this push:
new 277d6669 feat(java): let the storage API reach the local filesystem
(#960)
277d6669 is described below
commit 277d6669f9c495afe0806324bb1c3d496c724ae5
Author: alex <[email protected]>
AuthorDate: Mon Aug 24 17:06:13 2026 +0300
feat(java): let the storage API reach the local filesystem (#960)
* feat(java): let the storage API reach the local filesystem
The storage API merged in #958 has no implementation, so nothing can
open a GraphAr file yet. This adds the local filesystem adapter that
the same issue asked for, keeping the module free of any GraphAr
layout, format, or query concern: it resolves a URI to a path, opens a
seekable input over it, and writes through a position-reporting output.
Directories are created on demand for an output file, an existing
target is refused unless the caller asked to replace it, and a read
past the end of a file reports the end of the stream rather than a
partial buffer.
Closes the adapter half of #953.
Not-tested: only the file scheme is exercised; other schemes are the
concern of later adapters.
* feat(java): reuse one transfer buffer for direct-buffer writes
Review feedback on #960: a write from a buffer without a backing array
allocated a fresh staging array on every call. The output is single
threaded already, because it tracks its own position, so one instance
buffer can serve every such write. It is allocated on first use, so an
output that only ever sees heap buffers never pays for it.
Adds the missing test for a direct buffer larger than one transfer
chunk, which the previous test did not reach: with the copy loop
reduced to a single pass the new test fails with
"expected:<20000> but was:<8192>".
---
maven-projects/pom.xml | 1 +
maven-projects/storage-local/pom.xml | 90 +++++++++++++
.../graphar/storage/local/LocalInputFile.java | 50 +++++++
.../graphar/storage/local/LocalOutputFile.java | 67 +++++++++
.../graphar/storage/local/LocalPositionOutput.java | 77 +++++++++++
.../graphar/storage/local/LocalSeekableInput.java | 58 ++++++++
.../apache/graphar/storage/local/LocalStorage.java | 56 ++++++++
.../graphar/storage/local/LocalStorageTest.java | 149 +++++++++++++++++++++
8 files changed, 548 insertions(+)
diff --git a/maven-projects/pom.xml b/maven-projects/pom.xml
index 205ea123..81df9024 100644
--- a/maven-projects/pom.xml
+++ b/maven-projects/pom.xml
@@ -79,6 +79,7 @@
<module>spark</module>
<module>info</module>
<module>storage-api</module>
+ <module>storage-local</module>
</modules>
<build>
diff --git a/maven-projects/storage-local/pom.xml
b/maven-projects/storage-local/pom.xml
new file mode 100644
index 00000000..f3df2e4a
--- /dev/null
+++ b/maven-projects/storage-local/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.graphar</groupId>
+ <artifactId>graphar-root</artifactId>
+ <version>${graphar.version}</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>graphar-storage-local</artifactId>
+ <packaging>jar</packaging>
+ <version>${graphar.version}</version>
+
+ <name>graphar-storage-local</name>
+
+ <properties>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.graphar</groupId>
+ <artifactId>graphar-storage-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.13.2</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <version>${spotless-maven-plugin.version}</version>
+ <configuration>
+ <java>
+ <googleJavaFormat>
+ <version>1.7</version>
+ <style>AOSP</style>
+ </googleJavaFormat>
+ </java>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalInputFile.java
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalInputFile.java
new file mode 100644
index 00000000..bb730602
--- /dev/null
+++
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalInputFile.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.storage.local;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.graphar.storage.InputFile;
+import org.apache.graphar.storage.SeekableInput;
+
+final class LocalInputFile implements InputFile {
+ private final Path path;
+
+ LocalInputFile(Path path) {
+ this.path = path;
+ }
+
+ @Override
+ public URI uri() {
+ return path.toUri();
+ }
+
+ @Override
+ public long size() throws IOException {
+ return Files.size(path);
+ }
+
+ @Override
+ public SeekableInput open() throws IOException {
+ return new LocalSeekableInput(path);
+ }
+}
diff --git
a/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalOutputFile.java
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalOutputFile.java
new file mode 100644
index 00000000..d42986fe
--- /dev/null
+++
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalOutputFile.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.storage.local;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import org.apache.graphar.storage.OutputFile;
+import org.apache.graphar.storage.PositionOutput;
+
+final class LocalOutputFile implements OutputFile {
+ private final Path path;
+
+ LocalOutputFile(Path path) {
+ this.path = path;
+ }
+
+ @Override
+ public URI uri() {
+ return path.toUri();
+ }
+
+ @Override
+ public PositionOutput create() throws IOException {
+ createParentDirectories();
+ return new LocalPositionOutput(
+ Files.newOutputStream(
+ path, StandardOpenOption.CREATE_NEW,
StandardOpenOption.WRITE));
+ }
+
+ @Override
+ public PositionOutput createOrOverwrite() throws IOException {
+ createParentDirectories();
+ return new LocalPositionOutput(
+ Files.newOutputStream(
+ path,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE));
+ }
+
+ private void createParentDirectories() throws IOException {
+ Path parent = path.getParent();
+ if (parent != null) {
+ Files.createDirectories(parent);
+ }
+ }
+}
diff --git
a/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalPositionOutput.java
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalPositionOutput.java
new file mode 100644
index 00000000..7fc039ff
--- /dev/null
+++
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalPositionOutput.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.storage.local;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import org.apache.graphar.storage.PositionOutput;
+
+final class LocalPositionOutput implements PositionOutput {
+ private static final int BUFFER_SIZE = 8192;
+
+ private final OutputStream output;
+ private long position;
+ private byte[] transfer;
+
+ LocalPositionOutput(OutputStream output) {
+ this.output = output;
+ }
+
+ @Override
+ public long position() {
+ return position;
+ }
+
+ @Override
+ public void write(ByteBuffer source) throws IOException {
+ if (source.hasArray()) {
+ int length = source.remaining();
+ write(source.array(), source.arrayOffset() + source.position(),
length);
+ source.position(source.position() + length);
+ return;
+ }
+
+ if (transfer == null) {
+ transfer = new byte[BUFFER_SIZE];
+ }
+ while (source.hasRemaining()) {
+ int length = Math.min(source.remaining(), transfer.length);
+ source.get(transfer, 0, length);
+ write(transfer, 0, length);
+ }
+ }
+
+ @Override
+ public void write(byte[] source, int offset, int length) throws
IOException {
+ output.write(source, offset, length);
+ position += length;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ output.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ output.close();
+ }
+}
diff --git
a/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalSeekableInput.java
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalSeekableInput.java
new file mode 100644
index 00000000..9078fce9
--- /dev/null
+++
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalSeekableInput.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.storage.local;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import org.apache.graphar.storage.SeekableInput;
+
+final class LocalSeekableInput implements SeekableInput {
+ private final FileChannel channel;
+
+ LocalSeekableInput(Path path) throws IOException {
+ this.channel = FileChannel.open(path, StandardOpenOption.READ);
+ }
+
+ @Override
+ public long position() throws IOException {
+ return channel.position();
+ }
+
+ @Override
+ public void seek(long newPosition) throws IOException {
+ if (newPosition < 0) {
+ throw new IllegalArgumentException("Seek position cannot be
negative: " + newPosition);
+ }
+ channel.position(newPosition);
+ }
+
+ @Override
+ public int read(ByteBuffer destination) throws IOException {
+ return channel.read(destination);
+ }
+
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+}
diff --git
a/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalStorage.java
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalStorage.java
new file mode 100644
index 00000000..839442e0
--- /dev/null
+++
b/maven-projects/storage-local/src/main/java/org/apache/graphar/storage/local/LocalStorage.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.storage.local;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.graphar.storage.InputFile;
+import org.apache.graphar.storage.OutputFile;
+import org.apache.graphar.storage.Storage;
+
+/** Storage backed by the local filesystem. */
+public final class LocalStorage implements Storage {
+ @Override
+ public InputFile inputFile(URI uri) {
+ return new LocalInputFile(toPath(uri));
+ }
+
+ @Override
+ public OutputFile outputFile(URI uri) {
+ return new LocalOutputFile(toPath(uri));
+ }
+
+ @Override
+ public boolean exists(URI uri) throws IOException {
+ return Files.exists(toPath(uri));
+ }
+
+ private static Path toPath(URI uri) {
+ if (uri == null) {
+ throw new IllegalArgumentException("Storage URI cannot be null.");
+ }
+ if (!"file".equalsIgnoreCase(uri.getScheme())) {
+ throw new IllegalArgumentException("LocalStorage only supports
file URIs: " + uri);
+ }
+ return Path.of(uri);
+ }
+}
diff --git
a/maven-projects/storage-local/src/test/java/org/apache/graphar/storage/local/LocalStorageTest.java
b/maven-projects/storage-local/src/test/java/org/apache/graphar/storage/local/LocalStorageTest.java
new file mode 100644
index 00000000..cc04dcb2
--- /dev/null
+++
b/maven-projects/storage-local/src/test/java/org/apache/graphar/storage/local/LocalStorageTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.graphar.storage.local;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.graphar.storage.InputFile;
+import org.apache.graphar.storage.OutputFile;
+import org.apache.graphar.storage.PositionOutput;
+import org.apache.graphar.storage.SeekableInput;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class LocalStorageTest {
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private final LocalStorage storage = new LocalStorage();
+
+ @Test
+ public void writesReadsAndSeeksAcrossNestedFile() throws IOException {
+ Path path =
temporaryFolder.getRoot().toPath().resolve("nested/data.bin");
+ OutputFile outputFile = storage.outputFile(path.toUri());
+ assertFalse(storage.exists(path.toUri()));
+
+ try (PositionOutput output = outputFile.create()) {
+ output.write("ab".getBytes(UTF_8));
+ ByteBuffer direct = ByteBuffer.allocateDirect(2);
+ direct.put("cd".getBytes(UTF_8));
+ direct.flip();
+ output.write(direct);
+ ByteBuffer heap = ByteBuffer.wrap("00efgh".getBytes(UTF_8));
+ heap.position(2);
+ heap.limit(6);
+ ByteBuffer slicedHeap = heap.slice();
+ slicedHeap.position(1);
+ slicedHeap.limit(3);
+ output.write(slicedHeap);
+ output.flush();
+ assertEquals(6, output.position());
+ assertEquals(2, direct.position());
+ assertEquals(3, slicedHeap.position());
+ }
+
+ assertTrue(storage.exists(path.toUri()));
+ InputFile inputFile = storage.inputFile(path.toUri());
+ assertEquals(path.toUri(), inputFile.uri());
+ assertEquals(6, inputFile.size());
+ try (SeekableInput input = inputFile.open()) {
+ ByteBuffer allBytes = ByteBuffer.allocate(6);
+ input.readFully(allBytes);
+ assertEquals(6, input.position());
+ assertArrayEquals("abcdfg".getBytes(UTF_8), allBytes.array());
+
+ input.seek(1);
+ ByteBuffer suffix = ByteBuffer.allocate(2);
+ input.readFully(suffix);
+ assertArrayEquals("bc".getBytes(UTF_8), suffix.array());
+ assertEquals(3, input.position());
+ }
+ }
+
+ @Test
+ public void createDoesNotOverwriteAndCreateOrOverwriteTruncates() throws
IOException {
+ Path path = temporaryFolder.newFile("existing.bin").toPath();
+ Files.write(path, "old".getBytes(UTF_8));
+ OutputFile outputFile = storage.outputFile(path.toUri());
+
+ assertThrows(IOException.class, outputFile::create);
+ assertArrayEquals("old".getBytes(UTF_8), Files.readAllBytes(path));
+
+ try (PositionOutput output = outputFile.createOrOverwrite()) {
+ output.write("new".getBytes(UTF_8));
+ }
+ assertArrayEquals("new".getBytes(UTF_8), Files.readAllBytes(path));
+ }
+
+ @Test
+ public void rejectsNonFileUrisAndNegativeSeekPositions() throws
IOException {
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> storage.inputFile(URI.create("s3://bucket/a")));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> storage.inputFile(URI.create("relative/path")));
+
+ Path path = temporaryFolder.newFile("data.bin").toPath();
+ try (SeekableInput input = storage.inputFile(path.toUri()).open()) {
+ assertThrows(IllegalArgumentException.class, () -> input.seek(-1));
+ }
+ }
+
+ @Test
+ public void readFullyFailsWhenTheFileIsShorterThanTheDestination() throws
IOException {
+ Path path = temporaryFolder.newFile("short.bin").toPath();
+ Files.write(path, "x".getBytes(UTF_8));
+
+ try (SeekableInput input = storage.inputFile(path.toUri()).open()) {
+ assertThrows(IOException.class, () ->
input.readFully(ByteBuffer.allocate(2)));
+ }
+ }
+
+ @Test
+ public void writesADirectBufferLargerThanOneTransferChunk() throws
IOException {
+ Path path = temporaryFolder.getRoot().toPath().resolve("large.bin");
+ byte[] expected = new byte[20_000];
+ for (int index = 0; index < expected.length; index++) {
+ expected[index] = (byte) index;
+ }
+ ByteBuffer direct = ByteBuffer.allocateDirect(expected.length);
+ direct.put(expected);
+ direct.flip();
+
+ try (PositionOutput output =
storage.outputFile(path.toUri()).create()) {
+ output.write(direct);
+ assertEquals(expected.length, output.position());
+ }
+
+ assertFalse(direct.hasRemaining());
+ assertArrayEquals(expected, Files.readAllBytes(path));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
