github-advanced-security[bot] commented on code in PR #16709: URL: https://github.com/apache/lucene/pull/16709#discussion_r4621685929
########## lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/segmentivf/IoUring.java: ########## @@ -0,0 +1,1023 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.sandbox.codecs.segmentivf; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.foreign.AddressLayout; +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.StructLayout; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.VarHandle; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Batched reads of a file through Linux io_uring, by raw syscalls, used to read a rerank's + * scattered fine codes off disk (see {@link FineCodeIO}). + * + * <p>A rerank reads a few hundred small records at random offsets. Read through a mapping, each + * page-cache miss is a synchronous fault, so misses run one after another. One io_uring submission + * issues every read at once, so misses overlap. An instance owns only its file descriptor. Rings + * are shared by every instance: a batch takes an idle ring and gives it back, at most {@link + * #MAX_IDLE} stay idle, and they are freed when the last instance closes. Reads land in a buffer + * the caller owns ({@link #buffer}). + * + * <p>A batch is {@link #submit submitted} without waiting and {@link #await awaited} later, + * possibly on another thread, so a search can issue reads and keep working while they complete; + * {@link #readBatch} does both at once. + * + * <p>Opened {@code direct}, reads bypass the page cache (O_DIRECT): a fine code is read once per + * query and rarely again, so caching it only evicts the doc values and postings that every query + * rereads. That needs page-aligned offsets, lengths and buffers. + * + * <p>This class only reads; it knows nothing of the index format. On other platforms, or when + * io_uring cannot be set up, {@link #open} fails and callers read through the mapping instead. + * + * <h2>How io_uring works, as used here</h2> + * + * <p>A ring is memory shared between this process and the kernel, mapped from the ring's file + * descriptor, holding three things: + * + * <pre> + * SQE array requests: "read len bytes of fd at off into addr, tagged user_data" (64 bytes each) + * SQ ring indices of SQEs ready for the kernel we move the tail, the kernel the head + * CQ ring one completion per request (CQE): the kernel moves the tail, we the head + * its user_data tag and res, the bytes read or -errno + * </pre> + * + * <p>A batch writes one SQE per read, publishes them by moving the SQ tail, and makes one {@code + * io_uring_enter} syscall to submit them all; the device then works on every read at once. It waits + * with {@code io_uring_enter} again, asking for the remaining count, and walks the CQEs between the + * CQ head and tail, moving the head past those it consumed. Head and tail are 32-bit counters that + * only grow, wrapping around; {@code counter & mask} is the slot. + * + * <p>There is no liburing: setup, mapping and both syscalls are made directly through the foreign + * function API, so the struct layouts below are the kernel's ABI ({@code + * include/uapi/linux/io_uring.h}) written out as byte offsets. Two rules carry the safety of the + * class: + * + * <ul> + * <li><b>Ordering.</b> The kernel reads SQEs once it sees the tail cover them, and writes a CQE + * before moving the tail past it; on a weakly ordered CPU such as aarch64, fences make each + * side's writes visible before the counter that publishes them ({@link Ring#submit}, {@link + * Ring#reap}). + * <li><b>Buffer lifetime.</b> Once submitted, a read may land in its buffer at any moment until + * its CQE is reaped. A buffer is reused or freed only after every read of its batch is + * reaped; when that cannot be known, after a failure, the buffer is kept reachable for good + * ({@link Ring#retire}). + * </ul> + */ +@SuppressWarnings("restricted") +final class IoUring implements Closeable { + /** The most reads one batch holds; a rerank's batch rarely needs more. */ + static final int RING_ENTRIES = 256; + + // Syscall numbers, which are the same on x86-64 and aarch64: syscalls added since Linux 5.1 get + // one number on every architecture. Package-private for tests. + static final long SYS_IO_URING_SETUP = 425; + static final long SYS_IO_URING_ENTER = 426; + + // mmap offsets of the submission ring, completion ring and submission entries: not file offsets + // but tags that tell mmap on a ring descriptor which of its regions to map + // (IORING_OFF_SQ_RING, IORING_OFF_CQ_RING, IORING_OFF_SQES). + private static final long OFF_SQ_RING = 0L; + private static final long OFF_CQ_RING = 0x8000000L; + private static final long OFF_SQES = 0x10000000L; + + /** io_uring_enter flag: wait until min_complete completions are available. */ + private static final int IORING_ENTER_GETEVENTS = 1; + + /** The opcode of a plain read at an offset, like pread (kernel 5.6+). */ + private static final int IORING_OP_READ = 22; + + /** Feature bit: the SQ and CQ rings share one mapping (kernel 5.4+). */ + private static final int IORING_FEAT_SINGLE_MMAP = 1; + + private static final int PROT_READ_WRITE = 1 | 2; + private static final int MAP_SHARED = 1; + private static final int O_RDONLY = 0; + // O_DIRECT's value differs by architecture; a wrong one fails the open, which then falls back. + private static final int O_DIRECT = + System.getProperty("os.arch", "").contains("aarch64") ? 0x10000 : 0x4000; + private static final int POSIX_FADV_RANDOM = 1; + + /** O_DIRECT's alignment unit for offsets, lengths and buffers. */ + static final int PAGE_BYTES = 4096; + + private static final int EINTR = 4; + + // struct io_uring_params, with its embedded io_sqring_offsets and io_cqring_offsets. setup + // fills it in: entry counts, features, and where each ring's head, tail, mask and arrays sit + // within its mapping (offsets from the mapping's start, so they are read, not assumed). + private static final int PARAMS_SIZE = 120; + private static final int PARAMS_SQ_ENTRIES = 0; + private static final int PARAMS_CQ_ENTRIES = 4; + private static final int PARAMS_FEATURES = 20; + private static final int SQ_OFF_TAIL = 44; + private static final int SQ_OFF_RING_MASK = 48; + private static final int SQ_OFF_ARRAY = 64; + private static final int CQ_OFF_HEAD = 80; + private static final int CQ_OFF_TAIL = 84; + private static final int CQ_OFF_RING_MASK = 88; + private static final int CQ_OFF_CQES = 100; + + // struct io_uring_sqe. Only these fields are written; the rest stay zero (flags, ioprio, + // rw_flags, buf_index...), which a plain read wants. The opcode is the first byte. + private static final int SQE_SIZE = 64; + private static final int SQE_FD = 4; + private static final int SQE_OFF = 8; + private static final int SQE_ADDR = 16; + private static final int SQE_LEN = 24; + private static final int SQE_USER_DATA = 32; + + // struct io_uring_cqe: the request's user_data, then res (bytes read, or -errno), then flags. + private static final int CQE_SIZE = 16; + private static final int CQE_USER_DATA = 0; + private static final int CQE_RES = 8; + + private static final boolean LINUX = + System.getProperty("os.name", "").toLowerCase(Locale.ROOT).startsWith("linux"); + + // Core: store.PosixNativeAccess does the libc lookup, but it's package-private. + /** libc bindings, in their own class so that other platforms never resolve them. */ + private static final class Native { + static final MethodHandle SYSCALL; + static final MethodHandle MMAP; + static final MethodHandle MUNMAP; + static final MethodHandle OPEN; + static final MethodHandle CLOSE; + static final MethodHandle FADVISE; + + /** + * The thread state that {@link #SYSCALL} saves right after the call, before the JVM can run + * anything that overwrites errno. + */ + static final StructLayout CALL_STATE = Linker.Option.captureStateLayout(); + + static final long ERRNO_OFFSET = + CALL_STATE.byteOffset(MemoryLayout.PathElement.groupElement("errno")); + + static { + ValueLayout.OfLong j = ValueLayout.JAVA_LONG; + ValueLayout.OfInt i = ValueLayout.JAVA_INT; + AddressLayout p = ValueLayout.ADDRESS; + // syscall(long, ...) and open(const char *, int, ...) are variadic. libc's syscall returns -1 + // and sets errno on failure, so it captures errno: the handle takes a CALL_STATE segment + // first. + SYSCALL = + libc( + "syscall", + FunctionDescriptor.of(j, j, j, j, j, j, j, j), + Linker.Option.firstVariadicArg(1), + Linker.Option.captureCallState("errno")); + MMAP = libc("mmap", FunctionDescriptor.of(p, p, j, i, i, i, j)); + MUNMAP = libc("munmap", FunctionDescriptor.of(i, p, j)); + OPEN = libc("open", FunctionDescriptor.of(i, p, i), Linker.Option.firstVariadicArg(2)); + CLOSE = libc("close", FunctionDescriptor.of(i, i)); + FADVISE = libc("posix_fadvise", FunctionDescriptor.of(i, i, j, j, i)); + } + + /** A downcall to {@code name}. */ + private static MethodHandle libc(String name, FunctionDescriptor fd, Linker.Option... options) { + Linker linker = Linker.nativeLinker(); + MemorySegment symbol = linker.defaultLookup().find(name).orElseThrow(); + return linker.downcallHandle(symbol, fd, options); + } + } + + /** + * At most this many rings are kept idle between batches; others are freed after use. Opening a + * reader sets the pool up to this many ({@link #open}), so searches take rings already made + * instead of setting one up, and a setup that fails does so at open, not mid-search. + */ + static final int MAX_IDLE = Runtime.getRuntime().availableProcessors(); + + private static final Queue<Ring> IDLE = new ConcurrentLinkedQueue<>(); + + /** How many rings {@link #IDLE} holds, kept apart because the queue's own count is a walk. */ + private static final AtomicInteger IDLE_COUNT = new AtomicInteger(); + + /** Open instances: the last to close frees the idle rings. */ + private static final AtomicInteger OPEN = new AtomicInteger(); + + /** Whether io_uring is off for the process, after {@link #MAX_FAILED_BATCHES} failed batches. */ + static boolean disabled() { + return FAILED_BATCHES.get() >= MAX_FAILED_BATCHES; + } + + /** + * Whether reads may still land in {@code buffer}: whether it overlaps a buffer a failed batch + * retired. A batch that failed otherwise (before submitting, or after reaping every read) leaves + * no read in flight, and its buffer may be reused. + */ + static boolean retired(MemorySegment buffer) { + long start = buffer.address(); + long end = start + buffer.byteSize(); + for (MemorySegment retired : RETIRED) { + if (retired.address() < end && start < retired.address() + retired.byteSize()) { + return true; + } + } + return false; + } + + /** Buffers that reads may still land in after a failure; see {@link Ring#retire}. */ + private static final Queue<MemorySegment> RETIRED = new ConcurrentLinkedQueue<>(); + + /** + * Batches that failed after submitting, each retiring a ring and the buffer its reads may still + * reach. Counted apart from {@link #RETIRED}, whose own count is a walk. + */ + private static final AtomicInteger FAILED_BATCHES = new AtomicInteger(); + + /** + * After this many failed batches io_uring is turned off for the process, so a persistent failure + * cannot keep retiring, and leaking, buffers. Callers then read through the mapping. + */ + private static final int MAX_FAILED_BATCHES = 64; + + /** + * How long after a ring fails to set up no new ring is tried; batches meanwhile fail at once and + * their callers read through the mapping. A setup failure leaks nothing, so it is not counted + * towards {@link #MAX_FAILED_BATCHES}: it is usually transient, a limit that frees as other + * searches' rings go idle or are freed. On kernels before 5.12 ring memory counts against + * RLIMIT_MEMLOCK, whose common 64 KiB default holds about two 256-entry rings, so concurrent + * searches meet it often; the pause keeps them from paying a failing syscall per batch. + */ + private static final long SETUP_BACKOFF_NANOS = 1_000_000_000L; + + /** {@link System#nanoTime} before which no new ring is tried. */ + private static volatile long setupRetryAt = System.nanoTime(); + + /** Set once a ring setup failure has been logged, which is logged once per process. */ + private static final AtomicBoolean SETUP_FAILURE_LOGGED = new AtomicBoolean(); + + /** + * A read the kernel completed but failed or cut short, or an open the file system refused: the + * file or kernel refused the read itself. Only this sets {@link #directRefused} or {@link + * #readsRefused}; a failure to set up or submit to a ring, such as one under memlock limits, is + * transient and refuses nothing for later opens. + */ + static final class ReadRefusedException extends IOException { + ReadRefusedException(String message) { + super(message); + } + } + + /** Set once a file system accepted O_DIRECT but refused the reads: later opens skip it. */ + private static volatile boolean directRefused; + + /** + * Set once a buffered probe read failed: kernels from 5.1 to 5.5 set up rings but refuse + * IORING_OP_READ, so every later open fails at once instead of probing again. + */ + private static volatile boolean readsRefused; + + private final int fd; + + /** The file's length when opened; Lucene files are written once, so it never changes. */ + private final long fileLength; + + /** + * Guards {@link #fd} against {@link #close}. A submission writes the descriptor's number into its + * entries and the kernel resolves it at io_uring_enter, so a close in between, and an open that + * reuses the number, would have the reads served from another file. Submissions hold the read + * lock until the kernel has taken their entries (from then on a read holds its own reference to + * the file); close takes the write lock. + */ + private final ReentrantReadWriteLock fdLock = new ReentrantReadWriteLock(); + + /** + * Set by the first {@link #close}, under {@link #fdLock}; later calls do nothing, and submissions + * fail. A second close would otherwise close {@link #fd} again, by then possibly another file's + * descriptor, and miscount {@link #OPEN}. + */ + private boolean closed; + + /** + * Whether the file is read uncached (O_DIRECT): every read is then whole pages at a page offset. + */ + final boolean direct; + + private IoUring(int fd, long fileLength, boolean direct) { + this.fd = fd; + this.fileLength = fileLength; + this.direct = direct; + } + + /** + * Opens {@code path} for batched reads, uncached when {@code direct} and the file system allows + * it, else through the page cache. Fails if a ring cannot be set up or cannot read. + */ + static IoUring open(Path path, boolean direct) throws IOException { + if (direct && directRefused == false) { + try { + return open(path, O_RDONLY | O_DIRECT); + } catch (ReadRefusedException _) { + // The file system refuses O_DIRECT: read through the page cache instead. Any other + // failure, such as no ring to be had right now, would fail a cached open too. + } + } + return open(path, O_RDONLY); + } + + private static IoUring open(Path path, int flags) throws IOException { + if (LINUX == false) { + throw new IOException("io_uring is unsupported on this platform"); + } + if (readsRefused) { + throw new IOException("io_uring reads are unsupported by this kernel"); + } + int fd; + try (Arena arena = Arena.ofConfined()) { + fd = (int) Native.OPEN.invokeExact(arena.allocateFrom(path.toString()), flags); + } catch (Throwable t) { + throw new IOException("open failed for " + path, t); + } + if (fd < 0) { + if ((flags & O_DIRECT) != 0) { + // Some file systems refuse O_DIRECT at open (EINVAL); a file that cannot be opened at all + // fails the cached open after it too. + throw new ReadRefusedException("O_DIRECT open failed for " + path); + } + throw new IOException("open failed for " + path); + } + long fileLength; + try { + fileLength = Files.size(path); + } catch (IOException e) { + closeFd(fd); + throw e; + } + // Each read is one code: no readahead. A no-op when the page cache is bypassed. + try { + int _ = (int) Native.FADVISE.invokeExact(fd, 0L, 0L, POSIX_FADV_RANDOM); + } catch (Throwable _) { + // Best effort: reads are correct either way. + } + OPEN.incrementAndGet(); + boolean direct = (flags & O_DIRECT) != 0; + IoUring ioUring = new IoUring(fd, fileLength, direct); + // Probe with one read: a file system may accept O_DIRECT and still refuse its reads, and an + // older kernel may set up a ring yet refuse every read. Only such a refusal is remembered for + // later opens; ring setup failing, say under memlock limits, fails only this open. + try { + ioUring.readBatch(new long[] {0}, direct ? PAGE_BYTES : 1, buffer(MemorySegment.NULL, 1)); + } catch (Throwable t) { + if (t instanceof ReadRefusedException) { + if (direct) { + directRefused = true; + } else { + readsRefused = true; + } + } + try { + ioUring.close(); + } catch (IOException suppressed) { + t.addSuppressed(suppressed); + } + throw t; + } + fillIdle(); + return ioUring; + } + + /** + * Sets up idle rings until {@link #MAX_IDLE} are idle; a no-op once they are, as for every open + * after the first. Stops at the first ring that cannot be set up: batches then set up their own, + * as without this. + */ + private static void fillIdle() { + while (IDLE_COUNT.get() < MAX_IDLE) { + Ring ring; + try { + ring = Ring.create(); + } catch (RuntimeException _) { + return; + } + IDLE_COUNT.incrementAndGet(); + IDLE.add(ring); + } + } + + /** + * Returns {@code current} if it holds at least {@code size} bytes, else a new buffer that does, + * page-aligned as O_DIRECT reads require. It is freed once unreachable. + */ + static MemorySegment buffer(MemorySegment current, long size) { + if (current.byteSize() >= size) { + return current; + } + return Arena.ofAuto().allocate(Math.max(size, PAGE_BYTES), PAGE_BYTES); + } + + /** + * Reads {@code length} bytes at every position into {@code buffer} ({@link #buffer}), read {@code + * k} landing at {@code k * length}. After a failure, reads may still land in {@code buffer}, so + * the caller must not reuse it. + */ + MemorySegment readBatch(long[] positions, int length, MemorySegment buffer) throws IOException { + return readBatch(positions, positions.length, length, buffer); + } + + /** + * Reads the first {@code n} of {@code positions}, as {@link #readBatch(long[], int, + * MemorySegment)}: every chunk is submitted before any is awaited, so all the reads are in flight + * at once, and whatever is thrown every submitted chunk is awaited before this returns. + */ + MemorySegment readBatch(long[] positions, int n, int length, MemorySegment buffer) + throws IOException { + read(this, null, positions, n, length, buffer); + return buffer; + } + + /** + * {@link #readBatch(long[], int, int, MemorySegment)} over several files: read {@code k} reads + * {@code files[k]}. One ring can read any number of files, each submission entry naming its own, + * so a rerank spanning many segments costs one submission and one wait, not one of each per + * segment. + */ + static void readBatch(IoUring[] files, long[] positions, int n, int length, MemorySegment buffer) + throws IOException { + read(null, files, positions, n, length, buffer); + } + + /** + * Reads {@code n} positions of {@code file}, or, when it is null, read {@code k} of {@code + * files[k]}: every chunk is submitted before any is awaited, so all the reads are in flight at + * once, and whatever is thrown every submitted chunk is awaited before this returns. Each file is + * locked against {@link #close} until its reads are submitted. + */ + private static void read( + IoUring file, IoUring[] files, long[] positions, int n, int length, MemorySegment buffer) + throws IOException { + if (buffer.byteSize() < (long) n * length) { + throw new IllegalArgumentException("buffer of " + buffer.byteSize() + " bytes is too small"); + } + if (FAILED_BATCHES.get() >= MAX_FAILED_BATCHES) { + throw new IOException("io_uring disabled after " + MAX_FAILED_BATCHES + " failed batches"); + } + // The distinct files, each locked once. + List<IoUring> locked = new ArrayList<>(); + int[] fds = null; + Batch[] batches = new Batch[(n + RING_ENTRIES - 1) / RING_ENTRIES]; + int submitted = 0; + Throwable failure = null; + try { + if (file != null) { + lock(file, locked); + } else { + fds = new int[n]; + for (int k = 0; k < n; k++) { + if (locked.contains(files[k]) == false) { + lock(files[k], locked); + } + fds[k] = files[k].fd; + } + } + for (; submitted < batches.length; submitted++) { + int from = submitted * RING_ENTRIES; + int count = Math.min(RING_ENTRIES, n - from); + batches[submitted] = submit(file, fds, files, positions, from, count, length, buffer); + } + } catch (Throwable t) { + failure = t; + } finally { + // Submitted reads hold the files themselves; the locks only guarded their descriptors. + for (IoUring f : locked) { + f.fdLock.readLock().unlock(); + } + } + for (int b = 0; b < submitted; b++) { + try { + await(batches[b]); + } catch (Throwable t) { + if (failure == null) { + failure = t; + } else { + failure.addSuppressed(t); + } + } + } + if (failure == null) { + return; + } + if (failure instanceof IOException e) { + throw e; + } + if (failure instanceof RuntimeException e) { + throw e; + } + throw (Error) failure; + } + + /** Locks {@code file}'s descriptor against close, failing if it is closed already. */ + private static void lock(IoUring file, List<IoUring> locked) throws IOException { + file.fdLock.readLock().lock(); Review Comment: ## CodeQL / Unreleased lock This lock might not be unlocked or might be locked more times than it is unlocked. [Show more details](https://github.com/apache/lucene/security/code-scanning/273) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
