github-actions[bot] commented on code in PR #69619:
URL: https://github.com/apache/doris/pull/69619#discussion_r4597245596


##########
be/src/io/cache/block_file_cache.cpp:
##########
@@ -1969,6 +1975,154 @@ void BlockFileCache::remove(FileBlockSPtr file_block, 
T& cache_lock, U& block_lo
     *_num_removed_blocks << 1;
 }
 
+void BlockFileCache::run_storage_remove(const UInt128Wrapper& hash, size_t 
offset,
+                                        bvar::LatencyRecorder* latency_us) 
const {
+    FileCacheKey key;
+    bool again = _pending_storage_removes.claim(hash, offset, &key);
+    while (again) {
+        TEST_SYNC_POINT_CALLBACK("BlockFileCache::run_storage_remove", &key);
+        int64_t duration_ns = 0;
+        Status st;
+        {
+            SCOPED_RAW_TIMER(&duration_ns);
+            st = _storage->remove(key);
+        }
+        if (latency_us != nullptr) {
+            *latency_us << (duration_ns / 1000);
+        }
+        if (!st.ok()) {
+            LOG_WARNING("failed to remove file cache block, hash={} offset={}",
+                        key.hash.to_string(), key.offset)
+                    .error(st);
+        }
+        again = _pending_storage_removes.finish(hash, offset) &&
+                _pending_storage_removes.claim(hash, offset, &key);
+    }
+}
+
+void BlockFileCache::drain_pending_storage_remove(const UInt128Wrapper& hash, 
size_t offset) const {
+    FileCacheKey key;
+    while (_pending_storage_removes.wait_and_claim(hash, offset, &key)) {
+        
TEST_SYNC_POINT_CALLBACK("BlockFileCache::drain_pending_storage_remove", &key);
+        Status st = _storage->remove(key);
+        if (!st.ok()) {
+            LOG_WARNING("failed to remove file cache block, hash={} offset={}",
+                        key.hash.to_string(), key.offset)
+                    .error(st);
+        }
+        _pending_storage_removes.finish(hash, offset);
+    }
+}
+
+PendingStorageRemoves::Shard& PendingStorageRemoves::shard_of(const 
FileWriterMapKey& key) const {
+    return _shards[FileWriterMapKeyHash {}(key) % kShardCount];
+}
+
+void PendingStorageRemoves::mark(const FileCacheKey& key) {
+    FileWriterMapKey map_key {key.hash, key.offset};
+    auto& shard = shard_of(map_key);
+    std::lock_guard lock(shard.mtx);
+    auto [it, inserted] = shard.entries.try_emplace(map_key);
+    if (!inserted && it->second.running) {
+        it->second.marked_while_running = true;
+    }
+    it->second.key = key;
+}
+
+bool PendingStorageRemoves::claim(const UInt128Wrapper& hash, size_t offset, 
FileCacheKey* key) {
+    FileWriterMapKey map_key {hash, offset};
+    auto& shard = shard_of(map_key);
+    std::lock_guard lock(shard.mtx);
+    auto it = shard.entries.find(map_key);
+    if (it == shard.entries.end() || it->second.running) {
+        return false;
+    }
+    it->second.running = true;
+    *key = it->second.key;
+    return true;
+}
+
+bool PendingStorageRemoves::wait_and_claim(const UInt128Wrapper& hash, size_t 
offset,
+                                           FileCacheKey* key) {
+    FileWriterMapKey map_key {hash, offset};
+    auto& shard = shard_of(map_key);
+    std::unique_lock lock(shard.mtx);
+    while (true) {
+        auto it = shard.entries.find(map_key);
+        if (it == shard.entries.end()) {
+            return false;
+        }
+        if (!it->second.running) {
+            it->second.running = true;
+            *key = it->second.key;
+            return true;
+        }
+        shard.cv.wait(lock);
+    }
+}
+
+bool PendingStorageRemoves::finish(const UInt128Wrapper& hash, size_t offset) {
+    FileWriterMapKey map_key {hash, offset};
+    auto& shard = shard_of(map_key);
+    bool again = false;
+    {
+        std::lock_guard lock(shard.mtx);
+        auto it = shard.entries.find(map_key);
+        DCHECK(it != shard.entries.end() && it->second.running);
+        if (it != shard.entries.end()) {
+            if (it->second.marked_while_running) {
+                it->second.running = false;
+                it->second.marked_while_running = false;
+                again = true;
+            } else {
+                shard.entries.erase(it);
+            }
+        }
+    }
+    shard.cv.notify_all();
+    return again;
+}
+
+bool PendingStorageRemoves::contains(const UInt128Wrapper& hash, size_t 
offset) const {
+    FileWriterMapKey map_key {hash, offset};
+    auto& shard = shard_of(map_key);
+    std::lock_guard lock(shard.mtx);
+    return shard.entries.contains(map_key);
+}
+
+size_t PendingStorageRemoves::size() const {
+    size_t total = 0;
+    for (auto& shard : _shards) {
+        std::lock_guard lock(shard.mtx);
+        total += shard.entries.size();
+    }
+    return total;
+}
+
+namespace {
+thread_local DeferredStorageRemoveScope* tls_deferred_storage_remove_scope = 
nullptr;
+} // namespace
+
+DeferredStorageRemoveScope::DeferredStorageRemoveScope(const BlockFileCache* 
cache)
+        : _cache(cache), _prev(tls_deferred_storage_remove_scope) {
+    tls_deferred_storage_remove_scope = this;
+}
+
+DeferredStorageRemoveScope::~DeferredStorageRemoveScope() {
+    // Scopes nest on one thread; nothing under the cache lock may switch 
threads (bthreads do not
+    // migrate while blocked on a std::mutex or in a syscall), so this scope 
is still innermost.
+    DCHECK(tls_deferred_storage_remove_scope == this);
+    tls_deferred_storage_remove_scope = _prev;
+    for (const auto& key : _keys) {
+        _cache->run_storage_remove(key.hash, key.offset,

Review Comment:
   [P2] Wait for competing removal owners in synchronous scopes
   
   This calls `run_storage_remove()`, whose `claim()` returns false immediately 
if another thread is already removing the position. After the cache mutex is 
released, a replacement downloader can claim the entry via 
`drain_pending_storage_remove()` before this destructor runs; GC can also win 
using a queued record left after an earlier downloader drain. The synchronous 
evictor then returns while that owner is still before unlink. Consequently 
`remove_if_cached`/synchronous clear can finish with the victim still present, 
and a reactive reservation on a full disk can reach its new append before the 
space is reclaimed and fail with ENOSPC. Synchronous scopes should wait for or 
drain an existing owner; keep nonblocking claiming for GC. Please cover the 
reverse ownership order in a deterministic test.



##########
be/test/io/cache/block_file_cache_evict_io_test.cpp:
##########
@@ -0,0 +1,565 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Storage removal of evicted file cache blocks: it must run outside the cache 
lock, must never
+// delete a block re-created at the same position, and must treat missing 
files as removed.
+
+#include <atomic>
+#include <future>
+
+#include "io/cache/block_file_cache_test_common.h"
+#include "util/defer_op.h"
+
+namespace doris::io {
+
+namespace {
+
+constexpr size_t kBlockSize = 10;
+
+std::string evict_io_cache_path(const std::string& name) {
+    return (caches_dir / ("evict_io_" + name) / "").string();
+}
+
+FileCacheSettings normal_only_settings(size_t capacity, size_t block_size) {
+    FileCacheSettings settings;
+    settings.index_queue_elements = 0;
+    settings.index_queue_size = 0;
+    settings.disposable_queue_size = 0;
+    settings.disposable_queue_elements = 0;
+    settings.ttl_queue_size = 0;
+    settings.ttl_queue_elements = 0;
+    settings.query_queue_size = capacity;
+    settings.query_queue_elements = capacity / block_size + 1;
+    settings.max_file_block_size = block_size;
+    settings.max_query_cache_size = capacity;
+    settings.capacity = capacity;
+    return settings;
+}
+
+std::string block_path(const std::string& base, const UInt128Wrapper& hash, 
size_t offset) {
+    auto key_str = hash.to_string();
+    return (fs::path(base) / key_str.substr(0, 3) / (key_str + "_0") / 
std::to_string(offset))
+            .string();
+}
+
+// Inserts [offset, offset + size) of hash, downloads it with `fill` and 
releases the holder.
+void insert_block(BlockFileCache& cache, const UInt128Wrapper& hash, size_t 
offset, size_t size,
+                  char fill = '0') {
+    CacheContext context;
+    ReadStatistics rstats;
+    context.stats = &rstats;
+    context.cache_type = FileCacheType::NORMAL;
+    auto holder = cache.get_or_set(hash, offset, size, context);
+    for (auto& block : holder.file_blocks) {
+        if (block->state() != FileBlock::State::EMPTY ||
+            block->get_or_set_downloader() != FileBlock::get_caller_id()) {
+            continue;
+        }
+        std::string data(block->range().size(), fill);
+        ASSERT_TRUE(block->append(Slice(data.data(), data.size())).ok());
+        ASSERT_TRUE(block->finalize().ok());
+    }
+}
+
+// Whether `mutex` is held by any thread. try_lock from the owning thread is 
undefined, so the
+// probe runs on its own thread.
+bool is_locked_elsewhere(std::mutex& mutex) {
+    bool locked = false;
+    std::thread probe([&] {
+        if (mutex.try_lock()) {

Review Comment:
   [P2] Isolate background cache workers from the lock probe
   
   `cache.initialize()` starts monitor and GC threads, and the shared fixture 
leaves them running. Both `run_background_monitor()` and 
`recycle_deleting_blocks()` acquire the same cache mutex independently of the 
evictor. If either owns it when this probe calls `try_lock()`, the removal 
callback counts that as removal under the lock and the zero-count assertions 
fail even though the evictor released it correctly. The stalled-lookup test 
uses the same observation. Pause/coordinate unrelated workers or use a 
handshake that establishes the evictor released the mutex, so these new tests 
do not fail on valid background scheduling.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to