jkorous updated this revision to Diff 187860. jkorous marked 2 inline comments as done. jkorous added a comment.
- handle EINTR - assert we are reading whole INotifyEvents CHANGES SINCE LAST ACTION https://reviews.llvm.org/D58418/new/ https://reviews.llvm.org/D58418 Files: clang/include/clang/DirectoryWatcher/DirectoryWatcher.h clang/lib/CMakeLists.txt clang/lib/DirectoryWatcher/CMakeLists.txt clang/lib/DirectoryWatcher/DirectoryWatcher-linux.inc.h clang/lib/DirectoryWatcher/DirectoryWatcher-mac.inc.h clang/lib/DirectoryWatcher/DirectoryWatcher.cpp clang/unittests/CMakeLists.txt clang/unittests/DirectoryWatcher/CMakeLists.txt clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
Index: clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp =================================================================== --- /dev/null +++ clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp @@ -0,0 +1,332 @@ +//===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "gtest/gtest.h" +#include <condition_variable> +#include <mutex> +#include <thread> + +using namespace llvm; +using namespace llvm::sys; +using namespace llvm::sys::fs; +using namespace clang; + +namespace { + +class EventCollection { + SmallVector<DirectoryWatcher::Event, 6> Events; + +public: + EventCollection() = default; + explicit EventCollection(ArrayRef<DirectoryWatcher::Event> events) { + append(events); + } + + void append(ArrayRef<DirectoryWatcher::Event> events) { + Events.append(events.begin(), events.end()); + } + + bool empty() const { return Events.empty(); } + size_t size() const { return Events.size(); } + void clear() { Events.clear(); } + + bool hasEvents(ArrayRef<StringRef> filenames, + ArrayRef<DirectoryWatcher::EventKind> kinds, + ArrayRef<file_status> stats) const { + assert(filenames.size() == kinds.size()); + assert(filenames.size() == stats.size()); + SmallVector<DirectoryWatcher::Event, 6> evts = Events; + bool hadError = false; + for (unsigned i = 0, e = filenames.size(); i < e; ++i) { + StringRef fname = filenames[i]; + DirectoryWatcher::EventKind kind = kinds[i]; + file_status stat = stats[i]; + auto it = std::find_if(evts.begin(), evts.end(), + [&](const DirectoryWatcher::Event &evt) -> bool { + return path::filename(evt.Filename) == fname; + }); + if (it == evts.end()) { + hadError = err(Twine("expected filename '" + fname + "' not found")); + continue; + } + if (it->Kind != kind) { + hadError = err(Twine("filename '" + fname + "' has event kind " + + std::to_string((int)it->Kind) + ", expected ") + + std::to_string((int)kind)); + } + if (it->Kind != DirectoryWatcher::EventKind::Removed && + it->ModTime != stat.getLastModificationTime()) + hadError = + err(Twine("filename '" + fname + "' has different mod time")); + evts.erase(it); + } + for (const auto &evt : evts) { + hadError = err(Twine("unexpected filename '" + + path::filename(evt.Filename) + "' found")); + } + return !hadError; + } + + bool hasAdded(ArrayRef<StringRef> filenames, + ArrayRef<file_status> stats) const { + std::vector<DirectoryWatcher::EventKind> kinds{ + filenames.size(), DirectoryWatcher::EventKind::Added}; + return hasEvents(filenames, kinds, stats); + } + + bool hasRemoved(ArrayRef<StringRef> filenames) const { + std::vector<DirectoryWatcher::EventKind> kinds{ + filenames.size(), DirectoryWatcher::EventKind::Removed}; + std::vector<file_status> stats{filenames.size(), file_status{}}; + return hasEvents(filenames, kinds, stats); + } + +private: + bool err(Twine msg) const { + SmallString<128> buf; + llvm::errs() << msg.toStringRef(buf) << '\n'; + return true; + } +}; + +struct EventOccurrence { + std::vector<DirectoryWatcher::Event> Events; + bool IsInitial; +}; + +class DirectoryWatcherTest + : public std::enable_shared_from_this<DirectoryWatcherTest> { + std::string WatchedDir; + std::string TempDir; + std::unique_ptr<DirectoryWatcher> DirWatcher; + + std::condition_variable Condition; + std::mutex Mutex; + std::deque<EventOccurrence> EvtOccurs; + +public: + void init() { + SmallString<128> pathBuf; + std::error_code EC = createUniqueDirectory("dirwatcher", pathBuf); + ASSERT_FALSE(EC); + TempDir = pathBuf.str(); + path::append(pathBuf, "watch"); + WatchedDir = pathBuf.str(); + EC = create_directory(WatchedDir); + ASSERT_FALSE(EC); + } + + ~DirectoryWatcherTest() { + stopWatching(); + remove_directories(TempDir); + } + +public: + StringRef getWatchedDir() const { return WatchedDir; } + + void addFile(StringRef filename, file_status &stat) { + SmallString<128> pathBuf; + pathBuf = TempDir; + path::append(pathBuf, filename); + Expected<file_t> ft = + openNativeFileForWrite(pathBuf, CD_CreateNew, OF_None); + ASSERT_TRUE((bool)ft); + closeFile(*ft); + + SmallString<128> newPath; + newPath = WatchedDir; + path::append(newPath, filename); + std::error_code EC = rename(pathBuf, newPath); + ASSERT_FALSE(EC); + + EC = status(newPath, stat); + ASSERT_FALSE(EC); + } + + void addFiles(ArrayRef<StringRef> filenames, + std::vector<file_status> &stats) { + for (auto fname : filenames) { + file_status stat; + addFile(fname, stat); + stats.push_back(stat); + } + } + + void addFiles(ArrayRef<StringRef> filenames) { + std::vector<file_status> stats; + addFiles(filenames, stats); + } + + void removeFile(StringRef filename) { + SmallString<128> pathBuf; + pathBuf = WatchedDir; + path::append(pathBuf, filename); + std::error_code EC = remove(pathBuf, /*IgnoreNonExisting=*/false); + ASSERT_FALSE(EC); + } + + void removeFiles(ArrayRef<StringRef> filenames) { + for (auto fname : filenames) { + removeFile(fname); + } + } + + /// \returns true for error. + bool startWatching(bool waitInitialSync) { + std::weak_ptr<DirectoryWatcherTest> weakThis = shared_from_this(); + auto receiver = [weakThis](ArrayRef<DirectoryWatcher::Event> events, + bool isInitial) { + if (auto this_ = weakThis.lock()) + this_->onEvents(events, isInitial); + }; + std::string error; + DirWatcher = DirectoryWatcher::create(getWatchedDir(), receiver, + waitInitialSync, error); + return DirWatcher == nullptr; + } + + void stopWatching() { DirWatcher.reset(); } + + /// \returns None if the timeout is reached before getting an event. + Optional<EventOccurrence> getNextEvent(unsigned timeout_seconds = 5) { + std::unique_lock<std::mutex> lck(Mutex); + auto pred = [&]() -> bool { return !EvtOccurs.empty(); }; + bool gotEvent = + Condition.wait_for(lck, std::chrono::seconds(timeout_seconds), pred); + if (!gotEvent) + return None; + + EventOccurrence occur = EvtOccurs.front(); + EvtOccurs.pop_front(); + return occur; + } + + EventOccurrence getNextEventImmediately() { + std::lock_guard<std::mutex> LG(Mutex); + assert(!EvtOccurs.empty()); + EventOccurrence occur = EvtOccurs.front(); + EvtOccurs.pop_front(); + return occur; + } + +private: + void onEvents(ArrayRef<DirectoryWatcher::Event> events, bool isInitial) { + std::lock_guard<std::mutex> LG(Mutex); + EvtOccurs.push_back({events, isInitial}); + Condition.notify_all(); + } +}; + +} // namespace + +TEST(DirectoryWatcherTest, initialScan) { + auto t = std::make_shared<DirectoryWatcherTest>(); + t->init(); + + std::vector<StringRef> fnames = {"a", "b", "c"}; + std::vector<file_status> stats; + t->addFiles(fnames, stats); + + bool err = t->startWatching(/*waitInitialSync=*/true); + ASSERT_FALSE(err); + + auto evt = t->getNextEventImmediately(); + EXPECT_TRUE(evt.IsInitial); + EventCollection coll1{evt.Events}; + EXPECT_TRUE(coll1.hasAdded(fnames, stats)); + + StringRef additionalFname = "d"; + file_status additionalStat; + t->addFile(additionalFname, additionalStat); + auto evtOpt = t->getNextEvent(); + ASSERT_TRUE(evtOpt.hasValue()); + EXPECT_FALSE(evtOpt->IsInitial); + EventCollection coll2{evtOpt->Events}; + EXPECT_TRUE(coll2.hasAdded({additionalFname}, {additionalStat})); +} + +TEST(DirectoryWatcherTest, fileEvents) { + auto t = std::make_shared<DirectoryWatcherTest>(); + t->init(); + + bool err = t->startWatching(/*waitInitialSync=*/false); + ASSERT_FALSE(err); + + auto evt = t->getNextEvent(); + ASSERT_TRUE(evt.hasValue()); + EXPECT_TRUE(evt->IsInitial); + EXPECT_TRUE(evt->Events.empty()); + return; + + { + std::vector<StringRef> fnames = {"a", "b"}; + std::vector<file_status> stats; + t->addFiles(fnames, stats); + + EventCollection coll{}; + while (coll.size() < 2) { + evt = t->getNextEvent(); + ASSERT_TRUE(evt.hasValue()); + coll.append(evt->Events); + } + EXPECT_TRUE(coll.hasAdded(fnames, stats)); + } + { + std::vector<StringRef> fnames = {"b", "c"}; + std::vector<file_status> stats; + t->addFiles(fnames, stats); + + EventCollection coll{}; + while (coll.size() < 2) { + evt = t->getNextEvent(); + ASSERT_TRUE(evt.hasValue()); + coll.append(evt->Events); + } + EXPECT_TRUE(coll.hasAdded(fnames, stats)); + } + { + std::vector<StringRef> fnames = {"a", "c"}; + std::vector<file_status> stats; + t->addFiles(fnames, stats); + t->removeFile("b"); + + EventCollection coll{}; + while (coll.size() < 3) { + evt = t->getNextEvent(); + ASSERT_TRUE(evt.hasValue()); + coll.append(evt->Events); + } + + EXPECT_TRUE(coll.hasEvents(std::vector<StringRef>{"a", "b", "c"}, + std::vector<DirectoryWatcher::EventKind>{ + DirectoryWatcher::EventKind::Added, + DirectoryWatcher::EventKind::Removed, + DirectoryWatcher::EventKind::Added, + }, + std::vector<file_status>{ + stats[0], + file_status{}, + stats[1], + })); + } + { + std::vector<StringRef> fnames = {"a", "c"}; + t->removeFiles(fnames); + + EventCollection coll{}; + while (coll.size() < 2) { + evt = t->getNextEvent(); + ASSERT_TRUE(evt.hasValue()); + coll.append(evt->Events); + } + EXPECT_TRUE(coll.hasRemoved(fnames)); + } +} Index: clang/unittests/DirectoryWatcher/CMakeLists.txt =================================================================== --- /dev/null +++ clang/unittests/DirectoryWatcher/CMakeLists.txt @@ -0,0 +1,22 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_unittest(DirectoryWatcherTests + DirectoryWatcherTest.cpp + ) + +target_link_libraries(DirectoryWatcherTests + PRIVATE + clangDirectoryWatcher + clangBasic + ) + +if(APPLE) + check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES_H) + if(HAVE_CORESERVICES_H) + set(DWT_LINK_FLAGS "${DWT_LINK_FLAGS} -framework CoreServices") + set_property(TARGET DirectoryWatcherTests APPEND_STRING PROPERTY + LINK_FLAGS ${DWT_LINK_FLAGS}) + endif() +endif() Index: clang/unittests/CMakeLists.txt =================================================================== --- clang/unittests/CMakeLists.txt +++ clang/unittests/CMakeLists.txt @@ -30,5 +30,6 @@ if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) add_subdirectory(libclang) endif() +add_subdirectory(DirectoryWatcher) add_subdirectory(Rename) add_subdirectory(Index) Index: clang/lib/DirectoryWatcher/DirectoryWatcher.cpp =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/DirectoryWatcher.cpp @@ -0,0 +1,152 @@ +//===- DirectoryWatcher.cpp - Listens for directory file changes ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// \brief Utility class for listening for file system changes in a directory. +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; +using namespace llvm; + +static Optional<sys::fs::file_status> getFileStatus(StringRef path) { + sys::fs::file_status Status; + std::error_code EC = status(path, Status); + if (EC) + return None; + return Status; +} + +namespace llvm { +// Specialize DenseMapInfo for sys::fs::UniqueID. +template <> struct DenseMapInfo<sys::fs::UniqueID> { + static sys::fs::UniqueID getEmptyKey() { + return sys::fs::UniqueID{DenseMapInfo<uint64_t>::getEmptyKey(), + DenseMapInfo<uint64_t>::getEmptyKey()}; + } + + static sys::fs::UniqueID getTombstoneKey() { + return sys::fs::UniqueID{DenseMapInfo<uint64_t>::getTombstoneKey(), + DenseMapInfo<uint64_t>::getEmptyKey()}; + } + + static unsigned getHashValue(const sys::fs::UniqueID &val) { + return DenseMapInfo<std::pair<uint64_t, uint64_t>>::getHashValue( + std::make_pair(val.getDevice(), val.getFile())); + } + + static bool isEqual(const sys::fs::UniqueID &LHS, + const sys::fs::UniqueID &RHS) { + return LHS == RHS; + } +}; +} // namespace llvm + +namespace { +/// Used for initial directory scan. +/// +/// Note that the caller must ensure serial access to it. It is not thread safe +/// to access it without additional protection. +struct DirectoryScan { + DenseSet<sys::fs::UniqueID> FileIDSet; + std::vector<std::tuple<std::string, sys::TimePoint<>>> Files; + + void scanDirectory(StringRef Path) { + using namespace llvm::sys; + + std::error_code EC; + for (auto It = fs::directory_iterator(Path, EC), + End = fs::directory_iterator(); + !EC && It != End; It.increment(EC)) { + auto status = getFileStatus(It->path()); + if (!status.hasValue()) + continue; + Files.push_back( + std::make_tuple(It->path(), status->getLastModificationTime())); + FileIDSet.insert(status->getUniqueID()); + } + } + + std::vector<DirectoryWatcher::Event> getAsFileEvents() const { + std::vector<DirectoryWatcher::Event> Events; + for (const auto &info : Files) { + DirectoryWatcher::Event Event{DirectoryWatcher::EventKind::Added, + std::get<0>(info), std::get<1>(info)}; + Events.push_back(std::move(Event)); + } + return Events; + } +}; +} // namespace + +// Add platform-specific functionality. + +#if !defined(__has_include) +#define __has_include(x) 0 +#endif + +#if __has_include(<CoreServices/FSEvents.h>) +#include "DirectoryWatcher-mac.inc.h" +#elif __has_include(<sys/inotify.h>) +#include "DirectoryWatcher-linux.inc.h" +#else + +struct DirectoryWatcher::Implementation { + bool initialize(StringRef Path, EventReceiver Receiver, bool waitInitialSync, + std::string &Error) { + Error = "directory listening not supported for this platform"; + return true; + } +}; + +#endif + +DirectoryWatcher::DirectoryWatcher() : Impl(*new Implementation()) {} + +DirectoryWatcher::~DirectoryWatcher() { delete &Impl; } + +std::unique_ptr<DirectoryWatcher> +DirectoryWatcher::create(StringRef Path, EventReceiver Receiver, + bool waitInitialSync, std::string &Error) { + using namespace llvm::sys; + + if (!fs::exists(Path)) { + std::error_code EC = fs::create_directories(Path); + if (EC) { + Error = EC.message(); + return nullptr; + } + } + + bool IsDir; + std::error_code EC = fs::is_directory(Path, IsDir); + if (EC) { + Error = EC.message(); + return nullptr; + } + if (!IsDir) { + Error = "path is not a directory: "; + Error += Path; + return nullptr; + } + + std::unique_ptr<DirectoryWatcher> DirWatch; + DirWatch.reset(new DirectoryWatcher()); + auto &Impl = DirWatch->Impl; + bool hasError = + Impl.initialize(Path, std::move(Receiver), waitInitialSync, Error); + if (hasError) + return nullptr; + + return DirWatch; +} Index: clang/lib/DirectoryWatcher/DirectoryWatcher-mac.inc.h =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/DirectoryWatcher-mac.inc.h @@ -0,0 +1,213 @@ +//===- DirectoryWatcher-mac.inc.h - Mac-platform directory listening ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <CoreServices/CoreServices.h> + +struct DirectoryWatcher::Implementation { + bool initialize(StringRef Path, EventReceiver Receiver, bool waitInitialSync, + std::string &Error); + ~Implementation() { stopFSEventStream(); }; + +private: + FSEventStreamRef EventStream = nullptr; + + bool setupFSEventStream(StringRef path, EventReceiver receiver, + dispatch_queue_t queue, + std::shared_ptr<DirectoryScan> initialScanPtr); + void stopFSEventStream(); +}; + +namespace { +struct EventStreamContextData { + std::string WatchedPath; + DirectoryWatcher::EventReceiver Receiver; + std::shared_ptr<DirectoryScan> InitialScan; + + EventStreamContextData(std::string watchedPath, + DirectoryWatcher::EventReceiver receiver, + std::shared_ptr<DirectoryScan> initialScanPtr) + : WatchedPath(std::move(watchedPath)), Receiver(std::move(receiver)), + InitialScan(std::move(initialScanPtr)) {} + + static void dispose(const void *ctx) { + delete static_cast<const EventStreamContextData *>(ctx); + } +}; +} // namespace + +static void eventStreamCallback(ConstFSEventStreamRef stream, + void *clientCallBackInfo, size_t numEvents, + void *eventPaths, + const FSEventStreamEventFlags eventFlags[], + const FSEventStreamEventId eventIds[]) { + auto *ctx = static_cast<EventStreamContextData *>(clientCallBackInfo); + + std::vector<DirectoryWatcher::Event> Events; + for (size_t i = 0; i < numEvents; ++i) { + StringRef path = ((const char **)eventPaths)[i]; + const FSEventStreamEventFlags flags = eventFlags[i]; + if (!(flags & kFSEventStreamEventFlagItemIsFile)) { + if ((flags & kFSEventStreamEventFlagItemRemoved) && + path == ctx->WatchedPath) { + DirectoryWatcher::Event Evt{ + DirectoryWatcher::EventKind::DirectoryDeleted, path, + llvm::sys::TimePoint<>{}}; + Events.push_back(Evt); + break; + } + continue; + } + DirectoryWatcher::EventKind K = DirectoryWatcher::EventKind::Modified; + bool hasAddedFlag = flags & (kFSEventStreamEventFlagItemCreated | + kFSEventStreamEventFlagItemRenamed); + bool hasRemovedFlag = flags & kFSEventStreamEventFlagItemRemoved; + Optional<sys::fs::file_status> statusOpt; + // NOTE: With low latency sometimes for a file that is moved inside the + // directory, or for a file that is removed from the directory, the flags + // have both 'renamed' and 'removed'. We use getting the file status as a + // way to distinguish between the two. + if (hasAddedFlag) { + statusOpt = getFileStatus(path); + if (statusOpt.hasValue()) { + K = DirectoryWatcher::EventKind::Added; + } else { + K = DirectoryWatcher::EventKind::Removed; + } + } else if (hasRemovedFlag) { + K = DirectoryWatcher::EventKind::Removed; + } else { + statusOpt = getFileStatus(path); + if (!statusOpt.hasValue()) { + K = DirectoryWatcher::EventKind::Removed; + } + } + + if (ctx->InitialScan && K == DirectoryWatcher::EventKind::Added) { + // For the first time we get the events, check that we haven't already + // sent the 'added' event at the initial scan. + if (ctx->InitialScan->FileIDSet.count(statusOpt->getUniqueID())) { + // Already reported this event at the initial directory scan. + continue; + } + } + + llvm::sys::TimePoint<> modTime{}; + if (statusOpt.hasValue()) + modTime = statusOpt->getLastModificationTime(); + DirectoryWatcher::Event Evt{K, path, modTime}; + Events.push_back(Evt); + } + + // We won't need to check again later on. + ctx->InitialScan.reset(); + + if (!Events.empty()) { + ctx->Receiver(Events, /*isInitial=*/false); + } +} + +bool DirectoryWatcher::Implementation::setupFSEventStream( + StringRef path, EventReceiver receiver, dispatch_queue_t queue, + std::shared_ptr<DirectoryScan> initialScanPtr) { + if (path.empty()) + return true; + + CFMutableArrayRef pathsToWatch = + CFArrayCreateMutable(nullptr, 0, &kCFTypeArrayCallBacks); + CFStringRef cfPathStr = + CFStringCreateWithBytes(nullptr, (const UInt8 *)path.data(), path.size(), + kCFStringEncodingUTF8, false); + CFArrayAppendValue(pathsToWatch, cfPathStr); + CFRelease(cfPathStr); + CFAbsoluteTime latency = 0.0; // Latency in seconds. + + std::string realPath; + { + SmallString<128> Storage; + StringRef P = llvm::Twine(path).toNullTerminatedStringRef(Storage); + char Buffer[PATH_MAX]; + // Use ::realpath to get the real path name + if (::realpath(P.begin(), Buffer) != nullptr) + realPath = Buffer; + else + realPath = path; + } + + EventStreamContextData *ctxData = new EventStreamContextData( + std::move(realPath), std::move(receiver), std::move(initialScanPtr)); + FSEventStreamContext context; + context.version = 0; + context.info = ctxData; + context.retain = nullptr; + context.release = EventStreamContextData::dispose; + context.copyDescription = nullptr; + + EventStream = FSEventStreamCreate( + nullptr, eventStreamCallback, &context, pathsToWatch, + kFSEventStreamEventIdSinceNow, latency, + kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagNoDefer); + CFRelease(pathsToWatch); + if (!EventStream) { + return true; + } + FSEventStreamSetDispatchQueue(EventStream, queue); + FSEventStreamStart(EventStream); + return false; +} + +void DirectoryWatcher::Implementation::stopFSEventStream() { + if (!EventStream) + return; + FSEventStreamStop(EventStream); + FSEventStreamInvalidate(EventStream); + FSEventStreamRelease(EventStream); + EventStream = nullptr; +} + +bool DirectoryWatcher::Implementation::initialize(StringRef Path, + EventReceiver Receiver, + bool waitInitialSync, + std::string &Error) { + auto initialScan = std::make_shared<DirectoryScan>(); + + dispatch_queue_t queue = + dispatch_queue_create("DirectoryWatcher", DISPATCH_QUEUE_SERIAL); + dispatch_semaphore_t initScanSema = dispatch_semaphore_create(0); + dispatch_semaphore_t setupFSEventsSema = dispatch_semaphore_create(0); + + std::string copiedPath = Path; + dispatch_retain(initScanSema); + dispatch_retain(setupFSEventsSema); + dispatch_async(queue, ^{ + // Wait for the event stream to be setup before doing the initial scan, + // to make sure we won't miss any events. + dispatch_semaphore_wait(setupFSEventsSema, DISPATCH_TIME_FOREVER); + initialScan->scanDirectory(copiedPath); + Receiver(initialScan->getAsFileEvents(), /*isInitial=*/true); + dispatch_semaphore_signal(initScanSema); + dispatch_release(setupFSEventsSema); + dispatch_release(initScanSema); + }); + bool fsErr = setupFSEventStream(Path, Receiver, queue, initialScan); + dispatch_semaphore_signal(setupFSEventsSema); + + if (waitInitialSync) { + dispatch_semaphore_wait(initScanSema, DISPATCH_TIME_FOREVER); + } + dispatch_release(setupFSEventsSema); + dispatch_release(initScanSema); + dispatch_release(queue); + + if (fsErr) { + raw_string_ostream(Error) + << "failed to setup FSEvents stream for path: " << Path; + return true; + } + + return false; +} Index: clang/lib/DirectoryWatcher/DirectoryWatcher-linux.inc.h =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/DirectoryWatcher-linux.inc.h @@ -0,0 +1,197 @@ +//===- DirectoryWatcher-linux.inc.h - Linux-platform directory listening --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/Errno.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/Path.h" +#include <sys/inotify.h> +#include <thread> +#include <unistd.h> + +namespace { + +struct INotifyEvent { + DirectoryWatcher::EventKind K; + std::string Filename; + Optional<sys::fs::file_status> Status; +}; + +class EventQueue { + DirectoryWatcher::EventReceiver Receiver; + sys::Mutex Mtx; + bool gotInitialScan = false; + std::vector<INotifyEvent> PendingEvents; + + DirectoryWatcher::Event toDirEvent(const INotifyEvent &evt) { + llvm::sys::TimePoint<> modTime{}; + if (evt.Status.hasValue()) + modTime = evt.Status->getLastModificationTime(); + return DirectoryWatcher::Event{evt.K, evt.Filename, modTime}; + } + +public: + explicit EventQueue(DirectoryWatcher::EventReceiver receiver) + : Receiver(receiver) {} + + void onDirectoryEvents(ArrayRef<INotifyEvent> evts) { + sys::ScopedLock L(Mtx); + + if (!gotInitialScan) { + PendingEvents.insert(PendingEvents.end(), evts.begin(), evts.end()); + return; + } + + SmallVector<DirectoryWatcher::Event, 8> dirEvents; + for (const auto &evt : evts) { + dirEvents.push_back(toDirEvent(evt)); + } + Receiver(dirEvents, /*isInitial=*/false); + } + + void onInitialScan(std::shared_ptr<DirectoryScan> dirScan) { + sys::ScopedLock L(Mtx); + + std::vector<DirectoryWatcher::Event> events = dirScan->getAsFileEvents(); + Receiver(events, /*isInitial=*/true); + + events.clear(); + for (const auto &evt : PendingEvents) { + if (evt.K == DirectoryWatcher::EventKind::Added && + dirScan->FileIDSet.count(evt.Status->getUniqueID())) { + // Already reported this event at the initial directory scan. + continue; + } + events.push_back(toDirEvent(evt)); + } + if (!events.empty()) { + Receiver(events, /*isInitial=*/false); + } + + gotInitialScan = true; + PendingEvents.clear(); + } +}; +} // namespace + +struct DirectoryWatcher::Implementation { + bool initialize(StringRef Path, EventReceiver Receiver, bool waitInitialSync, + std::string &Error); + ~Implementation() { stopListening(); }; + +private: + int inotifyFD = -1; + + void stopListening(); +}; + +static void runWatcher(std::string pathToWatch, int inotifyFD, + std::shared_ptr<EventQueue> evtQueue) { +#define EVT_BUF_LEN (30 * (sizeof(struct inotify_event) + NAME_MAX + 1)) + char buf[EVT_BUF_LEN] __attribute__((aligned(8))); + + while (1) { + ssize_t numRead = read(inotifyFD, buf, EVT_BUF_LEN); + if (numRead == -1) { + if (errno == EINTR) + continue; + return; // watcher is stopped. + } + + SmallVector<INotifyEvent, 8> iEvents; + for (char *p = buf; p < buf + numRead;) { + assert(p + sizeof(struct inotify_event) <= buf + numRead && "a whole inotify_event was read"); + struct inotify_event *ievt = dynamic_cast<struct inotify_event *>(p); + p += sizeof(struct inotify_event) + ievt->len; + + if (ievt->mask & IN_DELETE_SELF) { + INotifyEvent iEvt{DirectoryWatcher::EventKind::DirectoryDeleted, + pathToWatch, None}; + iEvents.push_back(iEvt); + break; + } + + DirectoryWatcher::EventKind K = DirectoryWatcher::EventKind::Added; + if (ievt->mask & IN_MODIFY) { + K = DirectoryWatcher::EventKind::Modified; + } + if (ievt->mask & IN_MOVED_TO) { + K = DirectoryWatcher::EventKind::Added; + } + if (ievt->mask & IN_DELETE) { + K = DirectoryWatcher::EventKind::Removed; + } + + assert(ievt->len > 0 && "expected a filename from inotify"); + SmallString<256> fullPath{pathToWatch}; + sys::path::append(fullPath, ievt->name); + + Optional<sys::fs::file_status> statusOpt; + if (K != DirectoryWatcher::EventKind::Removed) { + statusOpt = getFileStatus(fullPath); + if (!statusOpt.hasValue()) + K = DirectoryWatcher::EventKind::Removed; + } + INotifyEvent iEvt{K, fullPath.str(), statusOpt}; + iEvents.push_back(iEvt); + } + + if (!iEvents.empty()) + evtQueue->onDirectoryEvents(iEvents); + } +} + +bool DirectoryWatcher::Implementation::initialize(StringRef Path, + EventReceiver Receiver, + bool waitInitialSync, + std::string &errorMsg) { + auto error = [&](StringRef msg) -> bool { + errorMsg = msg; + errorMsg += ": "; + errorMsg += llvm::sys::StrError(); + return true; + }; + + auto evtQueue = std::make_shared<EventQueue>(std::move(Receiver)); + + inotifyFD = inotify_init(); + if (inotifyFD == -1) + return error("inotify_init failed"); + + std::string pathToWatch = Path; + int wd = inotify_add_watch(inotifyFD, pathToWatch.c_str(), + IN_MOVED_TO | IN_DELETE | IN_MODIFY | + IN_DELETE_SELF | IN_ONLYDIR); + if (wd == -1) + return error("inotify_add_watch failed"); + + std::thread watchThread( + std::bind(runWatcher, pathToWatch, inotifyFD, evtQueue)); + watchThread.detach(); + + auto initialScan = std::make_shared<DirectoryScan>(); + auto runScan = [pathToWatch, initialScan, evtQueue]() { + initialScan->scanDirectory(pathToWatch); + evtQueue->onInitialScan(std::move(initialScan)); + }; + + if (waitInitialSync) { + runScan(); + } else { + std::thread scanThread(runScan); + scanThread.detach(); + } + + return false; +} + +void DirectoryWatcher::Implementation::stopListening() { + if (inotifyFD == -1) + return; + close(inotifyFD); + inotifyFD = -1; +} Index: clang/lib/DirectoryWatcher/CMakeLists.txt =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/CMakeLists.txt @@ -0,0 +1,18 @@ +include(CheckIncludeFiles) + +set(LLVM_LINK_COMPONENTS support) + +add_clang_library(clangDirectoryWatcher + DirectoryWatcher.cpp + ) + +if(BUILD_SHARED_LIBS) + if(APPLE) + check_include_files("CoreServices/FSEvents.h" HAVE_CORESERVICES_H) + if(HAVE_CORESERVICES_H) + set(DIRECTORY_WATCHER_FLAGS "${DIRECTORY_WATCHER_FLAGS} -framework CoreServices") + endif() + set_property(TARGET clangDirectoryWatcher APPEND_STRING PROPERTY + LINK_FLAGS ${DIRECTORY_WATCHER_FLAGS}) + endif() +endif() Index: clang/lib/CMakeLists.txt =================================================================== --- clang/lib/CMakeLists.txt +++ clang/lib/CMakeLists.txt @@ -18,6 +18,7 @@ add_subdirectory(Frontend) add_subdirectory(FrontendTool) add_subdirectory(Tooling) +add_subdirectory(DirectoryWatcher) add_subdirectory(Index) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(StaticAnalyzer) Index: clang/include/clang/DirectoryWatcher/DirectoryWatcher.h =================================================================== --- /dev/null +++ clang/include/clang/DirectoryWatcher/DirectoryWatcher.h @@ -0,0 +1,77 @@ +//===- DirectoryWatcher.h - Listens for directory file changes --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// \brief Utility class for listening for file system changes in a directory. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H +#define LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H + +#include "clang/Basic/LLVM.h" +#include "llvm/Support/Chrono.h" +#include <functional> +#include <memory> +#include <string> + +namespace clang { + +/// Provides notifications for file system changes in a directory. +/// +/// Guarantees that the first time the directory is processed, the receiver will +/// be invoked even if the directory is empty. +class DirectoryWatcher { +public: + enum class EventKind { + /// A file was added to the directory. + /// + /// A file gets moved into the directory and replaces an existing file + /// with the same name will trigger an 'Added' event but no 'Removed' event. + /// If a file gets replaced multiple times within a short time period, it + /// may result in only one 'Added' event due to coalescing by the file + /// system notification mechanism. + Added, + /// A file was removed. + /// + /// A file that got replaced by another one with the same name will result + /// in a single 'Added' event, not a 'Removed' one. + Removed, + /// A file was modified. + Modified, + /// The watched directory got deleted. No more events will follow. + DirectoryDeleted, + }; + + struct Event { + EventKind Kind; + std::string Filename; + llvm::sys::TimePoint<> ModTime; + }; + + typedef std::function<void(ArrayRef<Event> Events, bool isInitial)> + EventReceiver; + + ~DirectoryWatcher(); + + static std::unique_ptr<DirectoryWatcher> create(StringRef Path, + EventReceiver Receiver, + bool waitInitialSync, + std::string &Error); + +private: + struct Implementation; + Implementation &Impl; + + DirectoryWatcher(); + + DirectoryWatcher(const DirectoryWatcher &) = delete; + DirectoryWatcher &operator=(const DirectoryWatcher &) = delete; +}; + +} // namespace clang + +#endif // LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
