I'm attaching a set of patches that I think includes a resolution of
this bug.
I've not yet done more than very basic testing but I'm expecting to
start running this in live from tomorrow backported to bookworm.
The patches are as follows:
fix-race.patch - supplied previously but acng is unusable without this.
fuse3-debian.patch - the changes to the debian directory to resolve this bug.
fuse3.patch - the changes to the source to resolve this bug.
deprecation-debian.patch - fix to resolve a deprecation warning in the debian
directory
deprecation.patch - fix to remove OpenSSL deprecation warnings plus extra
runtime tests for SHA256 and SHA512 correctness
buildhelper.patch - trivial change to use cmake if available and generate
compile-commands.json
I also spotted some deprecation warnings for ares but I'm not familiar
with that library so I couldn't quickly fix them. I might provide
patches later.
Tim.
diff -urN apt-cacher-ng-3.7.5.orig/CMakeLists.txt apt-cacher-ng-3.7.5/CMakeLists.txt
--- apt-cacher-ng-3.7.5.orig/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
@@ -81,6 +81,12 @@
# also influence it, need to be adjusted carefully by user.
SET(ACNG_COMPFLAGS "-D_FILE_OFFSET_BITS=64")
+find_program(CCACHE_FOUND ccache)
+if(CCACHE_FOUND)
+ set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
+endif()
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
find_package(Threads REQUIRED)
if(THREADS_HAVE_PTHREAD_ARG)
_append(CFLAGS_PTHREAD -pthread)
diff -urN apt-cacher-ng-3.7.5.orig/debian/rules apt-cacher-ng-3.7.5/debian/rules
--- apt-cacher-ng-3.7.5.orig/debian/rules 2024-07-22 15:34:48.000000000 +0000
+++ apt-cacher-ng-3.7.5/debian/rules 2024-07-22 15:34:48.000000000 +0000
@@ -43,7 +43,7 @@
test -r debian/apt-cacher-ng/etc/apt-cacher-ng/backends_ubuntu.default || mv debian/apt-cacher-ng/usr/lib/apt-cacher-ng/backends_ubuntu.default debian/apt-cacher-ng/etc/apt-cacher-ng/
override_dh_installinit:
- dh_installinit --no-restart-on-upgrade --restart-after-upgrade -- defaults 18
+ dh_installinit --no-stop-on-upgrade --restart-after-upgrade -- defaults 18
override_dh_compress:
dh_compress -X.pdf
diff -urN apt-cacher-ng-3.7.5.orig/debian/control apt-cacher-ng-3.7.5/debian/control
--- apt-cacher-ng-3.7.5.orig/debian/control 2024-07-22 15:34:48.000000000 +0000
+++ apt-cacher-ng-3.7.5/debian/control 2024-07-22 15:34:48.000000000 +0000
@@ -2,7 +2,7 @@
Section: net
Priority: optional
Maintainer: Eduard Bloch <[email protected]>
-Build-Depends: debhelper (>= 10), cmake (>= 3.5), libbz2-dev, zlib1g-dev, liblzma-dev, libfuse-dev [!hurd-i386], pkgconf | pkg-config, libwrap0-dev, sysvinit-utils (>= 3.05-4~) | lsb-base (>> 3.0-6), debhelper (>= 9.20160709), po-debconf, libssl-dev, libsystemd-dev (>= 210) [linux-any], libpthread-stubs0-dev, libevent-dev, libc-ares-dev (>> 1.15)
+Build-Depends: debhelper (>= 10), cmake (>= 3.5), libbz2-dev, zlib1g-dev, liblzma-dev, libfuse3-dev [!hurd-i386], pkgconf | pkg-config, libwrap0-dev, sysvinit-utils (>= 3.05-4~) | lsb-base (>> 3.0-6), debhelper (>= 9.20160709), po-debconf, libssl-dev, libsystemd-dev (>= 210) [linux-any], libpthread-stubs0-dev, libevent-dev, libc-ares-dev (>> 1.15)
Standards-Version: 4.5.1
Homepage: http://www.unix-ag.uni-kl.de/~bloch/acng/
Vcs-Git: https://salsa.debian.org/blade/apt-cacher-ng.git
diff -urN apt-cacher-ng-3.7.5.orig/src/job.cc apt-cacher-ng-3.7.5/src/job.cc
--- apt-cacher-ng-3.7.5.orig/src/job.cc 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/src/job.cc 2024-07-22 15:32:18.000000000 +0000
@@ -662,6 +662,19 @@
else
m_sFileLoc=theUrl.sHost+theUrl.sPath;
+ // Here we serialize multiple clients trying to download the
+ // same file. Only one thread at a time per URL is allowed to
+ // proceed further in this function.
+
+ lockuniq g{inProgressLock};
+
+ if (inProgress.contains(m_sFileLoc)) {
+ // Check if another job is running. If so link to that.
+ m_pItem = m_pParentCon.GetItemRegistry()->Create(m_sFileLoc, ESharingHow::ALWAYS_TRY_SHARING, fileitem::tSpecialPurposeAttr{});
+ USRDBG("Linked to other job");
+ return;
+ }
+
fileitem::tSpecialPurposeAttr attr {
! cfg::offlinemode && data_type == FILE_VOLATILE,
m_bIsHeadOnly,
@@ -697,8 +710,13 @@
if(cfg::trackfileuse && fistate >= fileitem::FIST_DLGOTHEAD && fistate < fileitem::FIST_DLERROR)
m_pItem.get()->UpdateHeadTimestamp();
- if(fistate==fileitem::FIST_COMPLETE)
+ if(fistate==fileitem::FIST_COMPLETE) {
+ // Tell everybody downloading this url that we already
+ // have a job to download it and register a cleanup
+ // when this job completes.
+ setInProgress(m_sFileLoc);
return; // perfect, done here
+ }
if(cfg::offlinemode) { // make sure there will be no problems later in SendData or prepare a user message
// error or needs download but freshness check was disabled, so it's really not complete.
@@ -760,6 +778,10 @@
return report_overload(__LINE__);
}
}
+ // Tell everybody downloading this url that we already have a
+ // job to download it and register a cleanup when this job
+ // completes.
+ setInProgress(m_sFileLoc);
}
catch (const std::bad_alloc&) // OOM, may this ever happen here?
{
@@ -1190,4 +1212,16 @@
<< "\r\nServer: Debian Apt-Cacher NG/" ACVERSION "\r\n"
"\r\n";
}
+
+job::inProgressCleanup::~inProgressCleanup() {
+ lockuniq g{inProgressLock};
+ LOGSTARTFUNC;
+ USRDBG("url=" << url);
+ if (url.size()) {
+ inProgress.erase(url);
+ }
+}
+
+std::set<std::string> job::inProgress;
+base_with_mutex job::inProgressLock;
}
diff -urN apt-cacher-ng-3.7.5.orig/src/job.h apt-cacher-ng-3.7.5/src/job.h
--- apt-cacher-ng-3.7.5.orig/src/job.h 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/src/job.h 2024-07-22 15:32:18.000000000 +0000
@@ -16,6 +16,24 @@
class job
{
+private:
+ // Lock controlling access to inProgress
+ static base_with_mutex inProgressLock;
+
+ // If there is an item in here then there is already a job downloading url
+ static std::set<std::string> inProgress;
+
+ // Simple class which is destroyed when the job is destroyed. It deletes the entry from inProgress.
+ struct inProgressCleanup {
+ std::string url;
+ inProgressCleanup() { }
+ ~inProgressCleanup();
+ };
+
+ void setInProgress(const std::string& url_) {
+ m_ipc.url = url_;
+ inProgress.insert(url_);
+ }
public:
enum eJobResult : short
@@ -48,6 +66,7 @@
} eActivity;
TFileItemHolder m_pItem;
+ inProgressCleanup m_ipc; // This MUST be destroyed before m_pItem
unique_fd m_filefd;
bool m_bIsHttp11 = true;
diff -urN apt-cacher-ng-3.7.5.orig/src/ac3rdparty.cc apt-cacher-ng-3.7.5/src/ac3rdparty.cc
--- apt-cacher-ng-3.7.5.orig/src/ac3rdparty.cc 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/src/ac3rdparty.cc 2024-07-22 15:32:18.000000000 +0000
@@ -37,12 +37,7 @@
if(inited)
return;
inited = true;
- SSL_load_error_strings();
- ERR_load_BIO_strings();
- ERR_load_crypto_strings();
- ERR_load_SSL_strings();
- OpenSSL_add_all_algorithms();
- SSL_library_init();
+ OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, nullptr);
g_ssl_locks.resize(CRYPTO_num_locks());
CRYPTO_set_id_callback(get_thread_id_cb);
diff -urN apt-cacher-ng-3.7.5.orig/src/acngtool.cc apt-cacher-ng-3.7.5/src/acngtool.cc
--- apt-cacher-ng-3.7.5.orig/src/acngtool.cc 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/src/acngtool.cc 2024-07-22 15:32:18.000000000 +0000
@@ -26,7 +26,6 @@
#include <list>
#include <queue>
-#include <cstdbool>
#include <cstdint>
#include <cstdlib>
#include <ctime>
@@ -814,12 +813,7 @@
void ssl_init()
{
#ifdef HAVE_SSL
- SSL_load_error_strings();
- ERR_load_BIO_strings();
- ERR_load_crypto_strings();
- ERR_load_SSL_strings();
- OpenSSL_add_all_algorithms();
- SSL_library_init();
+ OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, nullptr);
#endif
}
#endif
diff -urN apt-cacher-ng-3.7.5.orig/src/apt-cacher.cc apt-cacher-ng-3.7.5/src/apt-cacher.cc
--- apt-cacher-ng-3.7.5.orig/src/apt-cacher.cc 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/src/apt-cacher.cc 2024-07-22 15:32:18.000000000 +0000
@@ -18,7 +18,6 @@
#include <iostream>
-#include <cstdbool>
#include <cstdint>
#include <cstdlib>
#include <ctime>
diff -urN apt-cacher-ng-3.7.5.orig/src/filereader.cc apt-cacher-ng-3.7.5/src/filereader.cc
--- apt-cacher-ng-3.7.5.orig/src/filereader.cc 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/src/filereader.cc 2024-07-22 15:32:18.000000000 +0000
@@ -20,6 +20,7 @@
#include <atomic>
#ifdef HAVE_SSL
+#include <openssl/evp.h>
#include <openssl/sha.h>
#include <openssl/md5.h>
#elif defined(HAVE_TOMCRYPT)
@@ -543,33 +544,35 @@
#ifndef MINIBUILD
#ifdef HAVE_SSL
-class csumSHA1 : public csumBase, public SHA_CTX
+class csumEVP : public csumBase
{
public:
- csumSHA1() { SHA1_Init(this); }
- void add(const char *data, size_t size) override { SHA1_Update(this, (const void*) data, size); }
- void finish(uint8_t* ret) override { SHA1_Final(ret, this); }
+ EVP_MD_CTX *ctx;
+ unsigned int hash_len;
+ csumEVP() { ctx = EVP_MD_CTX_new(); }
+ ~csumEVP() { EVP_MD_CTX_free(ctx); }
+ void add(const char *data, size_t size) override { EVP_DigestUpdate(ctx, data, size); }
+ void finish(uint8_t* ret) override { EVP_DigestFinal_ex(ctx, ret, &hash_len);; }
};
-class csumSHA256 : public csumBase, public SHA256_CTX
+class csumSHA1 : public csumEVP, public SHA_CTX
{
public:
- csumSHA256() { SHA256_Init(this); }
- void add(const char *data, size_t size) override { SHA256_Update(this, (const void*) data, size); }
- void finish(uint8_t* ret) override { SHA256_Final(ret, this); }
+ csumSHA1() { EVP_DigestInit_ex(ctx, EVP_sha1(), NULL); }
};
-class csumSHA512 : public csumBase, public SHA512_CTX
+class csumSHA256 : public csumEVP, public SHA256_CTX
{
public:
- csumSHA512() { SHA512_Init(this); }
- void add(const char *data, size_t size) override { SHA512_Update(this, (const void*) data, size); }
- void finish(uint8_t* ret) override { SHA512_Final(ret, this); }
+ csumSHA256() { EVP_DigestInit_ex(ctx, EVP_sha256(), NULL); }
};
-class csumMD5 : public csumBase, public MD5_CTX
+class csumSHA512 : public csumEVP, public SHA512_CTX
{
public:
- csumMD5() { MD5_Init(this); }
- void add(const char *data, size_t size) override { MD5_Update(this, (const void*) data, size); }
- void finish(uint8_t* ret) override { MD5_Final(ret, this); }
+ csumSHA512() { EVP_DigestInit_ex(ctx, EVP_sha512(), NULL); }
+};
+class csumMD5 : public csumEVP, public MD5_CTX
+{
+public:
+ csumMD5() { EVP_DigestInit_ex(ctx, EVP_md5(), NULL); }
};
#elif defined(HAVE_TOMCRYPT)
class csumSHA1 : public csumBase
@@ -691,7 +694,7 @@
{
#ifdef HAVE_CHECKSUM
const char testvec[]="abc";
- uint8_t out[20];
+ uint8_t out[64];
unique_ptr<csumBase> ap(csumBase::GetChecker(CSTYPE_SHA1));
ap->add(testvec, sizeof(testvec)-1);
ap->finish(out);
@@ -709,6 +712,24 @@
cerr << "Incorrect MD5 implementation detected, check compilation settings!\n";
exit(EXIT_FAILURE);
}
+
+ ap = csumBase::GetChecker(CSTYPE_SHA256);
+ ap->add(testvec, sizeof(testvec) - 1);
+ ap->finish(out);
+ if (BytesToHexString(out, 32) != "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+ {
+ cerr << "Incorrect SHA256 implementation detected, check compilation settings!\n";
+ exit(EXIT_FAILURE);
+ }
+
+ ap = csumBase::GetChecker(CSTYPE_SHA512);
+ ap->add(testvec, sizeof(testvec) - 1);
+ ap->finish(out);
+ if (BytesToHexString(out, 64) != "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f")
+ {
+ cerr << "Incorrect SHA512 implementation detected, check compilation settings!\n";
+ exit(EXIT_FAILURE);
+ }
#endif
}
diff -urN apt-cacher-ng-3.7.5.orig/CMakeLists.txt apt-cacher-ng-3.7.5/CMakeLists.txt
--- apt-cacher-ng-3.7.5.orig/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10)
# try to set the best C++ language level
set(CMAKE_CXX_STANDARD 20)
diff -urN apt-cacher-ng-3.7.5.orig/fs/CMakeLists.txt apt-cacher-ng-3.7.5/fs/CMakeLists.txt
--- apt-cacher-ng-3.7.5.orig/fs/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/fs/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
@@ -3,16 +3,16 @@
## Attempt to build acngfs where possible ##
############################################
-pkg_check_modules(fuse fuse)
+pkg_check_modules(fuse fuse3)
_append(acngfs_cflags ${fuse_CFLAGS} ${CFLAGS_PTHREAD})
# double-check and make sure it compiles
-FILE(READ ${PROJECT_SOURCE_DIR}/${TESTKITDIR}/HAVE_FUSE_25.cc TESTSRC)
+FILE(READ ${PROJECT_SOURCE_DIR}/${TESTKITDIR}/HAVE_FUSE_30.cc TESTSRC)
SET(CMAKE_REQUIRED_FLAGS "${ACNG_COMPFLAGS} ${acngfs_cflags}")
-CHECK_CXX_SOURCE_COMPILES("${TESTSRC}" HAVE_FUSE_25)
+CHECK_CXX_SOURCE_COMPILES("${TESTSRC}" HAVE_FUSE_30)
-if(fuse_FOUND AND HAVE_FUSE_25)
+if(fuse_FOUND AND HAVE_FUSE_30)
# not linking acngstuff lib, too many dependencies, so building a custom variant with MINIBUILD flag
list(APPEND fsSRCS httpfs.cc)
include_directories("${CMAKE_SOURCE_DIR}/src")
@@ -24,8 +24,8 @@
endif()
TARGET_LINK_LIBRARIES(acngfs supacng ${fuse_LDFLAGS} ${BaseNetworkLibs} ${SSL_LIB_LIST} ${EXTRA_LIBS_ACNGFS} ${CMAKE_THREAD_LIBS_INIT})
-else(fuse_FOUND AND HAVE_FUSE_25)
+else(fuse_FOUND AND HAVE_FUSE_30)
message("- FUSE not found or not compatible, not building acngfs")
-endif(fuse_FOUND AND HAVE_FUSE_25)
+endif(fuse_FOUND AND HAVE_FUSE_30)
diff -urN apt-cacher-ng-3.7.5.orig/fs/httpfs.cc apt-cacher-ng-3.7.5/fs/httpfs.cc
--- apt-cacher-ng-3.7.5.orig/fs/httpfs.cc 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/fs/httpfs.cc 2024-07-22 15:32:18.000000000 +0000
@@ -4,7 +4,7 @@
// Description : Simple FUSE-based filesystem for HTTP access (apt-cacher NG)
//============================================================================
-#define FUSE_USE_VERSION 26
+#define FUSE_USE_VERSION 30
#include "debug.h"
@@ -32,7 +32,7 @@
#include <sys/vfs.h>
#endif
-#include <fuse.h>
+#include <fuse3/fuse.h>
#ifdef HAVE_DLOPEN
#include <dlfcn.h>
@@ -517,7 +517,7 @@
/// If found as downloadable, present as a file, or become a directory otherwise.
-static int acngfs_getattr(const char *path, struct stat *stbuf)
+static int acngfs_getattr(const char *path, struct stat *stbuf, fuse_file_info*)
{
if(!path)
return -1;
@@ -541,13 +541,6 @@
}
}
-static int acngfs_fgetattr(const char *path, struct stat *stbuf,
- struct fuse_file_info *)
-{
- // FIXME: reuse the con later? or better not, size might change during operation
- return acngfs_getattr(path, stbuf);
-}
-
static int acngfs_access(const char *, int mask)
{
// non-zero (failure) when trying to write
@@ -561,8 +554,8 @@
}
static int acngfs_readdir(const char *, void *, fuse_fill_dir_t,
- off_t, struct fuse_file_info *)
-{
+ off_t, struct fuse_file_info *, fuse_readdir_flags)
+{
return -EPERM;
}
@@ -667,7 +660,6 @@
memset(&acngfs_oper, 0, sizeof(acngfs_oper));
acngfs_oper.getattr = acngfs_getattr;
- acngfs_oper.fgetattr = acngfs_fgetattr;
acngfs_oper.access = acngfs_access;
acngfs_oper.opendir = acngfs_opendir;
acngfs_oper.readdir = acngfs_readdir;
@@ -757,31 +749,24 @@
evtimer_add(expTimer, &expTv);
#ifdef HAVE_DLOPEN
- auto pLib = dlopen("libfuse.so.2", RTLD_LAZY);
+ auto pLib = dlopen("libfuse.so.3", RTLD_LAZY);
if(!pLib)
{
- cerr << "Couldn't find libfuse.so.2" <<endl;
+ cerr << "Couldn't find libfuse.so.3" <<endl;
return -1;
}
- auto pFuseSetup = (decltype(&fuse_setup)) dlsym(pLib, "fuse_setup");
- auto pFuseLoop = (decltype(&fuse_loop_mt)) dlsym(pLib, "fuse_loop_mt");
- if(!pFuseSetup || !pFuseLoop)
+ auto pFuseMain = (decltype(&fuse_main_fn)) dlsym(pLib, "fuse_main_fn");
+ if(!pFuseMain)
{
- cerr << "Error loading libfuse.so.2" <<endl;
+ cerr << "Error loading libfuse.so.3" <<endl;
return -2;
}
#else
- auto pFuseSetup = &fuse_setup;
- auto pFuseLoop = &fuse_loop_mt;
+ auto pFuseMain = &fuse_main_fn;
#endif
- int mt = 1;
- auto fs = (*pFuseSetup) (fuseArgs.size(), &fuseArgs[0],
- &acngfs_oper, sizeof(acngfs_oper),
- &argv[2], &mt, nullptr);
- if (!fs)
- return 2;
- auto ret = (*pFuseLoop) (fs);
+ auto ret = pFuseMain(fuseArgs.size(), &fuseArgs[0],
+ &acngfs_oper, nullptr);
// shutdown agents in reliable fashion
spare_agents.clear();
return ret;
diff -urN apt-cacher-ng-3.7.5.orig/systemd/CMakeLists.txt apt-cacher-ng-3.7.5/systemd/CMakeLists.txt
--- apt-cacher-ng-3.7.5.orig/systemd/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
+++ apt-cacher-ng-3.7.5/systemd/CMakeLists.txt 2024-07-22 15:32:18.000000000 +0000
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10)
unset(SDTYPE)
if(HAVE_SD_NOTIFY)
diff -urN apt-cacher-ng-3.7.5.orig/tests/build/HAVE_FUSE_25.cc apt-cacher-ng-3.7.5/tests/build/HAVE_FUSE_25.cc
--- apt-cacher-ng-3.7.5.orig/tests/build/HAVE_FUSE_25.cc 2024-07-22 16:19:22.000000000 +0000
+++ apt-cacher-ng-3.7.5/tests/build/HAVE_FUSE_25.cc 1970-01-01 00:00:00.000000000 +0000
@@ -1,7 +0,0 @@
-#define _FILE_OFFSET_BITS 64
-#define FUSE_USE_VERSION 25
-#include <fuse.h>
-int main() {
- return 0;
-}
-
diff -urN apt-cacher-ng-3.7.5.orig/tests/build/HAVE_FUSE_30.cc apt-cacher-ng-3.7.5/tests/build/HAVE_FUSE_30.cc
--- apt-cacher-ng-3.7.5.orig/tests/build/HAVE_FUSE_30.cc 1970-01-01 00:00:00.000000000 +0000
+++ apt-cacher-ng-3.7.5/tests/build/HAVE_FUSE_30.cc 2025-11-16 18:23:03.495876428 +0000
@@ -0,0 +1,7 @@
+#define _FILE_OFFSET_BITS 64
+#define FUSE_USE_VERSION 30
+#include <fuse3/fuse.h>
+int main() {
+ return 0;
+}
+
