Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package grommunio-index for openSUSE:Factory checked in at 2024-06-14 19:03:05 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/grommunio-index (Old) and /work/SRC/openSUSE:Factory/.grommunio-index.new.19518 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "grommunio-index" Fri Jun 14 19:03:05 2024 rev:3 rq:1180912 version:1.0.6.f40d25b Changes: -------- --- /work/SRC/openSUSE:Factory/grommunio-index/grommunio-index.changes 2023-01-18 13:11:49.317083885 +0100 +++ /work/SRC/openSUSE:Factory/.grommunio-index.new.19518/grommunio-index.changes 2024-06-14 19:07:44.635827974 +0200 @@ -1,0 +2,6 @@ +Fri Jun 7 12:32:59 UTC 2024 - Jan Engelhardt <jeng...@inai.de> + +- Update to snapshot 1.0.6 + * Switch to user identity groindex/groweb+gromoxcf + +------------------------------------------------------------------- Old: ---- grommunio-index-0.1.18.6a0f73a.tar.xz New: ---- debian.grommunio-index.postinst grommunio-index-1.0.6.f40d25b.tar.xz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ grommunio-index.spec ++++++ --- /var/tmp/diff_new_pack.4DGvsW/_old 2024-06-14 19:07:45.163846744 +0200 +++ /var/tmp/diff_new_pack.4DGvsW/_new 2024-06-14 19:07:45.163846744 +0200 @@ -1,7 +1,7 @@ # # spec file for package grommunio-index # -# Copyright (c) 2023 SUSE LLC +# Copyright (c) 2024 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,7 +17,7 @@ Name: grommunio-index -Version: 0.1.18.6a0f73a +Version: 1.0.6.f40d25b Release: 0 Summary: Generator for grommunio-web search indexes License: AGPL-3.0-or-later @@ -30,12 +30,29 @@ %else BuildRequires: gcc-c++ %endif +%if 0%{?suse_version} +BuildRequires: libmysqlclient-devel >= 5.6 +%else +BuildRequires: mariadb-devel >= 5.6 +%endif BuildRequires: libexmdbpp-devel >= 1.8.0 BuildRequires: libexmdbpp0 >= 1.8.0 +BuildRequires: pkgconfig(libHX) >= 3.27 BuildRequires: pkgconfig(sqlite3) BuildRequires: pkgconfig(systemd) Requires: libexmdbpp0 >= 1.8.0 -Requires: user(groweb) +%if 0%{?suse_version} >= 1500 +BuildRequires: sysuser-tools +%sysusers_requires +%else +Requires(pre): %_sbindir/groupadd +Requires(pre): %_sbindir/useradd +%endif +Requires(pre): group(groweb) +Requires(pre): group(gromoxcf) +Requires: group(gromoxcf) +Requires: group(groweb) +Requires: user(groindex) %define services grommunio-index.service grommunio-index.timer %description @@ -45,48 +62,65 @@ %autosetup -p1 %build +>user.pre +%if 0%{?suse_version} >= 1500 +%sysusers_generate_pre %_sourcedir/system-user-groindex.conf user system-user-groindex.conf +%endif + +pushd . %if 0%{?suse_version} && 0%{?suse_version} < 1550 %cmake -DCMAKE_CXX_COMPILER=%_bindir/g++-11 %else -%if 0%{?centos_version} == 800 -echo '#!/bin/sh -ex' >cxx -echo 'exec g++ "$@" -lstdc++fs' >>cxx -ls -al cxx -chmod a+x cxx -export CXX="$PWD/cxx" -%cmake -%else %cmake %endif -%endif %cmake_build +popd %install -%if 0%{?centos_version} == 800 -export CXX="$PWD/cxx" -%endif +pushd . %cmake_install +popd mkdir -p "%buildroot/%_datadir/%name" -%pre +%pre -f user.pre +%if 0%{?rhel} || 0%{?fedora_version} +getent group groindex >/dev/null || %_sbindir/groupadd -r groindex +getent passwd groindex >/dev/null || %_sbindir/useradd -g groindex -s /bin/false \ + -r -c "user for %name" -d / groindex +usermod groindex -aG groweb +usermod groindex -aG gromoxcf +%endif +%if 0%{?service_add_pre:1} %service_add_pre %services +%endif %post +find /var/lib/grommunio-web/sqlite-index/ -mindepth 1 "(" -type d -o -type f ")" -exec chmod g+w,o-w {} + || : +find /var/lib/grommunio-web/sqlite-index/ -mindepth 1 "(" -type d -o -type f ")" -exec chgrp -h groweb {} + || : +%if 0%{?service_add_post:1} %service_add_post %services -if test -x /bin/systemctl; then - systemctl enable --now grommunio-index.timer || : -fi +%else +%systemd_post %services +%endif %preun +%if 0%{?service_del_preun:1} %service_del_preun %services +%else +%systemd_preun %services +%endif %postun +%if 0%{?service_del_postun:1} %service_del_postun %services +%else +%systemd_postun_with_restart %services +%endif %files %_bindir/grommunio-index* -%_sbindir/grommunio-index* %_datadir/%name/ +%_sysusersdir/*.conf %_unitdir/* %license LICENSE.txt ++++++ _service ++++++ --- /var/tmp/diff_new_pack.4DGvsW/_old 2024-06-14 19:07:45.195847880 +0200 +++ /var/tmp/diff_new_pack.4DGvsW/_new 2024-06-14 19:07:45.199848023 +0200 @@ -1,16 +1,15 @@ <services> - <service name="tar_scm" mode="disabled"> + <service name="tar_scm" mode="manual"> <param name="scm">git</param> <param name="url">https://github.com/grommunio/grommunio-index</param> <param name="filename">grommunio-index</param> <param name="revision">master</param> - <param name="parent-tag">0.1</param> <param name="versionformat">@PARENT_TAG@.@TAG_OFFSET@.%h</param> </service> - <service name="recompress" mode="disabled"> + <service name="recompress" mode="manual"> <param name="file">*.tar</param> <param name="compression">xz</param> </service> - <service name="set_version" mode="disabled"/> + <service name="set_version" mode="manual"/> </services> ++++++ debian.changelog ++++++ --- /var/tmp/diff_new_pack.4DGvsW/_old 2024-06-14 19:07:45.227849019 +0200 +++ /var/tmp/diff_new_pack.4DGvsW/_new 2024-06-14 19:07:45.231849161 +0200 @@ -1,4 +1,4 @@ -grommunio-index (0.1.18.6a0f73a) unstable; urgency=low +grommunio-index (1.0.6.f40d25b) unstable; urgency=low * Initial package. ++++++ debian.control ++++++ --- /var/tmp/diff_new_pack.4DGvsW/_old 2024-06-14 19:07:45.255850013 +0200 +++ /var/tmp/diff_new_pack.4DGvsW/_new 2024-06-14 19:07:45.259850156 +0200 @@ -10,6 +10,8 @@ Package: grommunio-index Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends} +Requires: system-user-groindex, + system-group-groweb, system-group-gromoxcf Description: Generator for grommunio-web search indexes . ++++++ debian.grommunio-index.postinst ++++++ usermod grommunio -aG groweb || : find /var/lib/grommunio-web/sqlite-index/ -mindepth 1 "(" -type d -o -type f ")" -exec chmod g+w,o-w {} + || : find /var/lib/grommunio-web/sqlite-index/ -mindepth 1 "(" -type d -o -type f ")" -exec chgrp -h groweb {} + || : ++++++ grommunio-index-0.1.18.6a0f73a.tar.xz -> grommunio-index-1.0.6.f40d25b.tar.xz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/CMakeLists.txt new/grommunio-index-1.0.6.f40d25b/CMakeLists.txt --- old/grommunio-index-0.1.18.6a0f73a/CMakeLists.txt 2023-01-16 20:03:36.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/CMakeLists.txt 2024-06-06 09:43:59.000000000 +0200 @@ -1,15 +1,24 @@ cmake_minimum_required(VERSION 3.14) -project(grommunio-index VERSION 0.1) +project(grommunio-index VERSION 1.0) set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) find_package(SQLite3 REQUIRED) find_package(exmdbpp 1.8 REQUIRED) find_package(PkgConfig REQUIRED) +pkg_get_variable(SYSUSERDIR systemd sysusersdir) pkg_get_variable(UNITDIR systemd systemdsystemunitdir) +pkg_check_modules(HX REQUIRED "libHX >= 3.27") +find_program(MARIADB_CONFIG NAMES mariadb_config mysql_config) +execute_process(COMMAND ${MARIADB_CONFIG} --cflags OUTPUT_VARIABLE MYSQL_CFLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) +separate_arguments(MYSQL_CFLAGS UNIX_COMMAND "${MYSQL_CFLAGS}") +execute_process(COMMAND ${MARIADB_CONFIG} --libs OUTPUT_VARIABLE MYSQL_LIBRARIES OUTPUT_STRIP_TRAILING_WHITESPACE) +separate_arguments(MYSQL_LIBRARIES UNIX_COMMAND "${MYSQL_LIBRARIES}") add_executable(grommunio-index grommunio-index.cpp) target_include_directories(grommunio-index PRIVATE ${SQLite3_INCLUDE_DIRS}) -target_link_libraries(grommunio-index ${SQLite3_LIBRARIES} exmdbpp::exmdbpp) +target_compile_options(grommunio-index PRIVATE ${HX_CFLAGS} ${MYSQL_CFLAGS} -Wall) +target_link_libraries(grommunio-index ${HX_LIBRARIES} ${MYSQL_LIBRARIES} ${SQLite3_LIBRARIES} exmdbpp::exmdbpp) option(LOGGING_TRACE "Enable TRACE logging level" OFF) if(LOGGING_TRACE) @@ -17,5 +26,5 @@ endif() install(TARGETS grommunio-index RUNTIME) -install(PROGRAMS grommunio-index-run.sh DESTINATION sbin) install(FILES grommunio-index.service grommunio-index.timer DESTINATION ${UNITDIR}) +install(FILES system-user-groindex.conf DESTINATION ${SYSUSERDIR}) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/README.rst new/grommunio-index-1.0.6.f40d25b/README.rst --- old/grommunio-index-0.1.18.6a0f73a/README.rst 2023-01-16 20:03:36.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/README.rst 2024-06-06 09:43:59.000000000 +0200 @@ -3,12 +3,12 @@ A C++17 program for the generation of grommunio-web fulltext search indexes. -|shield-agpl|_ |shield-release|_ |shield-cov|_ |shield-loc| +|shield-agpl| |shield-release| |shield-loc| .. |shield-agpl| image:: https://img.shields.io/badge/license-AGPL--3.0-green -.. _shield-agpl: LICENSE.txt + :target: LICENSE.txt .. |shield-release| image:: https://shields.io/github/v/tag/grommunio/grommunio-index -.. _shield-release: https://github.com/grommunio/grommunio-index/tags + :target: https://github.com/grommunio/grommunio-index/tags .. |shield-loc| image:: https://img.shields.io/github/languages/code-size/grommunio/grommunio-index Support diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/changelog.rst new/grommunio-index-1.0.6.f40d25b/changelog.rst --- old/grommunio-index-0.1.18.6a0f73a/changelog.rst 1970-01-01 01:00:00.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/changelog.rst 2024-06-06 09:43:59.000000000 +0200 @@ -0,0 +1,7 @@ +v1.0 (2023-12-31) +================= + +Behavioral changes: + +* Merge grommunio-index-run.sh function in grommunio-index itself; + new -A command-line option diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/grommunio-index-run.sh new/grommunio-index-1.0.6.f40d25b/grommunio-index-run.sh --- old/grommunio-index-0.1.18.6a0f73a/grommunio-index-run.sh 2023-01-16 20:03:36.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/grommunio-index-run.sh 1970-01-01 01:00:00.000000000 +0100 @@ -1,23 +0,0 @@ -#!/bin/bash - -MYSQL_CFG="/etc/gromox/mysql_adaptor.cfg" - -if [ ! -e "${MYSQL_CFG}" ] ; then - echo "MySQL configuration not found. ($MYSQL_CFG)" - exit 1 -fi - -mysql_params="--skip-column-names --skip-line-numbers" -mysql_username=$(sed -ne 's/mysql_username\s*=\s*\(.*\)/-u\1/p' ${MYSQL_CFG}) -mysql_password=$(sed -ne 's/mysql_password\s*=\s*\(.*\)/-p\1/p' ${MYSQL_CFG}) -mysql_dbname=$(sed -ne 's/mysql_dbname\s*=\s*\(.*\)/\1/p' ${MYSQL_CFG}) -mysql_host=$(sed -ne 's/mysql_host\s*=\s*\(.*\)/-h\1/p' ${MYSQL_CFG}) -mysql_port=$(sed -ne 's/mysql_port\s*=\s*\(.*\)/-P\1/p' ${MYSQL_CFG}) -mysql_query='select username, maildir from users where id <> 0 and maildir <> "";' -mysql_cmd="mysql ${mysql_params} ${mysql_username} ${mysql_password} ${mysql_host} ${mysql_port} ${mysql_dbname}" -web_index_path="/var/lib/grommunio-web/sqlite-index" - -echo "${mysql_query[@]}" | ${mysql_cmd} | while read -r username maildir ; do - [ -e "${web_index_path}/${username}/" ] || mkdir "${web_index_path}/${username}/" - grommunio-index "$maildir" -o "$web_index_path/$username/index.sqlite3" -done diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/grommunio-index.cpp new/grommunio-index-1.0.6.f40d25b/grommunio-index.cpp --- old/grommunio-index-0.1.18.6a0f73a/grommunio-index.cpp 2023-01-16 20:03:36.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/grommunio-index.cpp 2024-06-06 09:43:59.000000000 +0200 @@ -1,24 +1,98 @@ /* * SPDX-License-Identifier: AGPL-3.0-or-later - * SPDX-FileCopyrightText: 2022 grommunio GmbH + * SPDX-FileCopyrightText: 2022-2023 grommunio GmbH */ #include <algorithm> +#include <array> +#include <cerrno> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <ctime> #include <filesystem> +#include <getopt.h> #include <iostream> +#include <limits> +#include <map> +#include <memory> +#include <mysql.h> #include <optional> +#include <sqlite3.h> +#include <stdexcept> #include <string> - +#include <string_view> +#include <unistd.h> +#include <unordered_map> +#include <utility> +#include <vector> #include <exmdbpp/constants.h> #include <exmdbpp/queries.h> #include <exmdbpp/requests.h> #include <exmdbpp/util.h> -#include <sqlite3.h> +#include <libHX/proc.h> +#include <libHX/string.h> +#include <sys/stat.h> - -using namespace std; +using namespace std::string_literals; using namespace exmdbpp; using namespace exmdbpp::constants; using namespace exmdbpp::queries; +namespace fs = std::filesystem; + +namespace { + +using DB_ROW = char **; + +class DB_RESULT { /* from gromox/database_mysql.hpp */ + public: + DB_RESULT() = default; + DB_RESULT(MYSQL_RES *r) noexcept : m_res(r) {} + DB_RESULT(DB_RESULT &&o) noexcept : m_res(o.m_res) { o.m_res = nullptr; } + ~DB_RESULT() { clear(); } + + DB_RESULT &operator=(DB_RESULT &&o) noexcept + { + clear(); + m_res = o.m_res; + o.m_res = nullptr; + return *this; + } + void clear() { + if (m_res != nullptr) + mysql_free_result(m_res); + m_res = nullptr; + } + operator bool() const noexcept { return m_res != nullptr; } + bool operator==(std::nullptr_t) const noexcept { return m_res == nullptr; } + bool operator!=(std::nullptr_t) const noexcept { return m_res != nullptr; } + MYSQL_RES *get() const noexcept { return m_res; } + void *release() noexcept + { + void *p = m_res; + m_res = nullptr; + return p; + } + + size_t num_rows() const { return mysql_num_rows(m_res); } + DB_ROW fetch_row() { return mysql_fetch_row(m_res); } + + private: + MYSQL_RES *m_res = nullptr; +}; + +struct our_del { + inline void operator()(FILE *x) const { fclose(x); } + inline void operator()(MYSQL *x) const { mysql_close(x); } +}; + +struct user_row { + std::string username, dir, host; +}; + +} + +using kvpairs = std::map<std::string, std::string>; enum {RESULT_OK, RESULT_ARGERR_SYN, RESULT_ARGERR_SEM, RESULT_EXMDB_ERR}; ///< Exit codes enum LEVEL {FATAL, ERROR, WARNING, STATUS, INFO, DEBUG, TRACE, LOGLEVELS}; ///< Log levels @@ -52,8 +126,8 @@ { if(level > verbosity) return; - cout << "[" << levelname[level] << "] "; - (cout << ... << args) << endl; + std::cout << "[" << levelname[level] << "] "; + (std::cout << ... << args) << std::endl; } }; @@ -82,12 +156,12 @@ * @returns unordered_map containing objects */ template<typename K, typename V, class InputIt, typename F> -inline void mkMapMv(InputIt first, InputIt last, unordered_map<K, V>& umap, F&& key) +inline void mkMapMv(InputIt first, InputIt last, std::unordered_map<K, V>& umap, F&& key) { umap.clear(); umap.reserve(distance(first, last)); for(; first != last; ++first) - umap.try_emplace(key(*first), move(*first)); + umap.try_emplace(key(*first), std::move(*first)); } /** @@ -109,11 +183,11 @@ * @return Joined string */ template<class InputIt, typename F> -inline string strjoin(InputIt first, InputIt last, const std::string_view& glue = "",F&& tf=[](InputIt it){return *it;}) +inline std::string strjoin(InputIt first, InputIt last, const std::string_view& glue = "",F&& tf=[](InputIt it){return *it;}) { if(first == last) - return string(); - string str(tf(first)); + return std::string(); + std::string str(tf(first)); while(++first != last) { str += glue; @@ -133,9 +207,9 @@ * @return Reference to dest */ template<typename... Args> -inline string& strjoin(string& dest, const Args&... args) +inline std::string& strjoin(std::string& dest, Args&&... args) { - ((dest += args), ...); + ((dest += std::forward<Args>(args)), ...); return dest; } @@ -167,7 +241,7 @@ * @tparam tag { description } */ template<uint32_t tag> -inline void addTagStrLine(string& dest, const ExmdbQueries::PropvalList& pl) +inline void addTagStrLine(std::string& dest, const ExmdbQueries::PropvalList& pl) { static_assert(PropvalType::tagType(tag) != PropvalType::STRING || PropvalType::tagType(tag) != PropvalType::WSTRING, "Can only add string tags"); @@ -194,11 +268,11 @@ if(db) sqlite3_close(db); recheck = other.recheck; - dbpath = move(other.dbpath); - usrpath = move(other.usrpath); - client = move(other.client); - reuse = move(other.reuse); - namedProptags = move(other.namedProptags); + dbpath = std::move(other.dbpath); + usrpath = std::move(other.usrpath); + client = std::move(other.client); + reuse = std::move(other.reuse); + namedProptags = std::move(other.namedProptags); db = other.db; update = other.update; other.db = nullptr; @@ -224,7 +298,7 @@ * @param exmdbPort Port for exmdb connection * @param outpath Path of the output database or empty for default */ - IndexDB(const filesystem::path& userdir, const string& exmdbHost, const string& exmdbPort, const string& outpath, + IndexDB(const fs::path& userdir, const std::string& exmdbHost, const std::string& exmdbPort, const std::string& outpath, bool create=false, bool recheck=false) : usrpath(userdir), client(exmdbHost, exmdbPort, userdir, true, ExmdbClient::AUTO_RECONNECT), recheck(recheck) @@ -233,24 +307,24 @@ { dbpath = userdir; dbpath /= "exmdb"; - if(!filesystem::exists(dbpath)) - throw runtime_error("Cannot access "s + dbpath.c_str() + " (absent or permission problem)"); + if(!fs::exists(dbpath)) + throw std::runtime_error("Cannot access "s + dbpath.c_str() + " (absent or permission problem)"); dbpath /= "index.sqlite3"; } else { dbpath = outpath; - if(filesystem::is_directory(dbpath)) + if(fs::is_directory(dbpath)) dbpath /= "index.sqlite3"; } - update = filesystem::exists(dbpath); + update = fs::exists(dbpath); if (update) msg<STATUS>("Updating existing index "s + dbpath.c_str()); else msg<STATUS>("Creating new index "s + dbpath.c_str()); int res = sqlite3_open(dbpath.c_str(), &db); if(res != SQLITE_OK) - throw runtime_error(string("Failed to open index database: ")+sqlite3_errmsg(db)); + throw std::runtime_error("Failed to open index database: "s + sqlite3_errmsg(db)); if(update && create) { sqliteExec("DROP TABLE IF EXISTS hierarchy;" @@ -273,7 +347,7 @@ " date UNINDEXED, " " tokenize=unicode61)"); if(res != SQLITE_OK) - throw runtime_error(string("Failed to initialize index database: ")+sqlite3_errmsg(db)); + throw std::runtime_error("Failed to initialize index database: "s + sqlite3_errmsg(db)); } /** @@ -301,7 +375,7 @@ } private: - using PropvalMap = unordered_map<uint32_t, structures::TaggedPropval>; ///< Tag ID -> TaggedPropval mapping + using PropvalMap = std::unordered_map<uint32_t, structures::TaggedPropval>; ///< Tag ID -> TaggedPropval mapping /** * @brief SQLite3 statement wrapper class @@ -322,7 +396,7 @@ { int res = sqlite3_prepare_v2(db, zSql, -1, &stmt, nullptr); if(res != SQLITE_OK) - throw runtime_error(sqlite3_errmsg(db)); + throw std::runtime_error(sqlite3_errmsg(db)); } ~SQLiteStmt() {sqlite3_finalize(stmt);} @@ -348,9 +422,9 @@ template<typename... fArgs, typename... cArgs> void call(int(*func)(sqlite3_stmt*, fArgs...), cArgs&&... args) { - int res = func(stmt, args...); + int res = func(stmt, std::forward<cArgs>(args)...); if(res != SQLITE_OK) - throw runtime_error(sqlite3_errmsg(sqlite3_db_handle(stmt))); + throw std::runtime_error(sqlite3_errmsg(sqlite3_db_handle(stmt))); } /** @@ -367,8 +441,8 @@ { if(str.size() == 0) return call(sqlite3_bind_null, index); - if(str.size() > numeric_limits<int>::max()) - throw out_of_range("String lengths exceeds maximum"); + if(str.size() > std::numeric_limits<int>::max()) + throw std::out_of_range("String lengths exceeds maximum"); call(sqlite3_bind_text, index, str.data(), int(str.size()), SQLITE_STATIC); } @@ -396,7 +470,7 @@ { int index = sqlite3_bind_parameter_index(stmt, name); if(!index) - throw out_of_range(string("Cannot find named bind parameter ")+name); + throw std::out_of_range("Cannot find named bind parameter "s + name); return index; } @@ -413,7 +487,7 @@ { int result = sqlite3_step(stmt); if(result != SQLITE_DONE && result != SQLITE_ROW) - throw runtime_error("SQLite query failed: "+to_string(result)); + throw std::runtime_error("SQLite query failed: " + std::to_string(result)); return result; } @@ -425,7 +499,7 @@ */ struct Message { - inline Message(uint64_t mid, uint64_t fid, structures::TaggedPropval& entryid) : mid(mid), fid(fid), entryid(move(entryid)) {} + inline Message(uint64_t mid, uint64_t fid, structures::TaggedPropval& entryid) : mid(mid), fid(fid), entryid(std::move(entryid)) {} uint64_t mid, fid; structures::TaggedPropval entryid; @@ -442,7 +516,7 @@ struct { - string attchs, body, other, rcpts, sender, sending, subject, messageclass; + std::string attchs, body, other, rcpts, sender, sending, subject, messageclass; PropvalMap props; void reset() @@ -452,8 +526,8 @@ } } reuse; ///< Objects that can be reused to save on memory allocations - static array<structures::PropertyName, 14> namedTags; ///< Array of named tags to query - static constexpr array<uint16_t, 14> namedTagTypes = { + static std::array<structures::PropertyName, 14> namedTags; ///< Array of named tags to query + static constexpr std::array<uint16_t, 14> namedTagTypes = { PropvalType::STRING_ARRAY, PropvalType::STRING, PropvalType::STRING, PropvalType::STRING, PropvalType::STRING, PropvalType::STRING, PropvalType::STRING, @@ -463,7 +537,7 @@ PropvalType::STRING_ARRAY }; ///< Types of the named tags - static constexpr array<uint32_t, 12> msgtags1 = { + static constexpr std::array<uint32_t, 12> msgtags1 = { PropTag::ENTRYID, PropTag::SENTREPRESENTINGNAME, PropTag::SENTREPRESENTINGSMTPADDRESS, PropTag::SUBJECT, PropTag::BODY, PropTag::SENDERNAME, PropTag::SENDERSMTPADDRESS, PropTag::INTERNETCODEPAGE, @@ -471,7 +545,7 @@ PropTag::MESSAGEDELIVERYTIME, PropTag::LASTMODIFICATIONTIME }; ///< Part 1 of message tags to query - static constexpr array<uint32_t, 19> msgtags2 = { + static constexpr std::array<uint32_t, 19> msgtags2 = { PropTag::DISPLAYNAME, PropTag::DISPLAYNAMEPREFIX, PropTag::HOMETELEPHONENUMBER, PropTag::MOBILETELEPHONENUMBER, PropTag::BUSINESSTELEPHONENUMBER, PropTag::BUSINESSFAXNUMBER, PropTag::ASSISTANTTELEPHONENUMBER, @@ -482,10 +556,10 @@ PropTag::PRIMARYTELEPHONENUMBER, PropTag::RADIOTELEPHONENUMBER, PropTag::TELEXNUMBER, }; ///< Part 2 of message tags to query - filesystem::path usrpath; ///< Path to the user's home directory - filesystem::path dbpath; ///< Path to the index database + fs::path usrpath; ///< Path to the user's home directory + fs::path dbpath; ///< Path to the index database ExmdbClient client; ///< Exmdb client to use - vector<uint32_t> namedProptags; ///< Store specific named proptag IDs + std::vector<uint32_t> namedProptags; ///< Store specific named proptag IDs sqlite3* db = nullptr; ///< SQLite database connection bool update = false; ///< Whether index is updated bool recheck = false; ///< Whether to check all folders regardless of timestamp @@ -521,7 +595,7 @@ { auto res = find_if(tplist.begin(), tplist.end(), [tag](const structures::TaggedPropval& t){return t.tag == tag;}); if(res == tplist.end()) - throw out_of_range("Failed to find tag."); + throw std::out_of_range("Failed to find tag."); return *res; } @@ -530,7 +604,7 @@ * * @return Pair of messages and hierarchy entries to update */ - pair<vector<Message>, vector<Hierarchy>> getUpdates() + std::pair<std::vector<Message>, std::vector<Hierarchy>> getUpdates() { using namespace exmdbpp::constants; using namespace exmdbpp::requests; @@ -546,9 +620,9 @@ client.send<UnloadTableRequest>(usrpath, lhtResponse.tableId); msg<DEBUG>("Loaded ", qtResponse.entries.size(), " folders"); SQLiteStmt stmt(db, "SELECT commit_max, max_cn FROM hierarchy WHERE folder_id=?"); - vector<Message> messages; - vector<Hierarchy> hierarchy; - for(auto& entry : qtResponse.entries) + std::vector<Message> messages; + std::vector<Hierarchy> hierarchy; + for(auto& entry : qtResponse.entries) try { uint64_t lastCn = 0, maxCn = 0; uint64_t folderIdGc = getTag(entry, PropTag::FOLDERID).value.u64; @@ -577,12 +651,15 @@ uint64_t cn = util::gcToValue(getTag(content, PropTag::CHANGENUMBER).value.u64); if(cn <= lastCn) continue; - maxCn = max(maxCn, cn); + maxCn = std::max(maxCn, cn); messages.emplace_back(getTag(content, PropTag::MID).value.u64, folderId, getTag(content, PropTag::ENTRYID)); } msg<TRACE>("Checked folder ", folderId, " with ", contents.entries.size(), " messages. ", "Total updates now at ", messages.size(), "."); hierarchy.emplace_back(folderId, lctm, maxCn); + } catch (const std::out_of_range &e) { + msg<ERROR>("An essential property was missing from a folder; cannot proceed"); + throw EXIT_FAILURE; } msg<INFO>("Need to update ", messages.size(), " message", messages.size() == 1? "": "s", " and ", hierarchy.size(), " hierarchy entr", hierarchy.size() == 1? "y" : "ies", '.'); @@ -596,7 +673,7 @@ */ void removeMessages(const std::vector<Message>& messages) { - msg<DEBUG>("Removing modified messages"); + msg<DEBUG>("Removing ", messages.size(), " modified messages"); if(!update) return; SQLiteStmt stmt(db, "DELETE FROM messages WHERE message_id=?"); @@ -615,11 +692,12 @@ * * @param messages Messages to insert */ - void insertMessages(const vector<Message>& messages) + void insertMessages(const std::vector<Message>& messages) { msg<DEBUG>("Inserting new messages"); if(messages.empty()) return; + client.reconnect(); namedProptags = getNamedProptags(); std::vector<uint32_t> msgtags; msgtags.resize(namedProptags.size()+msgtags1.size()+msgtags2.size()); @@ -635,7 +713,7 @@ for(const Message& message : messages) { try {insertMessage(stmt, message, msgtags);} - catch (exception& e) + catch (const std::exception& e) {msg<ERROR>("Failed to insert message ", message.fid, "/", util::gcToValue(message.mid), ": ", e.what());} } sqliteExec("COMMIT"); @@ -648,7 +726,7 @@ * @param message Message to insert * @param msgtags List of tag IDs to query */ - void insertMessage(SQLiteStmt& stmt, const Message& message, const vector<uint32_t>& msgtags) + void insertMessage(SQLiteStmt& stmt, const Message& message, const std::vector<uint32_t>& msgtags) { using namespace constants; using namespace requests; @@ -657,7 +735,7 @@ reuse.reset(); stmt.call(sqlite3_reset); uint32_t instance = client.send<LoadMessageInstanceRequest>(usrpath, "", 65001, false, 0, message.mid).instanceId; - auto rcpts = client.send<GetMessageInstanceRecipientsRequest>(usrpath, instance, 0, numeric_limits<uint16_t>::max()); + auto rcpts = client.send<GetMessageInstanceRecipientsRequest>(usrpath, instance, 0, std::numeric_limits<uint16_t>::max()); auto attchs = client.send<QueryMessageInstanceAttachmentsTableRequest>(usrpath, instance, attchProps, 0, 0); auto propvals = client.send<GetInstancePropertiesRequest>(usrpath, 0, instance, msgtags).propvals; client.send<UnloadInstanceRequest>(usrpath, instance); @@ -728,7 +806,7 @@ using namespace requests; auto response = client.send<GetNamedPropIdsRequest>(usrpath, false, namedTags); if(response.propIds.size() != namedTagTypes.size()) - throw out_of_range("Number of named property IDs does not match expected count"); + throw std::out_of_range("Number of named property IDs does not match expected count"); std::vector<uint32_t> propTags(namedTagTypes.size()); transform(namedTagTypes.begin(), namedTagTypes.end(), response.propIds.begin(), propTags.begin(), [](uint16_t id, uint16_t type) {return uint32_t(id) << 16 | type;}); @@ -756,7 +834,7 @@ } }; -array<structures::PropertyName, 14> IndexDB::namedTags = { +std::array<structures::PropertyName, 14> IndexDB::namedTags = { structures::PropertyName(structures::GUID("00020329-0000-0000-C000-000000000046"), "Keywords"), //categories structures::PropertyName(structures::GUID("00062004-0000-0000-C000-000000000046"), 0x8005), //fileas structures::PropertyName(structures::GUID("00062002-0000-0000-C000-000000000046"), 0x8208), //location @@ -775,12 +853,13 @@ /////////////////////////////////////////////////////////////////////////////////////////////////// -static string exmdbHost; ///< Exmdb host to connect to -static string exmdbPort; ///< Port of the exmdb connection -static optional<string> userpath; ///< Path to the user's home directory -static string outpath; ///< Index database path (empty for default) +static std::string exmdbHost; ///< Exmdb host to connect to +static std::string exmdbPort; ///< Port of the exmdb connection +static std::optional<std::string> userpath; ///< Path to the user's home directory +static std::string outpath; ///< Index database path (empty for default) static bool recheck = false; ///< Check folders even when they were not changed since the last indexing static bool create = false; ///< Always create a new index instead of updating +static bool do_all_users; /** * @brief Print help message @@ -789,11 +868,13 @@ */ [[noreturn]] static void printHelp(const char* name) { - cout << "grommunio mailbox indexing tool\n" + std::cout << "grommunio mailbox indexing tool\n" "\nUsage: " << name << " [-c] [-e host] [-f] [-h] [-o file] [-p port] [-q] [-v] <userpath>\n" + "Usage: " << name << " -A [-c] [-f] [-h] [-p port] [-q] [-v]\n" "\nPositional arguments:\n" "\t userpath\t\tPath to the user's mailbox directory\n" "\nOptional arguments:\n" + "\t-A\t--all \tAutomatically process all local users (-e, -o ignored)\n" "\t-c\t--create \tCreate a new index instead of updating\n" "\t-e\t--host \tHostname of the exmdb server\n" "\t-h\t--help \tShow this help message and exit\n" @@ -806,108 +887,196 @@ } /** - * @brief Advance by one command line argument - * - * If there are no more arguments, exit with error. - * - * @param cur Pointer to current argument - * - * @return Next argument - */ -static const char* nextArg(const char** &cur) -{ - if(*++cur == nullptr) - { - msg<FATAL>("Missing option argument after '", *(cur-1), "'"); - exit(RESULT_ARGERR_SYN); - } - return *cur; -} - -/** * @brief Parse command line arguments * * Does not return in case of error. * * @param argv nullptr-terminated array of command line arguments */ -static void parseArgs(const char* argv[]) +static void parseArgs(int argc, char **argv) { - bool noopt = false; - for(const char** argp = argv+1; *argp != nullptr; ++argp) - { - const char* arg = *argp; - if(noopt || *arg == '-') - { - if(*(++arg) == '-') - { - ++arg; - if(!strcmp(arg, "create")) create = true; - else if(!strcmp(arg, "help")) printHelp(*argv); - else if(!strcmp(arg, "host")) exmdbHost = nextArg(argp); - else if(!strcmp(arg, "out")) outpath = nextArg(argp); - else if(!strcmp(arg, "port")) exmdbPort = nextArg(argp); - else if(!strcmp(arg, "quiet")) --verbosity; - else if(!strcmp(arg, "recheck")) recheck = true; - else if(!strcmp(arg, "verbose"))++verbosity; - else if(!*arg) noopt = true; - else - { - msg<FATAL>("Unknown option '", arg, "'"); - exit(RESULT_ARGERR_SYN); - } - } - else - for(const char* sopt = arg; *sopt; ++sopt) - { - switch(*sopt) - { - case 'c': create = true; break; - case 'e': exmdbHost = nextArg(argp); break; - case 'h': printHelp(*argv); //printHelp never return - case 'o': outpath = nextArg(argp); break; - case 'p': exmdbPort = nextArg(argp); break; - case 'q': --verbosity; break; - case 'r': recheck = true; break; - case 'v': ++verbosity; break; - default: - msg<FATAL>("Unknown short option '", *sopt, "'"); - exit(RESULT_ARGERR_SYN); - } - } - } - else if(userpath.has_value()) - { - msg<FATAL>("Too many arguments."); + static const struct option longopts[] = { + {"all", false, nullptr, 'A'}, + {"create", false, nullptr, 'c'}, + {"host", true, nullptr, 'e'}, + {"help", false, nullptr, 'h'}, + {"outpath", true, nullptr, 'o'}, + {"port", true, nullptr, 'p'}, + {"quiet", false, nullptr, 'q'}, + {"recheck", false, nullptr, 'r'}, + {"verbose", false, nullptr, 'v'}, + {}, + }; + + int c; + while ((c = getopt_long(argc, argv, "Ace:ho:p:qrv", longopts, nullptr)) >= 0) { + switch (c) { + case 'A': do_all_users = true; break; + case 'c': create = true; break; + case 'e': exmdbHost = optarg; break; + case 'h': printHelp(*argv); break; + case 'o': outpath = optarg; break; + case 'p': exmdbPort = optarg; break; + case 'q': --verbosity; break; + case 'r': recheck = true; break; + case 'v': ++verbosity; break; + default: exit(RESULT_ARGERR_SYN); } - else - userpath.emplace(arg); } - if(!userpath.has_value()) - { - msg<FATAL>("Usage: grommunio-index MAILDIR"); - msg<STATUS>("Option overview: grommunio-index -h"); - exit(RESULT_ARGERR_SYN); + if (do_all_users) { + if (!exmdbHost.empty() || !outpath.empty() || argc > optind) { + msg<FATAL>("Cannot combine -A with -e/-o/userpath"); + exit(RESULT_ARGERR_SYN); + } + } else { + if (argc > optind) + userpath.emplace(argv[optind++]); + if (!userpath.has_value()) { + msg<FATAL>("Usage: grommunio-index MAILDIR"); + msg<STATUS>("Option overview: grommunio-index -h"); + exit(RESULT_ARGERR_SYN); + } } if(exmdbHost.empty()) exmdbHost = "localhost"; if(exmdbPort.empty()) exmdbPort = "5000"; - verbosity = min(max(verbosity, 0), LOGLEVELS-1); + verbosity = std::min(std::max(verbosity, 0), LOGLEVELS-1); } -int main(int, const char* argv[]) +static int single_mode() { - parseArgs(argv); msg<DEBUG>("exmdb=", exmdbHost, ":", exmdbPort, ", user=", userpath.value(), ", output=", outpath.empty()? "<default>" : outpath); IndexDB cache; try { cache = IndexDB(userpath.value(), exmdbHost, exmdbPort, outpath, create, recheck); cache.refresh(); - } catch(const runtime_error& err) { + } catch(const std::runtime_error& err) { msg<FATAL>(err.what()); return RESULT_ARGERR_SEM; } return 0; } + +static kvpairs am_read_config(const char *path) +{ + std::unique_ptr<FILE, our_del> fp(fopen(path, "r")); + if (fp == nullptr) { + fprintf(stderr, "%s: %s\n", path, strerror(errno)); + throw EXIT_FAILURE; + } + kvpairs vars; + hxmc_t *ln = nullptr; + while (HX_getl(&ln, fp.get()) != nullptr) { + auto eq = strchr(ln, '='); + if (eq == nullptr) + continue; + if (*ln == '#') + continue; + HX_chomp(ln); + *eq++ = '\0'; + HX_strrtrim(ln); + HX_strltrim(eq); + vars[ln] = eq; + } + HXmc_free(ln); + /* fill with defaults if empty */ + vars.try_emplace("mysql_username", "root"); + vars.try_emplace("mysql_dbname", "grommunio"); + vars.try_emplace("mysql_host", "localhost"); + return vars; +} + +static std::vector<user_row> am_read_users(kvpairs &&vars) +{ + std::unique_ptr<MYSQL, our_del> conn(mysql_init(nullptr)); + if (conn == nullptr) + throw EXIT_FAILURE; + auto pass = vars.find("mysql_password"); + if (mysql_real_connect(conn.get(), vars["mysql_host"].c_str(), + vars["mysql_username"].c_str(), pass != vars.end() ? pass->second.c_str() : nullptr, + vars["mysql_dbname"].c_str(), strtoul(vars["mysql_port"].c_str(), nullptr, 0), + nullptr, 0) == nullptr) { + fprintf(stderr, "mysql_connect: %s\n", mysql_error(conn.get())); + throw EXIT_FAILURE; + } + if (mysql_set_character_set(conn.get(), "utf8mb4") != 0) { + fprintf(stderr, "\"utf8mb4\" not available: %s", mysql_error(conn.get())); + throw EXIT_FAILURE; + } + + static constexpr char query[] = + "SELECT u.username, u.maildir, s.hostname FROM users u " + "LEFT JOIN servers s ON u.homeserver=s.id WHERE u.id>0"; + if (mysql_query(conn.get(), query) != 0) { + fprintf(stderr, "%s: %s\n", query, mysql_error(conn.get())); + throw EXIT_FAILURE; + } + DB_RESULT myres = mysql_store_result(conn.get()); + if (myres == nullptr) { + fprintf(stderr, "result: %s\n", mysql_error(conn.get())); + throw EXIT_FAILURE; + } + std::vector<user_row> ulist; + for (DB_ROW row; (row = myres.fetch_row()) != nullptr; ) { + if (row[0] == nullptr) + continue; + struct stat sb; + if (row[1] == nullptr || stat(row[1], &sb) != 0 || !S_ISDIR(sb.st_mode)) + /* Homedir not present here */ + continue; + auto host = row[2] != nullptr && row[2][0] != '\0' ? row[2] : "::1"; + ulist.emplace_back(user_row{row[0], row[1], host}); + } + return ulist; +} + +int main(int argc, char **argv) try +{ + parseArgs(argc, argv); + + auto cfg = am_read_config("/etc/gromox/mysql_adaptor.cfg"); + /* Generated index files should not be world-readable */ + umask(07); + if (!do_all_users) + return single_mode(); + + if (geteuid() == 0) { + auto ret = HXproc_switch_user("groindex", "groweb"); + if (static_cast<int>(ret) < 0) { + fprintf(stderr, "switch_user grommunio/groweb: %s\n", strerror(errno)); + return EXIT_FAILURE; + } + /* setuid often disables coredumps, so restart to get them back. */ + execv(argv[0], argv); + } + int bigret = EXIT_SUCCESS; + static const std::string index_root = "/var/lib/grommunio-web/sqlite-index"; + for (auto &&u : am_read_users(std::move(cfg))) { + auto index_home = index_root + "/" + u.username; + if (mkdir(index_home.c_str(), 0777) != 0 && errno != EEXIST) { + fprintf(stderr, "mkdir %s: %s\n", index_home.c_str(), strerror(errno)); + bigret = EXIT_FAILURE; + continue; + } + auto index_file = index_home + "/index.sqlite3"; + auto now = time(nullptr); + char tmbuf[32]; + strftime(tmbuf, sizeof(tmbuf), "%FT%T", localtime(&now)); + fprintf(stderr, "[%s] %s %s -e %s -o %s\n", tmbuf, argv[0], + u.dir.c_str(), u.host.c_str(), index_file.c_str()); + userpath.emplace(std::move(u.dir)); + exmdbHost = std::move(u.host); + outpath = std::move(index_file); + auto ret = single_mode(); + if (ret != 0) { + fprintf(stderr, "\t... exited with status %d\n", ret); + bigret = EXIT_FAILURE; + } + fprintf(stderr, "\n"); + } + return bigret; +} catch (int e) { + return e; +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/grommunio-index.service new/grommunio-index-1.0.6.f40d25b/grommunio-index.service --- old/grommunio-index-0.1.18.6a0f73a/grommunio-index.service 2023-01-16 20:03:36.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/grommunio-index.service 2024-06-06 09:43:59.000000000 +0200 @@ -13,5 +13,5 @@ ProtectControlGroups=true RestrictRealtime=true Type=oneshot -User=groweb -ExecStart=/usr/sbin/grommunio-index-run.sh +User=groindex +ExecStart=/usr/bin/grommunio-index -Aq diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/grommunio-index.timer new/grommunio-index-1.0.6.f40d25b/grommunio-index.timer --- old/grommunio-index-0.1.18.6a0f73a/grommunio-index.timer 2023-01-16 20:03:36.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/grommunio-index.timer 2024-06-06 09:43:59.000000000 +0200 @@ -1,11 +1,10 @@ [Unit] -Description=Daily regeneration of grommunio FTS indexes +Description=Recurrent regeneration of grommunio FTS indexes [Timer] -OnCalendar=daily -AccuracySec=12h -Persistent=true -RandomizedDelaySec=6000 +OnCalendar=*:0/15 +RandomizedDelaySec=300 +Persistent=false [Install] WantedBy=timers.target diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/grommunio-index-0.1.18.6a0f73a/system-user-groindex.conf new/grommunio-index-1.0.6.f40d25b/system-user-groindex.conf --- old/grommunio-index-0.1.18.6a0f73a/system-user-groindex.conf 1970-01-01 01:00:00.000000000 +0100 +++ new/grommunio-index-1.0.6.f40d25b/system-user-groindex.conf 2024-06-06 09:43:59.000000000 +0200 @@ -0,0 +1,3 @@ +u groindex - "user for grommunio-index" +m groindex groweb +m groindex gromoxcf ++++++ grommunio-index.dsc ++++++ --- /var/tmp/diff_new_pack.4DGvsW/_old 2024-06-14 19:07:45.387854706 +0200 +++ /var/tmp/diff_new_pack.4DGvsW/_new 2024-06-14 19:07:45.391854848 +0200 @@ -1,7 +1,7 @@ Format: 1.0 Source: grommunio-index Architecture: any -Version: 0.1.18.6a0f73a +Version: 1.0.6.f40d25b DEBTRANSFORM-RELEASE: 1 Maintainer: Grommunio <n...@grommunio.com> Homepage: https://grommunio.com/