Hello community, here is the log from the commit of package ugrep for openSUSE:Factory checked in at 2020-10-23 14:28:03 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/ugrep (Old) and /work/SRC/openSUSE:Factory/.ugrep.new.3463 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "ugrep" Fri Oct 23 14:28:03 2020 rev:10 rq:843588 version:3.0.2 Changes: -------- --- /work/SRC/openSUSE:Factory/ugrep/ugrep.changes 2020-09-23 18:43:47.169508180 +0200 +++ /work/SRC/openSUSE:Factory/.ugrep.new.3463/ugrep.changes 2020-10-23 14:28:18.076826991 +0200 @@ -1,0 +2,6 @@ +Fri Oct 23 10:17:28 UTC 2020 - Andreas Stieger <andreas.stie...@gmx.de> + +- ugrep 3.0.2: + * speed improvements + +------------------------------------------------------------------- Old: ---- ugrep-2.5.6.tar.gz New: ---- ugrep-3.0.2.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ ugrep.spec ++++++ --- /var/tmp/diff_new_pack.koenVu/_old 2020-10-23 14:28:18.636827260 +0200 +++ /var/tmp/diff_new_pack.koenVu/_new 2020-10-23 14:28:18.640827262 +0200 @@ -17,7 +17,7 @@ Name: ugrep -Version: 2.5.6 +Version: 3.0.2 Release: 0 Summary: Universal grep: a feature-rich grep implementation with focus on speed License: BSD-3-Clause ++++++ ugrep-2.5.6.tar.gz -> ugrep-3.0.2.tar.gz ++++++ ++++ 1627 lines of diff (skipped) ++++ retrying with extended exclude list diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/README.md new/ugrep-3.0.2/README.md --- old/ugrep-2.5.6/README.md 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/README.md 2020-10-13 16:59:53.000000000 +0200 @@ -1,22 +1,28 @@ [![build status][travis-image]][travis-url] [![Language grade: C/C++][lgtm-image]][lgtm-url] [![license][bsd-3-image]][bsd-3-url] +**ugrep v3 is now available: more features & even faster than before** + Search for anything in everything... ultra fast *New option -Q opens a query UI to search files as you type!* <br> <img src="https://www.genivia.com/images/scranim.gif"; width="438" alt=""> -- Written in clean and efficient C++11 for features and speed, thoroughly tested +- Written in clean and efficient C++11 for advanced features and speed, thoroughly tested - Portable (Linux, Unix, MacOS, Windows, etc), includes x86 and x64 binaries for Windows in the [releases](https://github.com/Genivia/ugrep/releases) -- Ultra fast with new match algorithms and features, beating grep, ripgrep, silver searcher, hyperscan, sift, etc. see [performance](#speed) +- Supports all GNU/BSD grep standard options; a compatible replacement for GNU/BSD grep + +- Matches Unicode patterns by default in UTF-8, UTF-16, UTF-32 encoded files + +- Ultra fast with new match algorithms and features beating grep, ripgrep, silver searcher, ack, sift, etc. - User-friendly with sensible defaults and customizable [configuration files](#config) used by the `ug` command, which is the same as `ugrep --config` to load the .ugrep configuration file with your preferences ug PATTERN ... ugrep --config PATTERN ... -- Interactive [query UI](#query), press F1 or CTRL-Z for help +- Interactive [query UI](#query), press F1 or CTRL-Z for help and TAB to navigate to dirs and files ugrep -Q ... ugrep -Q -e PATTERN ... @@ -88,7 +94,7 @@ ugrep -P PATTERN ... ugrep -P --format='%1 and %2%~' 'PATTERN(SUB1)(SUB2)' ... -- Search files in ASCII and UTF-8/16/32 with Unicode regex patterns by default, or files with a specific [encoding](#encoding) format such as ISO-8859-1 thru 16, CP 437, CP 850, MACROMAN, KOI8, etc. +- Search files with a specific [encoding](#encoding) format such as ISO-8859-1 thru 16, CP 437, CP 850, MACROMAN, KOI8, etc. ugrep --encoding=LATIN1 PATTERN ... @@ -355,11 +361,11 @@ ---- | ---------------------------------------------------------------- | ----------------------------------------------------- T1 | `GREP -c quartz enwik8` | count "quartz" in a 100MB file (word with low frequency letters) T2 | `GREP -c sternness enwik8` | count "sternness" in a 100MB file (word with high frequency letters) -T3 | `GREP -cw -e char -e int -e long -e size_t -e void big.cpp` | count 5 short words in a 35MB C++ source code file -T4 | `GREP -Eon 'serialize_[a-zA-Z0-9_]+Type' big.cpp` | search and display C++ serialization functions in a 35MB source code file -T5 | `GREP -Fon -f words1+1000 enwik8` | search 1000 words of length 1 or longer in a 100MB Wikipedia file -T6 | `GREP -Fon -f words2+1000 enwik8` | search 1000 words of length 2 or longer in a 100MB Wikipedia file -T7 | `GREP -Fon -f words3+1000 enwik8` | search 1000 words of length 3 or longer in a 100MB Wikipedia file +T3 | `GREP -c 'Sherlock Holmes' en.txt` | count "Sherlock Holmes" in a huge [13GB decompressed file](http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.en.gz) +T4 | `GREP -cw -e char -e int -e long -e size_t -e void big.cpp` | count 5 short words in a 35MB C++ source code file +T5 | `GREP -Eon 'serialize_[a-zA-Z0-9_]+Type' big.cpp` | search and display C++ serialization functions in a 35MB source code file +T6 | `GREP -Fon -f words1+1000 enwik8` | search 1000 words of length 1 or longer in a 100MB Wikipedia file +T7 | `GREP -Fon -f words2+1000 enwik8` | search 1000 words of length 2 or longer in a 100MB Wikipedia file T8 | `GREP -Fon -f words4+1000 enwik8` | search 1000 words of length 4 or longer in a 100MB Wikipedia file T9 | `GREP -Fon -f words8+1000 enwik8` | search 1000 words of length 8 or longer in a 100MB Wikipedia file T10 | `GREP -ro '#[[:space:]]*include[[:space:]]+"[^"]+"' -Oh,hpp,cpp` | multi-threaded recursive search of `#include "..."` in the directory tree from the Qt 5.9.2 root, restricted to `.h`, `.hpp`, and `.cpp` files @@ -379,46 +385,55 @@ ### Performance results The following performance tests were conducted with a new and common MacBook -using clang 9.0.0 -O2 on a 2.9 GHz Intel Core i7, 16 GB 2133 MHz LPDDR3 Mac OS -10.12.6 machine. The best times of 30 runs is shown under minimal machine +Pro using clang 9.0.0 -O2 on a 2.9 GHz Intel Core i7, 16 GB 2133 MHz LPDDR3 Mac +OS 10.12.6 machine. The best times of 30 runs is shown under minimal machine load. When comparing tools, the same match counts were produced. Results are shown in real time (wall clock time) seconds elapsed. Best times are shown in **boldface** and *n/a* means that the running time exceeded 1 -minute or the selected options are not supported (e.g. option `-z`). +minute or the selected options are not supported (T12: option `-z`) or the +input file is too large (T3: 13GB file) resulting in an error. GREP | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | --------------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -ugrep | **0.05** | **0.06** | **0.08** | **0.03** | **0.98** | **0.97** | **0.86** | **0.87** | **0.29** | **0.10** | **0.19** | **0.02** | -hyperscan grep | 0.09 | 0.10 | 0.11 | 0.04 | 7.78 | 3.39 | 2.35 | 1.41 | 1.17 | *n/a* | *n/a* | *n/a* | -ripgrep | 0.06 | 0.10 | 0.19 | 0.06 | 2.20 | 2.07 | 2.00 | 2.01 | 2.14 | 0.12 | 0.36 | 0.03 | -silver searcher | 0.10 | 0.11 | 0.16 | 0.21 | *n/a* | *n/a* | *n/a* | *n/a* | *n/a* | 0.45 | 0.32 | 0.09 | -GNU grep 3.3 | 0.08 | 0.15 | 0.18 | 0.16 | 2.70 | 2.64 | 2.54 | 2.42 | 2.26 | *n/a* | 0.26 | *n/a* | -PCREGREP 8.42 | 0.17 | 0.17 | 0.26 | 0.08 | *n/a* | *n/a* | *n/a* | *n/a* | *n/a* | *n/a* | 2.37 | *n/a* | -BSD grep 2.5.1 | 0.81 | 1.60 | 1.85 | 0.83 | *n/a* | *n/a* | *n/a* | *n/a* | *n/a* | *n/a* | 3.35 | 0.60 | - -Note: [silver searcher 2.2.0](https://github.com/ggreer/the_silver_searcher) -runs slower with multiple threads (T10 0.45s) than single-threaded (T11 0.32s), -which was reported as an issue to the maintainers. - -[Hyperscan simple grep](https://github.com/intel/hyperscan/tree/master/examples) -was compiled with all optimizations enabled. This grep tool returns a few more -matches than other greps due to its "all matches reported" pattern matching -behavior. Option `-w` was emulated using the pattern -`\b(char|int|long|size_t|void)\b`. Option `-f` was emulated as follows: +ugrep | **0.03** | **0.04** | **5.06** | **0.07** | **0.02** | **0.98** | **0.97** | **0.87** | **0.26** | **0.10** | **0.19** | **0.02** | +hyperscan grep | 0.09 | 0.10 | **4.35** | 0.11 | 0.04 | 7.78 | 3.39 | 1.41 | 1.17 | *n/a* | *n/a* | *n/a* | +ripgrep | 0.06 | 0.10 | 7.50 | 0.19 | 0.06 | 2.20 | 2.07 | 2.01 | 2.14 | 0.12 | 0.36 | 0.03 | +silver searcher | 0.10 | 0.11 | *n/a* | 0.16 | 0.21 | *n/a* | *n/a* | *n/a* | *n/a* | 0.45 | 0.32 | 0.09 | +GNU grep 3.3 | 0.08 | 0.15 | 11.26 | 0.18 | 0.16 | 2.70 | 2.64 | 2.42 | 2.26 | *n/a* | 0.26 | *n/a* | +PCREGREP 8.42 | 0.17 | 0.17 | *n/a* | 0.26 | 0.08 | *n/a* | *n/a* | *n/a* | *n/a* | *n/a* | 2.37 | *n/a* | +BSD grep 2.5.1 | 0.81 | 1.60 | *n/a* | 1.85 | 0.83 | *n/a* | *n/a* | *n/a* | *n/a* | *n/a* | 3.35 | 0.60 | + +Note T3: [Hyperscan simplegrep](https://github.com/intel/hyperscan/tree/master/examples) +was compiled with optimizations enabled. Hyperscan results for T3 are slightly +better than ugrep, as expected because hyperscan simplegrep has one advantage +here: it does not maintain line numbers and other line-related information. By +contrast, line information should be tracked (as in ugrep) to determine if +matches are on the same line or not, as required by option `-c`. Hyperscan +simplegrep returns more matches than other greps due to its "all matches +reported" pattern matching behavior. + +Note T4-T9: Hyperscan simplegrep does not support command line options. Option +`-w` was emulated using the pattern `\b(char|int|long|size_t|void)\b`. Option +`-f` was emulated as follows: paste -d'|' -s words1+1000 > pattern.txt /usr/bin/time ./simplegrep `cat pattern.txt` enwik8 | ./null -Note: output is sent to a `null` utility to eliminate terminal display -overhead. The `null` utility source code: +Note T10+T11: [silver searcher 2.2.0](https://github.com/ggreer/the_silver_searcher) +runs slower with multiple threads (T10 0.45s) than single-threaded (T11 0.32s), +which was reported as an issue to the maintainers. + +Output is sent to a `null` utility to eliminate terminal display overhead +(`> /dev/null` cannot be used as some greps detect it to remove all output). +The `null` utility source code: #include <sys/types.h> #include <sys/uio.h> #include <unistd.h> int main() { char buf[65536]; while (read(0, buf, 65536) > 0) continue; } -Note: performance results may depend on warm/cold runs, compilers, libraries, +Performance results may depend on warm/cold runs, compilers, libraries, the OS, the CPU type, and file system latencies. However, comparable competitive results were obtained on many other types of machines. @@ -545,8 +560,8 @@ line instead of a sequence of `a`'s. By default in ugrep, the pattern `a*` matches a sequence of `a`'s. Moreover, in ugrep the pattern `a*b*c*` matches what it is supposed to match by default. See [improvements](#improvements). -- `-.` searches hidden files (dotfiles). By default, hidden files are ignored - by defaullt, like most Unix utilities such as `ls`. +- `-.` searches hidden files (dotfiles). By default, hidden files are ignored, + like most Unix utilities. - `-Dread` and `-dread` are the GNU/BSD grep defaults but are not recommended, see [improvements](#improvements) for an explanation. @@ -618,11 +633,11 @@ - **ugrep** searches pdf, doc, docx, xls, xlsx, epub, and more with `--filter` using third-party format conversion utilities as plugins. - **ugrep** offers *negative patterns* `-N PATTERN`, which are patterns of the - form `(?^X)` to skip input that matches `X`. Negative patterns can be used - to skip strings and comments when searching for identifiers in source code - and find matches that aren't in strings and comments. Predefined `zap` - patterns use nagative patterns, for example, use `-f cpp/zap_comments` to - ignore pattern matches in C++ comments. + form `(?^X)` that skip all `X` input, thus removing `X` from the search. + For example, negative patterns can be used to skip strings and comments when + searching for identifiers in source code and find matches that aren't in + strings and comments. Predefined `zap` patterns use nagative patterns, for + example, use `-f cpp/zap_comments` to ignore pattern matches in C++ comments. - **ugrep** option `-f` uses `GREP_PATH` environment variable or the predefined patterns installed in `/usr/local/share/ugrep/patterns`. If `-f` is specified and also one or more `-e` patterns are specified, then options @@ -642,11 +657,11 @@ - **ugrep** option `-b` with option `-o` or with option `-u`, **ugrep** displays the exact byte offset of the pattern match instead of the byte offset of the start of the matched line reported by GNU/BSD grep. -- **ugrep** option `-u`, `--ungroup` to not group matches per line. This - option displays a matched input line again for each additional pattern match - on the line. This option is particularly useful with option `-c` to report - the total number of pattern matches per file instead of the number of lines - matched per file. +- **ugrep** option `-u`, `--ungroup` to not group multiple matches per line. + This option displays a matched input line again for each additional pattern + match on the line. This option is particularly useful with option `-c` to + report the total number of pattern matches per file instead of the number of + lines matched per file. - **ugrep** option `-Y` enables matching empty patterns. Grepping with empty-matching patterns is weird and gives different results with GNU grep versus BSD grep. Empty matches are not output by **ugrep** by default, which @@ -888,6 +903,10 @@ ugrep -tlist +In the interactive query UI, press F1 or CTRL-Z for help and options: + + ugrep -Q + 🔝 [Back to table of contents](#toc) <a name="config"/> @@ -906,7 +925,7 @@ #### The ug command versus the ugrep command The `ug` command is intended for context-dependent interactive searching and is -equivalent to the `grep --config` command to load the configuration file +equivalent to the `ugrep --config` command to load the configuration file `.ugrep`, when present in the working directory or, when not found, in the home directory: @@ -982,8 +1001,10 @@ editor to edit the file shown on screen. The editor is taken from the environment variable GREP_EDIT if defined, or EDITOR. Press Tab and Shift-Tab to navigate directories and to select a file to - search. Press Enter to select lines to output. Press Alt-l for - option -l to list files, Alt-n for -n, etc. Enables --heading. + search. Press Enter to select lines to output. Press ALT-l for + option -l to list files, ALT-n for -n, etc. Non-option commands + include ALT-] to increase fuzziness and ALT-} to increase context. + Press F1 or CTRL-Z for more information. Enables --heading. --no-confirm Do not confirm actions in -Q query mode. The default is confirm. @@ -998,9 +1019,9 @@ - Press Alt-g to enter or edit option `-g` file and directory matching globs, a comma-separated list of gitignore-style glob patterns. Press ESC to return control to the query pattern prompt (the globs are saved). When a glob is - preceded by a `!' or a `^', skips files whose name matches the glob When a - glob contains a `/', full pathnames are matched. Otherwise basenames are - matched. When a glob ends with a `/', directories are matched. + preceded by a `!` or a `^`, skips files whose name matches the glob When a + glob contains a `/`, full pathnames are matched. Otherwise basenames are + matched. When a glob ends with a `/`, directories are matched. - The query UI prompt switches between `Q>` (normal), `F>` (fixed strings), `G>` (basic regex), `P>` (Perl matching), and `Z>` (fuzzy matching). When the `--glob=` prompt is shown, a comma-separated list of gitignore-style @@ -3713,8 +3734,10 @@ screen. The editor is taken from the environment variable GREP_EDIT if defined, or EDITOR. Press Tab and Shift-Tab to navigate directories and to select a file to search. Press - Enter to select lines to output. Press Alt-l for option -l to - list files, Alt-n for -n, etc. Enables --heading. + Enter to select lines to output. Press ALT-l for option -l to + list files, ALT-n for -n, etc. Non-option commands include + ALT-] to increase fuzziness and ALT-} to increase context. + Press F1 or CTRL-Z for more information. Enables --heading. -q, --quiet, --silent Quiet mode: suppress all output. ugrep will only search until a @@ -4371,7 +4394,7 @@ - ugrep 2.5.6 September 21, 2020 UGREP(1) + ugrep 3.0.1 October 03, 2020 UGREP(1) 🔝 [Back to table of contents](#toc) Binary files old/ugrep-2.5.6/bin/win32/ugrep.exe and new/ugrep-3.0.2/bin/win32/ugrep.exe differ Binary files old/ugrep-2.5.6/bin/win64/ugrep.exe and new/ugrep-3.0.2/bin/win64/ugrep.exe differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/config.h.in new/ugrep-3.0.2/config.h.in --- old/ugrep-2.5.6/config.h.in 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/config.h.in 2020-10-13 16:59:53.000000000 +0200 @@ -10,6 +10,9 @@ */ #undef HAVE_DIRENT_H +/* Define if F_RDAHEAD fcntl() is supported */ +#undef HAVE_F_RDAHEAD + /* Define to 1 if you have the `getpagesize' function. */ #undef HAVE_GETPAGESIZE @@ -37,6 +40,9 @@ /* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */ #undef HAVE_NDIR_H +/* Define if O_NOATIME open flag is supported */ +#undef HAVE_O_NOATIME + /* Define to 1 if you have `PCRE2' library (-lpcre2-8) */ #undef HAVE_PCRE2 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/configure.ac new/ugrep-3.0.2/configure.ac --- old/ugrep-2.5.6/configure.ac 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/configure.ac 2020-10-13 16:59:53.000000000 +0200 @@ -1,4 +1,4 @@ -AC_INIT(ugrep,2.5) +AC_INIT(ugrep,3.0) AM_INIT_AUTOMAKE([foreign]) AM_CONFIG_HEADER(config.h) AC_COPYRIGHT([Copyright (C) 2019-2020 Robert van Engelen, Genivia Inc.]) @@ -38,11 +38,29 @@ PLATFORM=${host} AC_SUBST(PLATFORM) -# Mac OS X has slow memory mapped files so we're disabling mmap -case "$PLATFORM" in - *-apple-*) EXTRA_CFLAGS="-DWITH_NO_MMAP -DWITH_MACOS_META_KEY ${EXTRA_CFLAGS}" - ;; -esac +# F_RDAHEAD fcntl() +AC_MSG_CHECKING(for F_RDAHEAD fcntl) +AC_TRY_COMPILE([ +#include <fcntl.h> +], +[ int cmd = F_RDAHEAD; ], +[ + AC_DEFINE(HAVE_F_RDAHEAD,1,[ Define if F_RDAHEAD fcntl() is supported]) + AC_MSG_RESULT(yes) +], AC_MSG_RESULT(no) +) + +# O_NOATIME open flag +AC_MSG_CHECKING(for O_NOATIME open flag) +AC_TRY_COMPILE([ +#include <fcntl.h> +], +[ int cmd = O_NOATIME; ], +[ + AC_DEFINE(HAVE_O_NOATIME,1,[ Define if O_NOATIME open flag is supported]) + AC_MSG_RESULT(yes) +], AC_MSG_RESULT(no) +) AX_CHECK_PCRE2([8], [], [echo "checking for Boost.Regex because PCRE2 is not usable"]) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/include/reflex/absmatcher.h new/ugrep-3.0.2/include/reflex/absmatcher.h --- old/ugrep-2.5.6/include/reflex/absmatcher.h 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/include/reflex/absmatcher.h 2020-10-13 16:59:53.000000000 +0200 @@ -101,7 +101,7 @@ static const int UNK = 256; ///< unknown/undefined character meta-char marker static const int BOB = 257; ///< begin of buffer meta-char marker static const int EOB = EOF; ///< end of buffer meta-char marker - static const size_t BLOCK = 8192; ///< buffer growth factor, buffer is initially 2*BLOCK size, at least 256 + static const size_t BLOCK = (256*1024); ///< buffer size and growth, buffer is initially 2*BLOCK size, at least 4096 bytes static const size_t REDO = 0x7FFFFFFF; ///< reflex::Matcher::accept() returns "redo" with reflex::Matcher option "A" static const size_t EMPTY = 0xFFFFFFFF; ///< accept() returns "empty" last split at end of input }; @@ -299,7 +299,11 @@ if (own_) { #if defined(WITH_REALLOC) +#if (defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(__BORLANDC__)) && !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__MINGW64__) + _aligned_free(static_cast<void*>(buf_)); +#else std::free(static_cast<void*>(buf_)); +#endif #else delete[] buf_; #endif @@ -337,10 +341,19 @@ } if (!own_) { + max_ = 2 * Const::BLOCK; #if defined(WITH_REALLOC) - buf_ = static_cast<char*>(std::malloc(max_ = 2 * Const::BLOCK)); +#if (defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(__BORLANDC__)) && !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__MINGW64__) + buf_ = static_cast<char*>(_aligned_malloc(max_, 4096)); + if (buf_ == NULL) + throw std::bad_alloc(); +#else + buf_ = NULL; + if (::posix_memalign(reinterpret_cast<void**>(&buf_), 4096, max_) != 0) + throw std::bad_alloc(); +#endif #else - buf_ = new char[max_ = 2 * Const::BLOCK]; + buf_ = new char[max_]; #endif } buf_[0] = '\0'; @@ -400,18 +413,18 @@ evh_ = handler; } /// Get the buffered context before the matching line. - Context before() + inline Context before() { (void)lineno(); return Context(buf_, bol_ - buf_, num_); } /// Get the buffered context after EOF is reached. - Context after() + inline Context after() { if (hit_end()) { (void)lineno(); - // if there is no \n at the end of input: increase line count by one + // if there is no \n at the end of input: increase line count by one to compensate if (bol_ < txt_) ++lno_; return Context(buf_, end_, num_); @@ -466,7 +479,11 @@ if (own_) { #if defined(WITH_REALLOC) +#if (defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(__BORLANDC__)) && !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__MINGW64__) + _aligned_free(static_cast<void*>(buf_)); +#else std::free(static_cast<void*>(buf_)); +#endif #else delete[] buf_; #endif @@ -501,7 +518,7 @@ } /// Returns nonzero capture index (i.e. true) if the entire input matches this matcher's pattern (and internally caches the true/false result to permit repeat invocations). - size_t matches() + inline size_t matches() /// @returns nonzero capture index (i.e. true) if the entire input matched this matcher's pattern, zero (i.e. false) otherwise { if (!mat_ && at_bob()) @@ -509,25 +526,25 @@ return mat_; } /// Returns a positive integer (true) indicating the capture index of the matched text in the pattern or zero (false) for a mismatch. - size_t accept() const + inline size_t accept() const /// @returns nonzero capture index of the match in the pattern, which may be matcher dependent, or zero for a mismatch, or Const::EMPTY for the empty last split { return cap_; } /// Returns pointer to the begin of the matched text (non-0-terminated), a constant-time operation, use with end() or use size() for text end/length. - const char *begin() const + inline const char *begin() const /// @returns const char* pointer to the matched text in the buffer { return txt_; } /// Returns pointer to the exclusive end of the matched text, a constant-time operation. - const char *end() const + inline const char *end() const /// @returns const char* pointer to the exclusive end of the matched text in the buffer { return txt_ + len_; } /// Returns 0-terminated string of the text matched, does not include matched \0s, this is a constant-time operation. - const char *text() + inline const char *text() /// @returns 0-terminated const char* string with text matched { if (chr_ == '\0') @@ -538,25 +555,25 @@ return txt_; } /// Returns the text matched as a string, a copy of text(), may include matched \0s. - std::string str() const + inline std::string str() const /// @returns string with text matched { return std::string(txt_, len_); } /// Returns the match as a wide string, converted from UTF-8 text(), may include matched \0s. - std::wstring wstr() const + inline std::wstring wstr() const /// @returns wide string with text matched { return wcs(txt_, len_); } /// Returns the length of the matched text in number of bytes, including matched \0s, a constant-time operation. - size_t size() const + inline size_t size() const /// @returns match size in bytes { return len_; } /// Returns the length of the matched text in number of wide characters. - size_t wsize() const + inline size_t wsize() const /// @returns the length of the match in number of wide (multibyte UTF-8) characters { size_t n = 0; @@ -566,40 +583,150 @@ return n; } /// Returns the first 8-bit character of the text matched. - int chr() const + inline int chr() const /// @returns 8-bit char { return *txt_; } /// Returns the first wide character of the text matched. - int wchr() const + inline int wchr() const /// @returns wide char (UTF-8 converted to Unicode) { return utf8(txt_); } /// Set or change the starting line number of the last match. - void lineno(size_t n) ///< new line number + inline void lineno(size_t n) ///< new line number { if (lpb_ < txt_) (void)lineno(); // update lno_ and bol_ (or cno_) before overriding lno_ lno_ = n; } /// Updates and returns the starting line number of the match in the input character sequence. - size_t lineno() + inline size_t lineno() /// @returns line number { #if defined(WITH_SPAN) if (lpb_ < txt_) { - while (bol_ < txt_) + char *s = bol_; + char *t = txt_; + // clang/gcc 4-way vectorizable loop + while (t - 4 >= s) { - char *s = static_cast<char*>(std::memchr(bol_, '\n', txt_ - bol_)); - if (s == NULL) + if ((t[-1] == '\n') | (t[-2] == '\n') | (t[-3] == '\n') | (t[-4] == '\n')) break; - ++lno_; - bol_ = s + 1; + t -= 4; } + // epilogue + if (--t >= s && *t != '\n') + if (--t >= s && *t != '\n') + if (--t >= s && *t != '\n') + --t; + bol_ = t + 1; lpb_ = txt_; + size_t n = lno_; +#if defined(HAVE_AVX512BW) && (!defined(_MSC_VER) || defined(_WIN64)) + if (have_HW_AVX512BW()) + { + __m512i vlcn = _mm512_set1_epi8('\n'); + while (s + 63 <= t) + { + __m512i vlcm = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(s)); + uint64_t mask = _mm512_cmpeq_epi8_mask(vlcm, vlcn); + n += popcountl(mask); + s += 64; + } + } + else if (have_HW_AVX2()) + { + __m256i vlcn = _mm256_set1_epi8('\n'); + while (s + 31 <= t) + { + __m256i vlcm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s)); + __m256i vlceq = _mm256_cmpeq_epi8(vlcm, vlcn); + uint32_t mask = _mm256_movemask_epi8(vlceq); + n += popcount(mask); + s += 32; + } + } + else if (have_HW_SSE2()) + { + __m128i vlcn = _mm_set1_epi8('\n'); + while (s + 15 <= t) + { + __m128i vlcm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s)); + __m128i vlceq = _mm_cmpeq_epi8(vlcm, vlcn); + uint32_t mask = _mm_movemask_epi8(vlceq); + n += popcount(mask); + s += 16; + } + } +#elif defined(HAVE_AVX2) + if (have_HW_AVX2()) + { + __m256i vlcn = _mm256_set1_epi8('\n'); + while (s + 31 <= t) + { + __m256i vlcm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s)); + __m256i vlceq = _mm256_cmpeq_epi8(vlcm, vlcn); + uint32_t mask = _mm256_movemask_epi8(vlceq); + n += popcount(mask); + s += 32; + } + } + else if (have_HW_SSE2()) + { + __m128i vlcn = _mm_set1_epi8('\n'); + while (s + 15 <= t) + { + __m128i vlcm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s)); + __m128i vlceq = _mm_cmpeq_epi8(vlcm, vlcn); + uint32_t mask = _mm_movemask_epi8(vlceq); + n += popcount(mask); + s += 16; + } + } +#elif defined(HAVE_SSE2) + if (have_HW_SSE2()) + { + __m128i vlcn = _mm_set1_epi8('\n'); + while (s + 15 <= t) + { + __m128i vlcm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s)); + __m128i vlceq = _mm_cmpeq_epi8(vlcm, vlcn); + uint32_t mask = _mm_movemask_epi8(vlceq); + n += popcount(mask); + s += 16; + } + } +#elif defined(HAVE_NEON) + { + // ARM AArch64/NEON SIMD optimized loop? - no code found yet that runs faster than the code below + } +#endif + uint32_t n0 = 0, n1 = 0, n2 = 0, n3 = 0; + // clang/gcc 4-way vectorizable loop + while (s + 3 <= t) + { + n0 += s[0] == '\n'; + n1 += s[1] == '\n'; + n2 += s[2] == '\n'; + n3 += s[3] == '\n'; + s += 4; + } + n += n0 + n1 + n2 + n3; + // epilogue + if (s <= t) + { + n += *s == '\n'; + if (++s <= t) + { + n += *s == '\n'; + if (++s <= t) + n += *s == '\n'; + } + } + lno_ = n; } #else size_t n = lno_; @@ -632,7 +759,7 @@ return lno_; } /// Returns the number of lines that the match spans. - size_t lines() + inline size_t lines() /// @returns number of lines { size_t n = 1; @@ -642,13 +769,13 @@ return n; } /// Returns the inclusive ending line number of the match in the input character sequence. - size_t lineno_end() + inline size_t lineno_end() /// @returns line number { return lineno() + lines() - 1; } /// Updates and returns the starting column number of the matched text, taking tab spacing into account and counting wide characters as one character each - size_t columno() + inline size_t columno() /// @returns column number { (void)lineno(); @@ -670,7 +797,7 @@ #endif } /// Returns the number of columns of the matched text, taking tab spacing into account and counting wide characters as one character each. - size_t columns() + inline size_t columns() /// @returns number of columns { // count columns in tabs and UTF-8 chars @@ -717,7 +844,7 @@ } #if defined(WITH_SPAN) /// Returns the inclusive ending column number of the matched text on the ending matching line, taking tab spacing into account and counting wide characters as one character each - size_t columno_end() + inline size_t columno_end() /// @returns column number { if (len_ == 0) @@ -1244,7 +1371,7 @@ /// @returns nonzero when input matched the pattern using method Const::SCAN, Const::FIND, Const::SPLIT, or Const::MATCH = 0; /// Shift or expand the internal buffer when it is too small to accommodate more input, where the buffer size is doubled when needed, change cur_, pos_, end_, max_, ind_, buf_, bol_, lpb_, and txt_. - bool grow(size_t need = Const::BLOCK) ///< optional needed space = Const::BLOCK size by default + inline bool grow(size_t need = Const::BLOCK) ///< optional needed space = Const::BLOCK size by default /// @returns true if buffer was shifted or enlarged { if (max_ - end_ >= need + 1) @@ -1280,7 +1407,11 @@ max_ *= 2; DBGLOG("Expand buffer to %zu bytes", max_); #if defined(WITH_REALLOC) +#if (defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(__BORLANDC__)) && !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__MINGW64__) + char *newbuf = static_cast<char*>(_aligned_realloc(static_cast<void*>(buf_), max_, 4096)); +#else char *newbuf = static_cast<char*>(std::realloc(static_cast<void*>(buf_), max_)); +#endif if (newbuf == NULL) throw std::bad_alloc(); #else @@ -1326,7 +1457,11 @@ num_ += gap; #if defined(WITH_REALLOC) std::memmove(buf_, txt_, end_); +#if (defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(__BORLANDC__)) && !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__MINGW64__) + char *newbuf = static_cast<char*>(_aligned_realloc(static_cast<void*>(buf_), max_, 4096)); +#else char *newbuf = static_cast<char*>(std::realloc(static_cast<void*>(buf_), max_)); +#endif if (newbuf == NULL) throw std::bad_alloc(); #else @@ -1397,7 +1532,7 @@ txt_ = buf_ + cur_; } /// Get the next character and grow the buffer to make more room if necessary. - int get_more() + inline int get_more() /// @returns the character read (unsigned char 0..255) or EOF (-1) { DBGLOG("AbstractMatcher::get_more()"); @@ -1419,7 +1554,7 @@ } } /// Peek at the next character and grow the buffer to make more room if necessary. - int peek_more() + inline int peek_more() /// @returns the character (unsigned char 0..255) or EOF (-1) { DBGLOG("AbstractMatcher::peek_more()"); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/include/reflex/input.h new/ugrep-3.0.2/include/reflex/input.h --- old/ugrep-2.5.6/include/reflex/input.h 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/include/reflex/input.h 2020-10-13 16:59:53.000000000 +0200 @@ -28,7 +28,7 @@ /** @file input.h -@brief RE/flex input character sequence class +@brief RE/flex input character sequence class and HW accelleration @author Robert van Engelen - enge...@genivia.com @copyright (c) 2016-2020, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt @@ -43,6 +43,22 @@ #include <iostream> #include <string> +#if defined(HAVE_AVX512BW) +# include <immintrin.h> +#elif defined(HAVE_AVX2) +# include <immintrin.h> +#elif defined(HAVE_SSE2) +# include <emmintrin.h> +#elif defined(HAVE_NEON) +# include <arm_neon.h> +#endif + +#if defined(HAVE_AVX512BW) || defined(HAVE_AVX2) || defined(HAVE_SSE2) +# ifdef _MSC_VER +# include <intrin.h> +# endif +#endif + namespace reflex { /// Input character sequence class for unified access to sources of input text. @@ -1149,6 +1165,67 @@ int ch2_; }; +#ifdef _MSC_VER +#pragma intrinsic(_BitScanForward) +inline uint32_t ctz(uint32_t x) +{ + unsigned long r; + _BitScanForward(&r, x); + return r; +} +inline uint32_t popcount(uint32_t x) +{ + return __popcnt(x); +} +#ifdef _WIN64 +#pragma intrinsic(_BitScanForward64) +inline uint32_t ctzl(uint64_t x) +{ + unsigned long r; + _BitScanForward64(&r, x); + return r; +} +inline uint32_t popcountl(uint64_t x) +{ + return static_cast<uint32_t>(__popcnt64(x)); +} +#endif +#else +inline uint32_t ctz(uint32_t x) +{ + return __builtin_ctz(x); +} +inline uint32_t ctzl(uint64_t x) +{ + return __builtin_ctzl(x); +} +inline uint32_t popcount(uint32_t x) +{ + return __builtin_popcount(x); +} +inline uint32_t popcountl(uint64_t x) +{ + return __builtin_popcountl(x); +} +#endif + +extern uint64_t HW; + +inline bool have_HW_AVX512BW() +{ + return HW & (1ULL << 62); +} + +inline bool have_HW_AVX2() +{ + return HW & (1ULL << 37); +} + +inline bool have_HW_SSE2() +{ + return HW & (1ULL << 26); +} + } // namespace reflex #endif diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/include/reflex/matcher.h new/ugrep-3.0.2/include/reflex/matcher.h --- old/ugrep-2.5.6/include/reflex/matcher.h 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/include/reflex/matcher.h 2020-10-13 16:59:53.000000000 +0200 @@ -376,21 +376,6 @@ { return isword(got_) == isword(static_cast<unsigned char>(txt_[len_])); } - /// Check CPU hardware for AVX512BW capability. - static bool have_HW_AVX512BW() - { - return HW & (1ULL << 62); - } - /// Check CPU hardware for AVX2 capability. - static bool have_HW_AVX2() - { - return HW & (1ULL << 37); - } - /// Check CPU hardware for SSE2 capability. - static bool have_HW_SSE2() - { - return HW & (1ULL << 26); - } protected: typedef std::vector<size_t> Stops; ///< indent margin/tab stops /// FSM data for FSM code @@ -400,10 +385,6 @@ bool nul; int c1; }; - /// Get CPU hardware info. - static uint64_t get_HW(); - /// CPU hardware info[2] - static uint64_t HW; /// Returns true if input matched the pattern using method Const::SCAN, Const::FIND, Const::SPLIT, or Const::MATCH. virtual size_t match(Method method) ///< Const::SCAN, Const::FIND, Const::SPLIT, or Const::MATCH /// @returns nonzero if input matched the pattern diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/lib/input.cpp new/ugrep-3.0.2/lib/input.cpp --- old/ugrep-2.5.6/lib/input.cpp 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/lib/input.cpp 2020-10-13 16:59:53.000000000 +0200 @@ -1346,4 +1346,39 @@ } } +#if defined(HAVE_AVX512BW) || defined(HAVE_AVX2) || defined(HAVE_SSE2) + +#ifdef _MSC_VER +# include <intrin.h> +# define cpuidex __cpuidex +#else +# include <cpuid.h> +# define cpuidex(CPUInfo, id, subid) __cpuid_count(id, subid, CPUInfo[0], CPUInfo[1], CPUInfo[2], CPUInfo[3]) +#endif + +static uint64_t get_HW() +{ + int CPUInfo1[4] = { 0, 0, 0, 0 }; + int CPUInfo7[4] = { 0, 0, 0, 0 }; + cpuidex(CPUInfo1, 0, 0); + int n = CPUInfo1[0]; + if (n <= 0) + return 0ULL; + cpuidex(CPUInfo1, 1, 0); // cpuid EAX=1 + if (n >= 7) + cpuidex(CPUInfo7, 7, 0); // cpuid EAX=7, ECX=0 + return static_cast<uint32_t>(CPUInfo1[2]) | (static_cast<uint64_t>(static_cast<uint32_t>(CPUInfo7[1])) << 32); +} + +#else + +static uint64_t get_HW() +{ + return 0ULL; +} + +#endif + +uint64_t HW = get_HW(); + } // namespace reflex diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/lib/matcher.cpp new/ugrep-3.0.2/lib/matcher.cpp --- old/ugrep-2.5.6/lib/matcher.cpp 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/lib/matcher.cpp 2020-10-13 16:59:53.000000000 +0200 @@ -36,83 +36,8 @@ #include <reflex/matcher.h> -#if defined(HAVE_AVX512BW) -# include <immintrin.h> -#elif defined(HAVE_AVX2) -# include <immintrin.h> -#elif defined(HAVE_SSE2) -# include <emmintrin.h> -#elif defined(HAVE_NEON) -# include <arm_neon.h> -#endif - -#if defined(HAVE_AVX512BW) || defined(HAVE_AVX2) || defined(HAVE_SSE2) -# ifdef _MSC_VER -# include <intrin.h> -# define cpuidex __cpuidex -# else -# include <cpuid.h> -# define cpuidex(CPUInfo, id, subid) __cpuid_count(id, subid, CPUInfo[0], CPUInfo[1], CPUInfo[2], CPUInfo[3]) -# endif -#endif - namespace reflex { -#if defined(HAVE_AVX512BW) || defined(HAVE_AVX2) || defined(HAVE_SSE2) - -#ifdef _MSC_VER -#pragma intrinsic(_BitScanForward) -inline uint32_t ctz(uint32_t x) -{ - unsigned long r; - _BitScanForward(&r, x); - return r; -} -#ifdef _WIN64 -#pragma intrinsic(_BitScanForward64) -inline uint32_t ctzl(uint64_t x) -{ - unsigned long r; - _BitScanForward64(&r, x); - return r; -} -#endif -#else -inline uint32_t ctz(uint32_t x) -{ - return __builtin_ctz(x); -} -inline uint32_t ctzl(uint64_t x) -{ - return __builtin_ctzl(x); -} -#endif - -uint64_t Matcher::get_HW() -{ - int CPUInfo1[4] = { 0, 0, 0, 0 }; - int CPUInfo7[4] = { 0, 0, 0, 0 }; - cpuidex(CPUInfo1, 0, 0); - int n = CPUInfo1[0]; - if (n <= 0) - return 0ULL; - cpuidex(CPUInfo1, 1, 0); // cpuid EAX=1 - if (n >= 7) - cpuidex(CPUInfo7, 7, 0); // cpuid EAX=7, ECX=0 - return static_cast<uint32_t>(CPUInfo1[2]) | (static_cast<uint64_t>(static_cast<uint32_t>(CPUInfo7[1])) << 32); -} - -#else - -uint64_t Matcher::get_HW() -{ - return 0ULL; -} - -#endif - -uint64_t Matcher::HW = Matcher::get_HW(); - /// Boyer-Moore preprocessing of the given pattern prefix pat of length len (<=255), generates bmd_ > 0 and bms_[] shifts. void Matcher::boyer_moore_init(const char *pat, size_t len) { @@ -180,7 +105,7 @@ if (loc + min > end_) { set_current_match(loc - 1); - peek_more(); + (void)peek_more(); loc = cur_ + 1; if (loc + min > end_) return false; @@ -216,7 +141,7 @@ { loc = s - buf_; set_current_match(loc - min); - peek_more(); + (void)peek_more(); loc = cur_ + min; if (loc >= end_) return false; @@ -254,7 +179,7 @@ { loc = s - buf_; set_current_match(loc - 3); - peek_more(); + (void)peek_more(); loc = cur_ + 3; if (loc >= end_) return false; @@ -291,7 +216,7 @@ { loc = s - buf_; set_current_match(loc - 2); - peek_more(); + (void)peek_more(); loc = cur_ + 2; if (loc >= end_) return false; @@ -324,7 +249,7 @@ { loc = s - buf_; set_current_match(loc - 1); - peek_more(); + (void)peek_more(); loc = cur_ + 1; if (loc >= end_) return false; @@ -348,7 +273,7 @@ } loc = e - buf_; set_current_match(loc - 1); - peek_more(); + (void)peek_more(); loc = cur_ + 1; if (loc + len > end_) return false; @@ -368,7 +293,7 @@ // implements AVX512 string search scheme based on in http://0x80.pl/articles/simd-friendly-karp-rabin.html __m512i vlcp = _mm512_set1_epi8(pre[lcp_]); __m512i vlcs = _mm512_set1_epi8(pre[lcs_]); - while (s + 64 < e) + while (s + 64 <= e) { __m512i vlcpm = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(s)); __m512i vlcsm = _mm512_loadu_si512(reinterpret_cast<const __m512i*>(s + lcs_ - lcp_)); @@ -393,7 +318,7 @@ return true; } } - mask &= ~(1ULL << offset); + mask &= mask - 1; } s += 64; } @@ -403,7 +328,7 @@ // implements AVX2 string search scheme based on in http://0x80.pl/articles/simd-friendly-karp-rabin.html __m256i vlcp = _mm256_set1_epi8(pre[lcp_]); __m256i vlcs = _mm256_set1_epi8(pre[lcs_]); - while (s + 32 < e) + while (s + 32 <= e) { __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s)); __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs_ - lcp_)); @@ -430,7 +355,7 @@ return true; } } - mask &= ~(1 << offset); + mask &= mask - 1; } s += 32; } @@ -440,7 +365,7 @@ // implements SSE2 string search scheme based on in http://0x80.pl/articles/simd-friendly-karp-rabin.html __m128i vlcp = _mm_set1_epi8(pre[lcp_]); __m128i vlcs = _mm_set1_epi8(pre[lcs_]); - while (s + 16 < e) + while (s + 16 <= e) { __m128i vlcpm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s)); __m128i vlcsm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + lcs_ - lcp_)); @@ -467,7 +392,7 @@ return true; } } - mask &= ~(1 << offset); + mask &= mask - 1; } s += 16; } @@ -478,7 +403,7 @@ // implements AVX2 string search scheme based on in http://0x80.pl/articles/simd-friendly-karp-rabin.html __m256i vlcp = _mm256_set1_epi8(pre[lcp_]); __m256i vlcs = _mm256_set1_epi8(pre[lcs_]); - while (s + 32 < e) + while (s + 32 <= e) { __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s)); __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs_ - lcp_)); @@ -505,7 +430,7 @@ return true; } } - mask &= ~(1 << offset); + mask &= mask - 1; } s += 32; } @@ -515,7 +440,7 @@ // implements SSE2 string search scheme based on in http://0x80.pl/articles/simd-friendly-karp-rabin.html __m128i vlcp = _mm_set1_epi8(pre[lcp_]); __m128i vlcs = _mm_set1_epi8(pre[lcs_]); - while (s + 16 < e) + while (s + 16 <= e) { __m128i vlcpm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s)); __m128i vlcsm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + lcs_ - lcp_)); @@ -542,7 +467,7 @@ return true; } } - mask &= ~(1 << offset); + mask &= mask - 1; } s += 16; } @@ -553,7 +478,7 @@ // implements SSE2 string search scheme based on in http://0x80.pl/articles/simd-friendly-karp-rabin.html __m128i vlcp = _mm_set1_epi8(pre[lcp_]); __m128i vlcs = _mm_set1_epi8(pre[lcs_]); - while (s + 16 < e) + while (s + 16 <= e) { __m128i vlcpm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s)); __m128i vlcsm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + lcs_ - lcp_)); @@ -580,7 +505,7 @@ return true; } } - mask &= ~(1 << offset); + mask &= mask - 1; } s += 16; } @@ -589,7 +514,7 @@ // implements NEON/AArch64 string search scheme based on in http://0x80.pl/articles/simd-friendly-karp-rabin.html but 64 bit optimized uint8x16_t vlcp = vdupq_n_u8(pre[lcp_]); uint8x16_t vlcs = vdupq_n_u8(pre[lcs_]); - while (s + 16 < e) + while (s + 16 <= e) { uint8x16_t vlcpm = vld1q_u8(reinterpret_cast<const uint8_t*>(s)); uint8x16_t vlcsm = vld1q_u8(reinterpret_cast<const uint8_t*>(s) + lcs_ - lcp_); @@ -681,7 +606,7 @@ } loc = s - lcp_ - buf_; set_current_match(loc - 1); - peek_more(); + (void)peek_more(); loc = cur_ + 1; if (loc + len > end_) return false; @@ -740,7 +665,7 @@ s -= len - 1; loc = s - buf_; set_current_match(loc - 1); - peek_more(); + (void)peek_more(); loc = cur_ + 1; if (loc + len > end_) return false; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/lib/pattern.cpp new/ugrep-3.0.2/lib/pattern.cpp --- old/ugrep-2.5.6/lib/pattern.cpp 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/lib/pattern.cpp 2020-10-13 16:59:53.000000000 +0200 @@ -1829,7 +1829,7 @@ Moves::iterator end = moves.end(); while (i != end) { - if (is_subset(i->second, follow)) + if (i->second == follow) { chars += i->first; moves.erase(i++); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/man/ugrep.1 new/ugrep-3.0.2/man/ugrep.1 --- old/ugrep-2.5.6/man/ugrep.1 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/man/ugrep.1 2020-10-13 16:59:53.000000000 +0200 @@ -1,4 +1,4 @@ -.TH UGREP "1" "September 21, 2020" "ugrep 2.5.6" "User Commands" +.TH UGREP "1" "October 03, 2020" "ugrep 3.0.1" "User Commands" .SH NAME \fBugrep\fR, \fBug\fR -- file pattern searcher .SH SYNOPSIS @@ -480,8 +480,10 @@ editor to edit the file shown on screen. The editor is taken from the environment variable GREP_EDIT if defined, or EDITOR. Press Tab and Shift\-Tab to navigate directories and to select a file to -search. Press Enter to select lines to output. Press Alt\-l for -option \fB\-l\fR to list files, Alt\-n for \fB\-n\fR, etc. Enables \fB\-\-heading\fR. +search. Press Enter to select lines to output. Press ALT\-l for +option \fB\-l\fR to list files, ALT\-n for \fB\-n\fR, etc. Non\-option commands +include ALT\-] to increase fuzziness and ALT\-} to increase context. +Press F1 or CTRL\-Z for more information. Enables \fB\-\-heading\fR. .TP \fB\-q\fR, \fB\-\-quiet\fR, \fB\-\-silent\fR Quiet mode: suppress all output. ugrep will only search until a diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/src/mmap.hpp new/ugrep-3.0.2/src/mmap.hpp --- old/ugrep-2.5.6/src/mmap.hpp 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/src/mmap.hpp 2020-10-13 16:59:53.000000000 +0200 @@ -44,7 +44,7 @@ # define MIN_MMAP_SIZE 16384LL // 16KB at minimum, smaller files are efficiently read in one go with read() #endif #ifndef MAX_MMAP_SIZE -# define MAX_MMAP_SIZE 1073741824LL // in the worst case each worker thread may use up to 1GB mmap space but not more +# define MAX_MMAP_SIZE 1073741824LL // each worker thread may use up to 1GB mmap space but not more #endif #if defined(HAVE_MMAP) && MAX_MMAP_SIZE > 0 @@ -87,12 +87,12 @@ // is this a regular file that is not too large (for size_t)? int fd = fileno(file); struct stat buf; - if (fstat(fd, &buf) != 0 || !S_ISREG(buf.st_mode) || static_cast<uint64_t>(buf.st_size) > static_cast<uint64_t>(std::numeric_limits<size_t>::max())) + if (fstat(fd, &buf) != 0 || !S_ISREG(buf.st_mode) || static_cast<uint64_t>(buf.st_size) < MIN_MMAP_SIZE || static_cast<uint64_t>(buf.st_size) > static_cast<uint64_t>(std::numeric_limits<size_t>::max())) return false; - // is this file not too small or too large? + // is this file not larger than --max-mmap? size = static_cast<size_t>(buf.st_size); - if (size < MIN_MMAP_SIZE || size > flag_max_mmap) + if (size > flag_max_mmap) return false; // mmap the file and round requested size up to 4K (typical page size) @@ -101,11 +101,17 @@ // allocate fixed mmap region to reuse mmap_size = (flag_max_mmap + 0xfff) & ~0xfffUL; mmap_base = mmap(NULL, mmap_size, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + + // files are sequentially read + if (mmap_base == MAP_FAILED) + mmap_base = NULL; + else + madvise(mmap_base, mmap_size, MADV_SEQUENTIAL); } if (mmap_base != NULL) { - // mmap file to the fixed mmap region + // mmap the (next) file to the fixed mmap region base = static_cast<const char*>(mmap_base = mmap(mmap_base, mmap_size, PROT_READ, MAP_FIXED | MAP_PRIVATE, fd, 0)); // mmap OK? diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/src/query.cpp new/ugrep-3.0.2/src/query.cpp --- old/ugrep-2.5.6/src/query.cpp 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/src/query.cpp 2020-10-13 16:59:53.000000000 +0200 @@ -60,8 +60,8 @@ HANDLE pipe_w = CreateFileA(pipe_name.c_str(), GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL); if (pipe_w != INVALID_HANDLE_VALUE) { - fd[0] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_r), 0); - fd[1] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_w), 0); + fd[0] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_r), _O_RDONLY); + fd[1] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_w), _O_WRONLY); } else { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/src/ugrep.cpp new/ugrep-3.0.2/src/ugrep.cpp --- old/ugrep-2.5.6/src/ugrep.cpp 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/src/ugrep.cpp 2020-10-13 16:59:53.000000000 +0200 @@ -67,7 +67,10 @@ */ // ugrep version -#define UGREP_VERSION "2.5.6" +#define UGREP_VERSION "3.0.2" + +// disable mmap because mmap is almost always slower than the file reading speed improvements since 3.0.0 +#define WITH_NO_MMAP #include "ugrep.hpp" #include "glob.hpp" @@ -232,7 +235,7 @@ // default --mmap #define DEFAULT_MIN_MMAP_SIZE MIN_MMAP_SIZE -// default --max-mmap: mmap is enabled by default, unless disabled with WITH_NO_MMAP, e.g. mmap is slow on MacOS +// default --max-mmap: mmap is disabled by default with WITH_NO_MMAP #ifdef WITH_NO_MMAP # define DEFAULT_MAX_MMAP_SIZE 0 #else @@ -1672,7 +1675,7 @@ // search a file virtual void search(const char *pathname); - // open a file for (binary) reading and assign input, decompress the file when --z, --decompress specified + // open a file for (binary) reading and assign input, decompress the file when -z, --decompress specified bool open_file(const char *pathname) { if (pathname == NULL) @@ -1683,12 +1686,48 @@ pathname = flag_label; file = source; } - else if (fopen_s(&file, pathname, (flag_binary || flag_decompress ? "rb" : "r")) != 0) + else +#ifdef OS_WIN + { + CREATEFILE2_EXTENDED_PARAMETERS params { + sizeof(CREATEFILE2_EXTENDED_PARAMETERS), + FILE_ATTRIBUTE_NORMAL, + FILE_FLAG_SEQUENTIAL_SCAN, + SECURITY_ANONYMOUS, + NULL, + NULL + }; + std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t> conversion; + std::wstring wpathname = conversion.from_bytes(pathname); + HANDLE hFile = CreateFile2(wpathname.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, ¶ms); + + if (hFile == INVALID_HANDLE_VALUE) + return false; + + int fd = _open_osfhandle(reinterpret_cast<intptr_t>(hFile), _O_RDONLY); + + if (fd == -1) + { + CloseHandle(hFile); + return false; + } + + file = _fdopen(fd, (flag_binary || flag_decompress ? "rb" : "r")); + + if (file == NULL) + { + _close(fd); + return false; + } + } +#else + if (fopen_s(&file, pathname, (flag_binary || flag_decompress ? "rb" : "r")) != 0) { warning("cannot read", pathname); return false; } +#endif // --filter: fork process to filter file, when applicable if (!filter(file, pathname)) @@ -3323,7 +3362,7 @@ { "swift", "swift", NULL }, { "tcl", "tcl,itcl,itk", NULL }, { "tex", "tex,cls,sty,bib", NULL }, - { "text", "text,txt,TXT,md", NULL }, + { "text", "text,txt,TXT,md,rst", NULL }, { "tiff", "tif,tiff", NULL }, { "Tiff", "tif,tiff", "\\x49\\x49\\x2a\\x00|\\x4d\\x4d\\x00\\x2a" }, { "tt", "tt,tt2,ttml", NULL }, @@ -3915,7 +3954,7 @@ else if (strcmp(arg, "max-count") == 0 || strcmp(arg, "max-files") == 0) usage("missing argument for --", arg); else - usage("invalid option --", arg, "--match, --max-count, --max-files or --mmap"); + usage("invalid option --", arg, "--match, --max-count, --max-files, --mmap or --messages"); break; case 'n': @@ -10019,8 +10058,10 @@ editor to edit the file shown on screen. The editor is taken from\n\ the environment variable GREP_EDIT if defined, or EDITOR. Press\n\ Tab and Shift-Tab to navigate directories and to select a file to\n\ - search. Press Enter to select lines to output. Press Alt-l for\n\ - option -l to list files, Alt-n for -n, etc. Enables --heading.\n\ + search. Press Enter to select lines to output. Press ALT-l for\n\ + option -l to list files, ALT-n for -n, etc. Non-option commands\n\ + include ALT-] to increase fuzziness and ALT-} to increase context.\n\ + Press F1 or CTRL-Z for more information. Enables --heading.\n\ -q, --quiet, --silent\n\ Quiet mode: suppress all output. ugrep will only search until a\n\ match has been found.\n\ @@ -10283,11 +10324,11 @@ #endif std::cout << "ugrep " UGREP_VERSION " " PLATFORM << #if defined(HAVE_AVX512BW) - (reflex::Matcher::have_HW_AVX512BW() ? " +avx512" : (reflex::Matcher::have_HW_AVX2() ? " +avx2" : reflex::Matcher::have_HW_SSE2() ? " +sse2" : " (-avx512)")) << + (reflex::have_HW_AVX512BW() ? " +avx512" : (reflex::have_HW_AVX2() ? " +avx2" : reflex::have_HW_SSE2() ? " +sse2" : " (-avx512)")) << #elif defined(HAVE_AVX2) - (reflex::Matcher::have_HW_AVX2() ? " +avx2" : reflex::Matcher::have_HW_SSE2() ? " +sse2" : " (-avx2)") << + (reflex::have_HW_AVX2() ? " +avx2" : reflex::have_HW_SSE2() ? " +sse2" : " (-avx2)") << #elif defined(HAVE_SSE2) - (reflex::Matcher::have_HW_SSE2() ? " +sse2" : " (-sse2)") << + (reflex::have_HW_SSE2() ? " +sse2" : " (-sse2)") << #elif defined(HAVE_NEON) " +neon" << #endif diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/ugrep-2.5.6/src/ugrep.hpp new/ugrep-3.0.2/src/ugrep.hpp --- old/ugrep-2.5.6/src/ugrep.hpp 2020-09-21 21:47:09.000000000 +0200 +++ new/ugrep-3.0.2/src/ugrep.hpp 2020-10-13 16:59:53.000000000 +0200 @@ -53,6 +53,7 @@ #include <io.h> #include <strsafe.h> #include <string.h> +#include <fcntl.h> #define STDIN_FILENO 0 #define STDOUT_FILENO 1 @@ -71,8 +72,8 @@ HANDLE pipe_w = NULL; if (CreatePipe(&pipe_r, &pipe_w, NULL, 0)) { - fd[0] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_r), 0); - fd[1] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_w), 0); + fd[0] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_r), _O_RDONLY); + fd[1] = _open_osfhandle(reinterpret_cast<intptr_t>(pipe_w), _O_WRONLY); return 0; } return -1; @@ -101,6 +102,8 @@ // not compiling for a windows OS +#define _FILE_OFFSET_BITS 64 + #include <cerrno> #include <cstdlib> #include <cstdio> @@ -114,6 +117,13 @@ # include <sys/statvfs.h> #endif +#if defined(HAVE_F_RDAHEAD) || defined(HAVE_O_NOATIME) +# include <fcntl.h> +# ifndef O_NOATIME +# define O_NOATIME 0 +# endif +#endif + #define PATHSEPCHR '/' #define PATHSEPSTR "/" @@ -132,6 +142,16 @@ // Windows-compatible fopen_s() inline int fopen_s(FILE **file, const char *filename, const char *mode) { +#if defined(HAVE_F_RDAHEAD) + if (strchr(mode, 'a') == NULL && strchr(mode, 'w') == NULL) + { + int fd = open(filename, O_RDONLY); // removed O_NOATIME which may fail + if (fd < 0) + return errno; + fcntl(fd, F_RDAHEAD, 1); + return (*file = fdopen(fd, mode)) == NULL ? errno : 0; + } +#endif return (*file = fopen(filename, mode)) == NULL ? errno : 0; }
![https://www.genivia.com/images/scranim.gif"](https://www.genivia.com/images/scranim.gif"){kind=link}