Re: wc(1): add -L flag to write length of longest line
> On Sep 29, 2022, at 8:24 PM, Joerg Sonnenberger wrote: > > On Thu, Sep 29, 2022 at 08:39:16PM +1000, Jonathan Gray wrote: >> wc counts items in files. Finding the longest item indeed sounds >> like a task better suited to awk. Doesn’t gnu wc show that tabs have length 8 rather than length 1? Do the other wc implementations differ? > > Finding outliers, means and counting are all parts of the same basic > class of operations. A good implementation of all of them requires > constant space and a fixed time per input character. An implementation > in awk will generally not have that property. Did you run any benchmarks to check this? I’m not doubting you but just wondering if there’s a speed difference that matters in practice. > > Joerg >
Re: wc(1): add -L flag to write length of longest line
On Thu, Sep 29, 2022 at 08:39:16PM +1000, Jonathan Gray wrote: > wc counts items in files. Finding the longest item indeed sounds > like a task better suited to awk. Finding outliers, means and counting are all parts of the same basic class of operations. A good implementation of all of them requires constant space and a fixed time per input character. An implementation in awk will generally not have that property. Joerg
malloc: prep for immutable pages
Hi,
Rearrange things so that we do not have to flip protection of r/o
pages back and forth when swicthing from single-threaded to
multi-threaded. Also saves work in many cases by not initing pools
until they are used: the pool used for MAP_CONCEAL pages is not used
by very many processes and if you have just a few threads only a few
pools will be set up.
The actual mimmutable calls are to be added later. I marked the places where
they should end up.
Please test,
-Otto
Index: stdlib/malloc.c
===
RCS file: /cvs/src/lib/libc/stdlib/malloc.c,v
retrieving revision 1.274
diff -u -p -r1.274 malloc.c
--- stdlib/malloc.c 30 Jun 2022 17:15:48 - 1.274
+++ stdlib/malloc.c 29 Sep 2022 09:42:57 -
@@ -142,6 +142,7 @@ struct dir_info {
int malloc_junk;/* junk fill? */
int mmap_flag; /* extra flag for mmap */
int mutex;
+ int malloc_mt; /* multi-threaded mode? */
/* lists of free chunk info structs */
struct chunk_head chunk_info_list[MALLOC_MAXSHIFT + 1];
/* lists of chunks with free slots */
@@ -181,8 +182,6 @@ struct dir_info {
#endif /* MALLOC_STATS */
u_int32_t canary2;
};
-#define DIR_INFO_RSZ ((sizeof(struct dir_info) + MALLOC_PAGEMASK) & \
- ~MALLOC_PAGEMASK)
static void unmap(struct dir_info *d, void *p, size_t sz, size_t clear);
@@ -208,7 +207,6 @@ struct malloc_readonly {
/* Main bookkeeping information */
struct dir_info *malloc_pool[_MALLOC_MUTEXES];
u_int malloc_mutexes; /* how much in actual use? */
- int malloc_mt; /* multi-threaded mode? */
int malloc_freecheck; /* Extensive double free check */
int malloc_freeunmap; /* mprotect free pages PROT_NONE? */
int def_malloc_junk;/* junk fill? */
@@ -257,7 +255,7 @@ static void malloc_exit(void);
static inline void
_MALLOC_LEAVE(struct dir_info *d)
{
- if (mopts.malloc_mt) {
+ if (d->malloc_mt) {
d->active--;
_MALLOC_UNLOCK(d->mutex);
}
@@ -266,7 +264,7 @@ _MALLOC_LEAVE(struct dir_info *d)
static inline void
_MALLOC_ENTER(struct dir_info *d)
{
- if (mopts.malloc_mt) {
+ if (d->malloc_mt) {
_MALLOC_LOCK(d->mutex);
d->active++;
}
@@ -291,7 +289,7 @@ hash(void *p)
static inline struct dir_info *
getpool(void)
{
- if (!mopts.malloc_mt)
+ if (mopts.malloc_pool[1] == NULL || !mopts.malloc_pool[1]->malloc_mt)
return mopts.malloc_pool[1];
else/* first one reserved for special pool */
return mopts.malloc_pool[1 + TIB_GET()->tib_tid %
@@ -496,46 +494,22 @@ omalloc_init(void)
}
static void
-omalloc_poolinit(struct dir_info **dp, int mmap_flag)
+omalloc_poolinit(struct dir_info *d, int mmap_flag)
{
- char *p;
- size_t d_avail, regioninfo_size;
- struct dir_info *d;
int i, j;
- /*
-* Allocate dir_info with a guard page on either side. Also
-* randomise offset inside the page at which the dir_info
-* lies (subject to alignment by 1 << MALLOC_MINSHIFT)
-*/
- if ((p = MMAPNONE(DIR_INFO_RSZ + (MALLOC_PAGESIZE * 2), mmap_flag)) ==
- MAP_FAILED)
- wrterror(NULL, "malloc init mmap failed");
- mprotect(p + MALLOC_PAGESIZE, DIR_INFO_RSZ, PROT_READ | PROT_WRITE);
- d_avail = (DIR_INFO_RSZ - sizeof(*d)) >> MALLOC_MINSHIFT;
- d = (struct dir_info *)(p + MALLOC_PAGESIZE +
- (arc4random_uniform(d_avail) << MALLOC_MINSHIFT));
-
- rbytes_init(d);
- d->regions_free = d->regions_total = MALLOC_INITIAL_REGIONS;
- regioninfo_size = d->regions_total * sizeof(struct region_info);
- d->r = MMAP(regioninfo_size, mmap_flag);
- if (d->r == MAP_FAILED) {
- d->regions_total = 0;
- wrterror(NULL, "malloc init mmap failed");
- }
+ d->r = NULL;
+ d->rbytesused = sizeof(d->rbytes);
+ d->regions_free = d->regions_total = 0;
for (i = 0; i <= MALLOC_MAXSHIFT; i++) {
LIST_INIT(>chunk_info_list[i]);
for (j = 0; j < MALLOC_CHUNK_LISTS; j++)
LIST_INIT(>chunk_dir[i][j]);
}
- STATS_ADD(d->malloc_used, regioninfo_size + 3 * MALLOC_PAGESIZE);
d->mmap_flag = mmap_flag;
d->malloc_junk = mopts.def_malloc_junk;
d->canary1 = mopts.malloc_canary ^ (u_int32_t)(uintptr_t)d;
d->canary2 = ~d->canary1;
-
- *dp = d;
}
static int
@@ -550,7 +524,8 @@ omalloc_grow(struct dir_info *d)
if (d->regions_total > SIZE_MAX / sizeof(struct region_info) / 2)
return 1;
-
tsc: AMD Family 17h, 19h: compute frequency from MSRs
This patch computes the TSC frequency for AMD family 17h and 19h CPUs
(Zen microarchitecture and up) from AMD-specific MSRs. Computing the
TSC frequency is faster than calibrating with a separate timer and
introduces no error.
We already do this for Intel CPUs in tsc_freq_cpuid().
I got several successful test reports on family 17h and 19h CPUs in
response to this mail:
https://marc.info/?l=openbsd-tech=166394236029484=2
The details for computing the frequency are in the PPR for 17h and
19h, found here (page numbers are cited in the patch):
https://www.amd.com/system/files/TechDocs/55570-B1-PUB.zip
https://www.amd.com/system/files/TechDocs/56214-B0-PUB.zip
The process is slightly more complicated on AMD CPU families 10h-16h.
I will deal with them in a separate commit.
ok?
Index: include/specialreg.h
===
RCS file: /cvs/src/sys/arch/amd64/include/specialreg.h,v
retrieving revision 1.94
diff -u -p -r1.94 specialreg.h
--- include/specialreg.h30 Aug 2022 17:09:21 - 1.94
+++ include/specialreg.h29 Sep 2022 14:12:53 -
@@ -540,6 +540,10 @@
*/
#defineMSR_HWCR0xc0010015
#defineHWCR_FFDIS 0x0040
+#defineHWCR_TSCFREQSEL 0x0100
+
+#defineMSR_PSTATEDEF(_n) (0xc0010064 + (_n))
+#definePSTATEDEF_EN0x8000ULL
#defineMSR_NB_CFG 0xc001001f
#defineNB_CFG_DISIOREQLOCK 0x0004ULL
Index: amd64/tsc.c
===
RCS file: /cvs/src/sys/arch/amd64/amd64/tsc.c,v
retrieving revision 1.29
diff -u -p -r1.29 tsc.c
--- amd64/tsc.c 22 Sep 2022 04:57:08 - 1.29
+++ amd64/tsc.c 29 Sep 2022 14:12:53 -
@@ -100,6 +100,67 @@ tsc_freq_cpuid(struct cpu_info *ci)
return (0);
}
+uint64_t
+tsc_freq_msr(struct cpu_info *ci)
+{
+ uint64_t base, def, divisor, multiplier;
+
+ if (strcmp(cpu_vendor, "AuthenticAMD") != 0)
+ return 0;
+
+ /*
+* All 10h+ CPUs have Core::X86::Msr:HWCR and the TscFreqSel
+* bit. If TscFreqSel hasn't been set, the TSC isn't advancing
+* at the core P0 frequency and we need to calibrate by hand.
+*/
+ if (ci->ci_family < 0x10)
+ return 0;
+ if (!ISSET(rdmsr(MSR_HWCR), HWCR_TSCFREQSEL))
+ return 0;
+
+ /*
+* In 10h+ CPUs, Core::X86::Msr::PStateDef defines the voltage
+* and frequency for each core P-state. We want the P0 frequency.
+* If the En bit isn't set, the register doesn't define a valid
+* P-state.
+*/
+ def = rdmsr(MSR_PSTATEDEF(0));
+ if (!ISSET(def, PSTATEDEF_EN))
+ return 0;
+
+ switch (ci->ci_family) {
+ case 0x17:
+ case 0x19:
+ /*
+* PPR for AMD Family 17h [...]:
+* Models 01h,08h B2, Rev 3.03, pp. 33, 139-140
+* Model 18h B1, Rev 3.16, pp. 36, 143-144
+* Model 60h A1, Rev 3.06, pp. 33, 155-157
+* Model 71h B0, Rev 3.06, pp. 28, 150-151
+*
+* PPR for AMD Family 19h [...]:
+* Model 21h B0, Rev 3.05, pp. 33, 166-167
+*
+* OSRR for AMD Family 17h processors,
+* Models 00h-2Fh, Rev 3.03, pp. 130-131
+*/
+ base = 2; /* 200.0 MHz */
+ divisor = (def >> 8) & 0x3f;
+ if (divisor <= 0x07 || divisor >= 0x2d)
+ return 0; /* reserved */
+ if (divisor >= 0x1b && divisor % 2 == 1)
+ return 0; /* reserved */
+ multiplier = def & 0xff;
+ if (multiplier <= 0x0f)
+ return 0; /* reserved */
+ break;
+ default:
+ return 0;
+ }
+
+ return base * multiplier / divisor;
+}
+
void
tsc_identify(struct cpu_info *ci)
{
@@ -118,6 +179,8 @@ tsc_identify(struct cpu_info *ci)
tsc_is_invariant = 1;
tsc_frequency = tsc_freq_cpuid(ci);
+ if (tsc_frequency == 0)
+ tsc_frequency = tsc_freq_msr(ci);
if (tsc_frequency > 0)
delay_init(tsc_delay, 5000);
}
Re: wc(1): add -L flag to write length of longest line
On Thu, Sep 29, 2022 at 08:57:04AM +, Job Snijders wrote:
> Hi all,
>
> I often find myself piping data through ... | awk '{print length}' | ...
> I figured there should be a more direct way that requires less typing.
> Perhaps other developers have a similar itch?
>
> The FreeBSD, NetBSD, Dragonfly, and GNU variants of the wc(1) utility
> have a similar -L feature.
That isn't an argument for merit or good taste. Choice of flag, sure.
wc counts items in files. Finding the longest item indeed sounds
like a task better suited to awk.
>
> Kind regards,
>
> Job
>
> Index: wc.1
> ===
> RCS file: /cvs/src/usr.bin/wc/wc.1,v
> retrieving revision 1.27
> diff -u -p -r1.27 wc.1
> --- wc.1 24 Oct 2016 13:46:58 - 1.27
> +++ wc.1 21 Sep 2022 15:47:29 -
> @@ -41,7 +41,7 @@
> .Sh SYNOPSIS
> .Nm wc
> .Op Fl c | m
> -.Op Fl hlw
> +.Op Fl hLlw
> .Op Ar
> .Sh DESCRIPTION
> The
> @@ -68,6 +68,14 @@ is written to the standard output.
> Use unit suffixes: Byte, Kilobyte, Megabyte, Gigabyte, Terabyte,
> Petabyte, and Exabyte in order to reduce the number of digits to four or
> fewer
> using powers of 2 for sizes (K=1024, M=1048576, etc.).
> +.It Fl L
> +Write the length of the longest line to the standard output.
> +Length is the number of bytes counted, or the number of characters if the
> +.Fl m
> +flag is specified.
> +If more than one input file is specified,
> +the length of the longest line of all files is reported as the value of
> +.Qq total .
> .It Fl l
> The number of lines in each input file
> is written to the standard output.
> @@ -128,9 +136,9 @@ utility is compliant with the
> .St -p1003.1-2008
> specification.
> .Pp
> -The flag
> -.Op Fl h
> -is an extension to that specification.
> +The flags
> +.Op Fl Lh
> +are extensions to that specification.
> .Sh HISTORY
> A
> .Nm
> Index: wc.c
> ===
> RCS file: /cvs/src/usr.bin/wc/wc.c,v
> retrieving revision 1.30
> diff -u -p -r1.30 wc.c
> --- wc.c 2 Sep 2022 15:21:40 - 1.30
> +++ wc.c 21 Sep 2022 15:47:29 -
> @@ -44,12 +44,12 @@
>
> #define _MAXBSIZE (64 * 1024)
>
> -int64_t tlinect, twordct, tcharct;
> -int doline, doword, dochar, humanchar, multibyte;
> +int64_t tlinect, twordct, tcharct, tlongest;
> +int doline, doword, dochar, dolongest, humanchar, multibyte;
> int rval;
> extern char *__progname;
>
> -static void print_counts(int64_t, int64_t, int64_t, const char *);
> +static void print_counts(int64_t, int64_t, int64_t, int64_t, const char *);
> static void format_and_print(int64_t);
> static void cnt(const char *);
>
> @@ -63,8 +63,11 @@ main(int argc, char *argv[])
> if (pledge("stdio rpath", NULL) == -1)
> err(1, "pledge");
>
> - while ((ch = getopt(argc, argv, "lwchm")) != -1)
> + while ((ch = getopt(argc, argv, "Llwchm")) != -1)
> switch(ch) {
> + case 'L':
> + dolongest = 1;
> + break;
> case 'l':
> doline = 1;
> break;
> @@ -84,7 +87,7 @@ main(int argc, char *argv[])
> case '?':
> default:
> fprintf(stderr,
> - "usage: %s [-c | -m] [-hlw] [file ...]\n",
> + "usage: %s [-c | -m] [-hLlw] [file ...]\n",
> __progname);
> return 1;
> }
> @@ -96,7 +99,7 @@ main(int argc, char *argv[])
>* if you don't get any arguments, you have to turn them
>* all on.
>*/
> - if (!doline && !doword && !dochar)
> + if (!doline && !doword && !dochar && !dolongest)
> doline = doword = dochar = 1;
>
> if (!*argv) {
> @@ -109,7 +112,8 @@ main(int argc, char *argv[])
> } while(*++argv);
>
> if (dototal)
> - print_counts(tlinect, twordct, tcharct, "total");
> + print_counts(tlinect, twordct, tcharct, tlongest,
> + "total");
> }
>
> return rval;
> @@ -127,11 +131,11 @@ cnt(const char *path)
> wchar_t wc;
> short gotsp;
> ssize_t len;
> - int64_t linect, wordct, charct;
> + uint64_t linect, wordct, charct, longct, tmpll;
> struct stat sbuf;
> int fd;
>
> - linect = wordct = charct = 0;
> + linect = wordct = charct = longct = tmpll = 0;
> stream = NULL;
> if (path != NULL) {
> file = path;
> @@ -180,12 +184,19 @@ cnt(const char *path)
>* faster to get lines than to get words, since
>* the word count requires some logic.
>*/
> - else if (doline) {
> + else if (doline || dolongest) {
> while ((len = read(fd, buf, _MAXBSIZE))
wc(1): add -L flag to write length of longest line
Hi all,
I often find myself piping data through ... | awk '{print length}' | ...
I figured there should be a more direct way that requires less typing.
Perhaps other developers have a similar itch?
The FreeBSD, NetBSD, Dragonfly, and GNU variants of the wc(1) utility
have a similar -L feature.
Kind regards,
Job
Index: wc.1
===
RCS file: /cvs/src/usr.bin/wc/wc.1,v
retrieving revision 1.27
diff -u -p -r1.27 wc.1
--- wc.124 Oct 2016 13:46:58 - 1.27
+++ wc.121 Sep 2022 15:47:29 -
@@ -41,7 +41,7 @@
.Sh SYNOPSIS
.Nm wc
.Op Fl c | m
-.Op Fl hlw
+.Op Fl hLlw
.Op Ar
.Sh DESCRIPTION
The
@@ -68,6 +68,14 @@ is written to the standard output.
Use unit suffixes: Byte, Kilobyte, Megabyte, Gigabyte, Terabyte,
Petabyte, and Exabyte in order to reduce the number of digits to four or fewer
using powers of 2 for sizes (K=1024, M=1048576, etc.).
+.It Fl L
+Write the length of the longest line to the standard output.
+Length is the number of bytes counted, or the number of characters if the
+.Fl m
+flag is specified.
+If more than one input file is specified,
+the length of the longest line of all files is reported as the value of
+.Qq total .
.It Fl l
The number of lines in each input file
is written to the standard output.
@@ -128,9 +136,9 @@ utility is compliant with the
.St -p1003.1-2008
specification.
.Pp
-The flag
-.Op Fl h
-is an extension to that specification.
+The flags
+.Op Fl Lh
+are extensions to that specification.
.Sh HISTORY
A
.Nm
Index: wc.c
===
RCS file: /cvs/src/usr.bin/wc/wc.c,v
retrieving revision 1.30
diff -u -p -r1.30 wc.c
--- wc.c2 Sep 2022 15:21:40 - 1.30
+++ wc.c21 Sep 2022 15:47:29 -
@@ -44,12 +44,12 @@
#define_MAXBSIZE (64 * 1024)
-int64_ttlinect, twordct, tcharct;
-intdoline, doword, dochar, humanchar, multibyte;
+int64_ttlinect, twordct, tcharct, tlongest;
+intdoline, doword, dochar, dolongest, humanchar, multibyte;
intrval;
extern char *__progname;
-static void print_counts(int64_t, int64_t, int64_t, const char *);
+static void print_counts(int64_t, int64_t, int64_t, int64_t, const char *);
static void format_and_print(int64_t);
static void cnt(const char *);
@@ -63,8 +63,11 @@ main(int argc, char *argv[])
if (pledge("stdio rpath", NULL) == -1)
err(1, "pledge");
- while ((ch = getopt(argc, argv, "lwchm")) != -1)
+ while ((ch = getopt(argc, argv, "Llwchm")) != -1)
switch(ch) {
+ case 'L':
+ dolongest = 1;
+ break;
case 'l':
doline = 1;
break;
@@ -84,7 +87,7 @@ main(int argc, char *argv[])
case '?':
default:
fprintf(stderr,
- "usage: %s [-c | -m] [-hlw] [file ...]\n",
+ "usage: %s [-c | -m] [-hLlw] [file ...]\n",
__progname);
return 1;
}
@@ -96,7 +99,7 @@ main(int argc, char *argv[])
* if you don't get any arguments, you have to turn them
* all on.
*/
- if (!doline && !doword && !dochar)
+ if (!doline && !doword && !dochar && !dolongest)
doline = doword = dochar = 1;
if (!*argv) {
@@ -109,7 +112,8 @@ main(int argc, char *argv[])
} while(*++argv);
if (dototal)
- print_counts(tlinect, twordct, tcharct, "total");
+ print_counts(tlinect, twordct, tcharct, tlongest,
+ "total");
}
return rval;
@@ -127,11 +131,11 @@ cnt(const char *path)
wchar_t wc;
short gotsp;
ssize_t len;
- int64_t linect, wordct, charct;
+ uint64_t linect, wordct, charct, longct, tmpll;
struct stat sbuf;
int fd;
- linect = wordct = charct = 0;
+ linect = wordct = charct = longct = tmpll = 0;
stream = NULL;
if (path != NULL) {
file = path;
@@ -180,12 +184,19 @@ cnt(const char *path)
* faster to get lines than to get words, since
* the word count requires some logic.
*/
- else if (doline) {
+ else if (doline || dolongest) {
while ((len = read(fd, buf, _MAXBSIZE)) > 0) {
charct += len;
- for (C = buf; len--; ++C)
- if (*C == '\n')
+ for (C = buf; len--; ++C) {
+ if (*C == '\n') {
+ if (tmpll > longct)
+
