Re: stdioext on musl

[Bruno Haible]

Rich Felker wrote:
> stdio_ext.h okay?

OK, of course.

With these patches from your side, I am applying these changes to gnulib.
No need for "#ifdef __MUSL__" for now.


2012-06-19  Bruno Haible  

stdioext: Add support for musl libc.

* m4/fbufmode.m4 (gl_FUNC_FBUFMODE): Test whether __fbufsize exists.
* lib/fbufmode.c (fbufmode): Add conditional code for musl.

* m4/fseterr.m4: New file.
* lib/fseterr.h (fseterr): Define as an alias of __fseterr if that
function exists.
* modules/fseterr (Files): Add m4/fseterr.m4.
(configure.ac): Invoke gl_FUNC_FSETERR. Compile fseterr.c if
__fseterr does not exist.
(Makefile.am): Remove fseterr.c from lib_SOURCES.

* lib/freadable.h: Update comment.

* lib/fwritable.h: Update comment.

* lib/freading.h: Update comment.

* lib/fwriting.h: Update comment.

* m4/freadahead.m4: New file.
* lib/freadahead.h (freadahead): Define as an alias of __freadahead if
that function exists.
* modules/freadahead (Files): Add m4/freadahead.m4.
(configure.ac): Invoke gl_FUNC_FREADAHEAD. Compile freadahead.c if
__freadahead does not exist.
(Makefile.am): Remove freadahead.c from lib_SOURCES.

* m4/freadptr.m4: New file.
* lib/freadptr.h (freadptr): Define as an alias of __freadptr if that
function exists.
* modules/freadptr (Files): Add m4/freadptr.m4.
(configure.ac): Invoke gl_FUNC_FREADPTR. Compile freadptr.c if
__freadptr does not exist.
(Makefile.am): Remove freadptr.c from lib_SOURCES.

* m4/freadseek.m4: New file.
* lib/freadseek.c (freadptrinc): Use __freadptrinc if that function
exists.
* modules/freadseek (Files): Add m4/freadseek.m4.
(configure.ac): Invoke gl_FUNC_FREADSEEK.

* lib/fpurge.c (fpurge): Update comment.

Reported by and with help from Rich Felker .

--- lib/fbufmode.c.orig Tue Jun 19 23:18:35 2012
+++ lib/fbufmode.c  Tue Jun 19 22:32:55 2012
@@ -79,6 +79,10 @@
   if (fp->__linebuf)
 return _IOLBF;
   return (fp->__bufsize > 0 ? _IOFBF : _IONBF);
+#elif HAVE___FLBF && HAVE___FBUFSIZE /* musl libc */
+  if (__flbf (fp))
+return _IOLBF;
+  return (__fbufsize (fp) > 0 ? _IOFBF : _IONBF);
 #elif defined EPLAN9/* Plan9 */
   if (fp->flags & 2 /* LINEBUF */)
 return _IOLBF;
--- lib/fpurge.c.orig   Tue Jun 19 23:18:35 2012
+++ lib/fpurge.cSun Jun 17 21:06:31 2012
@@ -29,7 +29,7 @@
 int
 fpurge (FILE *fp)
 {
-#if HAVE___FPURGE   /* glibc >= 2.2, Haiku, Solaris >= 7 */
+#if HAVE___FPURGE   /* glibc >= 2.2, Haiku, Solaris >= 7, musl 
libc */
 
   __fpurge (fp);
   /* The __fpurge function does not have a return value.  */
--- lib/freadable.h.origTue Jun 19 23:18:35 2012
+++ lib/freadable.h Sun Jun 17 21:06:31 2012
@@ -22,7 +22,7 @@
STREAM must not be wide-character oriented.
The result doesn't change until the stream is closed or re-opened.  */
 
-#if HAVE___FREADABLE /* glibc >= 2.2, Solaris >= 7 */
+#if HAVE___FREADABLE /* glibc >= 2.2, Solaris >= 7, musl libc */
 
 # include 
 # define freadable(stream) (__freadable (stream) != 0)
--- lib/freadahead.h.orig   Tue Jun 19 23:18:35 2012
+++ lib/freadahead.hTue Jun 19 22:15:38 2012
@@ -17,10 +17,6 @@
 #include 
 #include 
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 /* Assuming the stream STREAM is open for reading:
Return the number of bytes waiting in the input buffer of STREAM.
This includes both the bytes that have been read from the underlying input
@@ -31,8 +27,21 @@
 
STREAM must not be wide-character oriented.  */
 
+#if HAVE___FREADAHEAD /* musl libc */
+
+# include 
+# define freadahead(stream) __freadahead (stream)
+
+#else
+
+# ifdef __cplusplus
+extern "C" {
+# endif
+
 extern size_t freadahead (FILE *stream) _GL_ATTRIBUTE_PURE;
 
-#ifdef __cplusplus
+# ifdef __cplusplus
 }
+# endif
+
 #endif
--- lib/freading.h.orig Tue Jun 19 23:18:35 2012
+++ lib/freading.h  Tue Jun 19 22:17:43 2012
@@ -33,7 +33,7 @@
STREAM must not be wide-character oriented.  */
 
 #if HAVE___FREADING && (!defined __GLIBC__ || defined __UCLIBC__ || __GLIBC__ 
> 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 7))
-/* Solaris >= 7, not glibc >= 2.2, but glibc >= 2.7  */
+/* Solaris >= 7, not glibc >= 2.2, but glibc >= 2.7, or musl libc  */
 
 # include 
 # define freading(stream) (__freading (stream) != 0)
--- lib/freadptr.h.orig Tue Jun 19 23:18:35 2012
+++ lib/freadptr.h  Tue Jun 19 22:19:32 2012
@@ -17,10 +17,6 @@
 #include 
 #include 
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 /* Assuming the stream STREAM is open for reading:
Return a pointer to the input buffer of STREAM, or NULL.
If the returned pointer is non-NULL, *SIZEP is set to the (positive) size
@@ -32,8 +28,21 @@
 
STREAM must not be wide-character oriented.  */
 
+#if HAV

Re: stdioext on musl

[Rich Felker]

On Tue, Jun 19, 2012 at 01:46:40PM +0200, Bruno Haible wrote:
> Hi Rich,
> 
> The patches that you've committed at
>   
> http://git.etalabs.net/cgi-bin/gitweb.cgi?p=musl;a=commitdiff;h=deb90c79e5c498fbb48de1423df034447f330e38
>   
> http://git.etalabs.net/cgi-bin/gitweb.cgi?p=musl;a=commitdiff;h=e15171b8d8e80e8b5bcf4e95b1709697858f545a
> go a long way at implementing the support that gnulib needs. Just
> one question: When trying to use the new 4 functions, I get gcc warnings
> 
> freadahead.c: In function 'freadahead':
> freadahead.c:84:3: warning: implicit declaration of function '__freadahead' 
> [-Wimplicit-function-declaration]
> freadptr.c: In function 'freadptr':
> freadptr.c:105:3: warning: implicit declaration of function '__freadptr' 
> [-Wimplicit-function-declaration]
> freadptr.c:105:3: warning: return makes pointer from integer without a cast 
> [enabled by default]
> freadseek.c: In function 'freadptrinc':
> freadseek.c:62:3: warning: implicit declaration of function '__freadptrinc' 
> [-Wimplicit-function-declaration]
> fseterr.c: In function 'fseterr':
> fseterr.c:49:3: warning: implicit declaration of function '__fseterr' 
> [-Wimplicit-function-declaration]
> 
> Will you add declarations of these functions to a public .h file?
> Or should gnulib provide these declarations?

Yes, I simply forgot to add them. stdio_ext.h okay?

Rich

Re: stdioext on musl

[Bruno Haible]

Hi Rich,

The patches that you've committed at
  
http://git.etalabs.net/cgi-bin/gitweb.cgi?p=musl;a=commitdiff;h=deb90c79e5c498fbb48de1423df034447f330e38
  
http://git.etalabs.net/cgi-bin/gitweb.cgi?p=musl;a=commitdiff;h=e15171b8d8e80e8b5bcf4e95b1709697858f545a
go a long way at implementing the support that gnulib needs. Just
one question: When trying to use the new 4 functions, I get gcc warnings

freadahead.c: In function 'freadahead':
freadahead.c:84:3: warning: implicit declaration of function '__freadahead' 
[-Wimplicit-function-declaration]
freadptr.c: In function 'freadptr':
freadptr.c:105:3: warning: implicit declaration of function '__freadptr' 
[-Wimplicit-function-declaration]
freadptr.c:105:3: warning: return makes pointer from integer without a cast 
[enabled by default]
freadseek.c: In function 'freadptrinc':
freadseek.c:62:3: warning: implicit declaration of function '__freadptrinc' 
[-Wimplicit-function-declaration]
fseterr.c: In function 'fseterr':
fseterr.c:49:3: warning: implicit declaration of function '__fseterr' 
[-Wimplicit-function-declaration]

Will you add declarations of these functions to a public .h file?
Or should gnulib provide these declarations?

Bruno

Re: stdioext on musl [was: gnulib portability issues]

[Paul Eggert]

On 06/18/2012 06:27 AM, John Spencer wrote:

> I just couldn't withstand to express my disgust

Please refrain from such rhetoric in the future.
The bug-gnulib mailing list is for discussing ways to
improve gnulib, and personal attacks get in the
way of its purpose.

Re: stdioext on musl [was: gnulib portability issues]

[John Spencer]

On Sun, 17 Jun 2012 16:59:45 -0700, Bruno Haible wrote

 Rich Felker wrote
> If gnulib is willing to _detect_ working functions rather than trying 

to detect musl

[...]

> We often, but not always, use an autoconf test that verifies that a
> function works. Why not always? Because such a test is ca. 20-50 lines of
> code, and a #ifdef is just 1 line of code.

ah, the problem is that you're LAZY.

i might notify your employer about that.


instead of fixing a problem once and for all, you prefer to add a quick hack 
whenever a new system appears.

*this is the gnulib approach to "portability".*



well, if stuff "just worked", you would lose your position of power, where 
people have to come here begging for help/patches, and you can play the big zampano.
additionally this way of handling things keeps a constant incoming flow of work 
so you can keep your jobs.

thanks though for showing the world what a piece of crap gnulib and its design 
philosophy is.
this creates a valuable reference to convince people not to use this turd.



-- JS

P.S.

Note that this is my private opinion and might not correspond to the opinion of 
musl's authors or other users.
I just couldn't withstand to express my disgust when reading the latest mails 
of bruno.

Re: stdioext on musl

[Rich Felker]

On Mon, Jun 18, 2012 at 02:59:56AM +0200, Bruno Haible wrote:
> For 2), the issue is that for a stream opened in write-only mode,
> immediately after the fopen() call, gnulib expects fwriting(fp) to be
> true:

Thanks. I committed a change whereby __freading now returns 1 whenever
the stream is read-only or read mode is active (even if the buffer is
empty), and likewise __fwriting returns 1 whenever the stream is
write-only or write mode is active (even if the buffer is empty).
Hopefully this covers everything gnulib cares about.

Rich

Re: stdioext on musl

[Bruno Haible]

Rich Felker wrote:
> > This is not hypothetical at all. The __freading, __fwriting functions
> > exist in various libcs (glibc, Solaris, uClibc, musl). But only in musl
> > the value is different in some particular case. Therefore I ask you to
> 
> Would you mind telling me how it's different and how that's a problem?
> I meant for these functions to follow the traditional stdio_ext.h API,
> and if they're not doing the right thing I'd rather fix them than have
> ugly workarounds going into gnulib.

Sure. You can reproduce the issues by applying the patch I sent to a
gnulib checkout, then do
  $ ./gnulib-tool --create-testdir --dir=/tmp/testdir-stdioext --with-tests 
--single-configure --avoid=havelib-tests fseterr freadable fwritable fbufmode 
freading fwriting freadptr freadseek freadahead fpurge fseeko ftello fpending 
fflush
then
  $ cd /tmp/testdir-stdioext
  $ ./configure CC=musl-gcc CPPFLAGS=-Wall
  $ make
  $ make check


1) If I attempt to write

bool
freading (FILE *fp)
{
  return __freading (fp);
}

instead of the current code

bool
freading (FILE *fp)
{
  return !__fwritable (fp) || __freading (fp);
}

then I get the test failure

test-freading.c:48: assertion failed
FAIL: test-freading


2) Independently of 1): If I attempt to write

bool
fwriting (FILE *fp)
{
  return __fwriting (fp);
}

instead of the current code

bool
fwriting (FILE *fp)
{
  return !__freadable (fp) || __fwriting (fp);
}

then I get the test failure

test-fwriting.c:41: assertion failed
FAIL: test-fwriting


> Is the issue that you want __fwriting to return 1 even if there's no
> data buffered as long as the last action on the file was a write? If
> so, I think I can easily fix that.

For 2), the issue is that for a stream opened in write-only mode,
immediately after the fopen() call, gnulib expects fwriting(fp) to be
true:

  fp = fopen (TESTFILE, "w");
  if (fp == NULL)
goto skip;
  ASSERT (fwriting (fp));  // <== test-fwriting.c line 41


Bruno

Re: stdioext on musl

[Bruno Haible]

Rich,

> Then Bruno came back to us with this monstrosity of a patch that
> insists, for no good reason, on trying to detect musl specifically,
> thereby increasing the amount of broken special-cased non-portable
> code in gnulib rather than modernizing it.
> ...
> What is my business is that Bruno wants to start poking at internals
> of musl in a way that WILL BREAK in nearly every single application
> that's using gnulib, and that will have users coming to us with bug
> reports when gnulib is the code at fault. I can't and won't work with
> that. If it's your final decision to do things that way ...

There is a big misunderstanding here. I don't want to commit that large
patch with lots of "#ifdef __MUSL__" if there will soon be a better way
to do, after some changes have gone into musl.

The purpose of the patch was
  1. to get down from the "gnulib is unportable" discussion, and provide
 a patch that provides the interoperability today,
  2. give you some hints about which kinds of primitives in musl would
 be needed for gnulib to get rid of these "#ifdef __MUSL__".

I am writing to you precisely to make the plan that Eric outlined more
concrete.

I have told you constructively what our needs are:
  1) a macro like __MUSL__
  2) the 4 functions __freadahead, __freadptr, __freadptrinc, __fseterr.

Now it's your turn. Please consider implementing these needs or parts of
these needs in musl. (Even if you don't agree that these are or should
be needs of gnulib. Just believe me and Paul that we need this.)

Then I, on the gnulib side, will see to which extent the (admittedly large)
patch can be reduced to something more reasonable.

Once again: My goal here is to support all the stdioext functions on future
versions of musl, with as little #ifdef as possible. The result depends on
what will go into .

Bruno

Re: stdioext on musl

[Rich Felker]

On Mon, Jun 18, 2012 at 02:01:28AM +0200, Bruno Haible wrote:
> Rich Felker wrote:
> > > 2) Provide 4 primitive functions.
> > 
> > This is less offensive at least, but will configure even find them?
> > Eric Blake offered to make it work that way, but you seem to want to
> > test against macros that identify the implementation...? If gnulib is
> > willing to _detect_ working functions rather than trying to detect
> > musl, I'd be willing, albeit reluctant, to add them.
> 
> If you accept the proposed function names __freadahead, __freadptr,
> __freadptrinc, __fseterr, then we will start by using an autoconf check,
> and use the obvious conditionals like
>   #if HAVE___FREADPTR
> in the code. This way, if other libcs do the same, gnulib will not have
> to change much.

OK, that sounds reasonable.

Would you also be willing to get the fallback cases for these working?
I believe you're already using a variant of an algorithm I designed a
long time ago for emulating fseterr in the SLOW_BUT_NO_HACKS case.
There was also a discussion earlier on this list with a similar
fallback for freadahead that would work on all systems (but only
returning 0/1, not an actual buffered byte count), and of course
freadptr can fallback to returning NULL. I agree these implementations
are horribly slow and undesirable, but it would be nice if the next
new system that runs into trouble with gnulib-based programs is coming
to you with "What do we need to do to get gnulib to be fast on our
system?" rather than "Dammit why is gnulib throwing #error in the
default case?!"

> We often, but not always, use an autoconf test that verifies that a
> function works. Why not always? Because such a test is ca. 20-50 lines of
> code, and a #ifdef is just 1 line of code.
> 
> Additionally, we need the #ifdef for platforms that are cross-compilation
> targets (because AC_RUN_IFELSE's 4th argument is a cross-compilation guess).
> I believe musl is a candidate for cross-compilation, due to its small size?

I think assuming the function works whenever it exists is probably a
sane default for cross-compiling, but perhaps you have experience with
hideous systems contrary to this seemingly-natural line of thinking...

> This is not hypothetical at all. The __freading, __fwriting functions
> exist in various libcs (glibc, Solaris, uClibc, musl). But only in musl
> the value is different in some particular case. Therefore I ask you to

Would you mind telling me how it's different and how that's a problem?
I meant for these functions to follow the traditional stdio_ext.h API,
and if they're not doing the right thing I'd rather fix them than have
ugly workarounds going into gnulib.

Is the issue that you want __fwriting to return 1 even if there's no
data buffered as long as the last action on the file was a write? If
so, I think I can easily fix that.

Rich

Re: stdioext on musl

[Bruno Haible]

Rich Felker wrote:
> > 2) Provide 4 primitive functions.
> 
> This is less offensive at least, but will configure even find them?
> Eric Blake offered to make it work that way, but you seem to want to
> test against macros that identify the implementation...? If gnulib is
> willing to _detect_ working functions rather than trying to detect
> musl, I'd be willing, albeit reluctant, to add them.

If you accept the proposed function names __freadahead, __freadptr,
__freadptrinc, __fseterr, then we will start by using an autoconf check,
and use the obvious conditionals like
  #if HAVE___FREADPTR
in the code. This way, if other libcs do the same, gnulib will not have
to change much.

But it is in our discretion to use #ifdef __MUSL__ instead. For example,
hypothetically speaking, if all other libc authors create incompatible
or buggy variants of __freadptr. Then we may want to use #ifdef __MUSL__.

We often, but not always, use an autoconf test that verifies that a
function works. Why not always? Because such a test is ca. 20-50 lines of
code, and a #ifdef is just 1 line of code.

Additionally, we need the #ifdef for platforms that are cross-compilation
targets (because AC_RUN_IFELSE's 4th argument is a cross-compilation guess).
I believe musl is a candidate for cross-compilation, due to its small size?

> > 1) Currently, gnulib has to go to a great length to detect musl.
> >It uses the presence of __stdio_read and __stdio_write as an indicator;
> 
> That's not valid. These are internal functions that could be renamed
> or disappear (e.g. be changed to visibility=hidden) at any time.

Yup. That's precisely why we need an indicator such as __MUSL__.

> I hope it doesn't sound like I'm just being pedantic here. All along,
> a big part of musl's design goal has been to avoid implementation
> lock-in caused by exposing internals to applications. By keeping the
> interface basically just the standard C and POSIX functions, ...

You are not alone on the world. Some other people may well do the same.
Then we have two libcs, which both don't identify themselves. And in real
life, there are bugs that gnulib needs to work around. So gnulib needs a
way to distinguish your libc from the other guys' libc.

This is not hypothetical at all. The __freading, __fwriting functions
exist in various libcs (glibc, Solaris, uClibc, musl). But only in musl
the value is different in some particular case. Therefore I ask you to
spent 1 line of code to identify musl. Or, you can contribute the two
autoconf tests - written according to gnulib standards and with a
reasonable cross-compilation default - that will allow gnulib to
distinguish the glibc, Solaris, uClibc __freading / __fwriting functions
from the musl one.

> Then either test that __freading works, or use something like
> 
> #if defined(__GLIBC__) && __GLIBC__MAJOR__ == 2 && __GLIBC_MINOR__ < 7
> 
> to special-case the workaround only for particular known broken
> versions, and otherwise assume it works.

Yes we want to assume, in general, that a function works. BUT
  1) in the case of __fwriting, musl 0.9.1 is among the platforms
 which are "known broken" (means, not exactly like gnulib expects),
  2) when cross-compiling, we would like to have a conservative guess
 that classifies all musl versions as "possibly bad, better not use"
 and all other platforms as "likely OK".

For this purpose, we need a macro such as __MUSL__.

Bruno

Re: stdioext on musl

[Rich Felker]

On Sun, Jun 17, 2012 at 04:10:52PM -0700, Paul Eggert wrote:
> On 06/17/2012 03:53 PM, Rich Felker wrote:
> > By "better interfaces" do you mean the
> > 4 already-mentioned stdio extension functions, or something else?
> 
> Yes, I mean the functions that Bruno mentioned.
> 
> > I mean will it find them without needing a special macro like
> > __MUSL__?
> 
> Having a symbol like __MUSL__ helps simplify gnulib's job, because
> it needn't bother to use a heuristic like "if it has __stdio_read and
> __stdio_write, it must be musl", but can simply use __MUSL__ directly.

This heuristic is wrong. So is checking for musl. The correct check is
to test for the _behavior_ you need and use the results of those
tests. This is the most basic principle of autoconf.

> If musl doesn't want to define __MUSL__, that's OK, gnulib can just
> define __MUSL__ on its own using that heuristic, as in Bruno's patch.

Nope, __MUSL__ is in the namespace reserved for the implementation. In
particular, adding #ifdef __MUSL__ #error to any of the standard
headers is valid, and it's what one of our users just suggested we do
if this patch to gnulib is done as-is...

It would be valid to define something in the application-reserved
namespace, but there is no reliable test by which to determine whether
to define it. __stdio_read is NOT public.

> If the heuristic goes wrong in some future platform, we'll fix it.
> This is normal; it's no big deal.  That being said, it's nicer for
> gnulib if musl announced its presence with a symbol like __MUSL__.

It's nicer if gnulib plays by the same rules as everybody else rather
than considering itself above the rules.

Up until Bruno jumped in (when I was communicating mainly with Eric),
I was pleasantly surprised at how sane and reasonable the discussion
was. It completely defied the reputation gnulib has, and folks seemed
genuinely interested in working with us and addressing the systemic
non-portability problems of gnulib -- stuff like fixing all the
fallback cases and removing the need for -DSLOW_BUT_NO_HACKS, etc.

Then Bruno came back to us with this monstrosity of a patch that
insists, for no good reason, on trying to detect musl specifically,
thereby _increasing_ the amount of broken special-cased non-portable
code in gnulib rather than modernizing it.

I would really like to see gnulib modernize. I would like to see it
work out-of-the-box, even if somewhat slowly, on ANY not-yet-invented
system that satisfies the standard interface contracts. However gnulib
is not my project, and it's not my business to tell you that you must
do these things if you're horribly opposed to it.

What is my business is that Bruno wants to start poking at internals
of musl in a way that WILL BREAK in nearly every single application
that's using gnulib, and that will have users coming to us with bug
reports when gnulib is the code at fault. I can't and won't work with
that. If it's your final decision to do things that way, the only
option I'm left with is to keep publicly expressing frustration with
how broken gnulib is. It's not the option I like, because I'm really
sick of dealing with apps breaking because of gnulib already. I'd like
to have a nice clean solution like we discussed with Eric last week,
so I can get on to better things and not worry that compiling gnulib
software is going to start breaking again -- or worse yet, that
binaries using gnulib are going to stop working.

Rich

Re: stdioext on musl

[Paul Eggert]

On 06/17/2012 03:53 PM, Rich Felker wrote:
> By "better interfaces" do you mean the
> 4 already-mentioned stdio extension functions, or something else?

Yes, I mean the functions that Bruno mentioned.

> I mean will it find them without needing a special macro like
> __MUSL__?

Having a symbol like __MUSL__ helps simplify gnulib's job, because
it needn't bother to use a heuristic like "if it has __stdio_read and
__stdio_write, it must be musl", but can simply use __MUSL__ directly.

If musl doesn't want to define __MUSL__, that's OK, gnulib can just
define __MUSL__ on its own using that heuristic, as in Bruno's patch.
If the heuristic goes wrong in some future platform, we'll fix it.
This is normal; it's no big deal.  That being said, it's nicer for
gnulib if musl announced its presence with a symbol like __MUSL__.

Re: stdioext on musl

[Rich Felker]

On Sun, Jun 17, 2012 at 03:44:16PM -0700, Paul Eggert wrote:
> > will configure even find them?
> 
> Yes, it can be made to find them, as part of gnulib.

I mean will it find them without needing a special macro like
__MUSL__?

> This is a pragmatic approach that works well in practice.  Bruno has
> ported it to musl too, which is a good thing.  It would be nicer if
> the port used "approved" musl interfaces, but if musl can't or won't

I already said, if gnulib can find and use these interfaces by testing
that they exist and work, rather than by asking "is this musl?", I'm
(reluctantly) willing to add them. Mainly because I'd like to be done
with the "fighting with gnulib" issue that's made trouble for me and
my userbase from day one...

> do that we'll just have to continue to poke under musl's covers to get
> the behavior that we want, just like we do with many other libraries.
> If these libraries change in the future, gnulib will too.  But it would

Please don't do this. I'd rather just keep private interfaces private
by having civil discussions with developers who try to use them and
finding a mutually agreeable alternative, but if software starts
seriously trying to use musl internals, I'm going to update the build
process to make them all hidden and then a new solution will be
required anyway. This isn't for the sake of fighting; it's because I
don't want users' systems to break when they upgrade libc and suddenly
a lot of software aborts with dynamic linking errors because it was
using a private symbol that was exposed for a long time then later
changed or removed...

> be better if musl provided better interfaces for this useful functionality;
> as that would make gnulib-based applications more likely to port to
> future musl implementations without hassle.

Can you explain what you mean? By "better interfaces" do you mean the
4 already-mentioned stdio extension functions, or something else?

Rich

Re: stdioext on musl

[Paul Eggert]

On 06/17/2012 02:56 PM, Rich Felker wrote:

>> 1) Currently, gnulib has to go to a great length to detect musl.
>>It uses the presence of __stdio_read and __stdio_write as an indicator;
> That's not valid. These are internal functions that could be renamed
> or disappear (e.g. be changed to visibility=hidden) at any time.

OK, so why not change musl to define a symbol like __MUSL__ that
will make gnulib's job easier?  The arguments given so far against
such a symbol seem to be based primarily on theoretical objections,
which don't have much practical force.  The argument for having a
symbol is based on real code that we need to write.

> will configure even find them?

Yes, it can be made to find them, as part of gnulib.

> a big part of musl's design goal has been to avoid implementation
> lock-in caused by exposing internals to applications.

And a big part of gnulib's design goal is to provide useful functionality
that is portable and efficient on practical systems.  These goals clash.
Sorry, but that's life.  Now, we have to decide what to do about it.

To us, conforming to the standard is merely a means towards writing portable
and efficient software.  If not-conforming to the standard achieves the
goal better, then we will fail to conform to the standard.  For us,
standards are our servants, not our masters.

This is a pragmatic approach that works well in practice.  Bruno has
ported it to musl too, which is a good thing.  It would be nicer if
the port used "approved" musl interfaces, but if musl can't or won't
do that we'll just have to continue to poke under musl's covers to get
the behavior that we want, just like we do with many other libraries.
If these libraries change in the future, gnulib will too.  But it would
be better if musl provided better interfaces for this useful functionality;
as that would make gnulib-based applications more likely to port to
future musl implementations without hassle.

Re: stdioext on musl

[Rich Felker]

On Sun, Jun 17, 2012 at 10:56:48PM +0200, Bruno Haible wrote:
> Rich Felker wrote:
> > 1. freadahead is inherently non-portable and has no working portable
> > fallback version. At some point in the discussions, it was suggested
> > that this function should not be pulled in except on old broken
> > systems where stdio doesn't work and needs replacement functions.
> > However, at least some packages, notably GNU M4, seem to use it
> > directly.
> 
> freadahead is one of the "extended" stdio functions that gnulib provides
> portably. "Portable" sometimes means that some porting effort is needed.

I find this a really warped definition of "portable". In a worst-case,
it's not possible to implement many of these operations at all. If for
example you had a system design where stdio was in kernelspace...

To me, portable means write once, run anywhere. I could somewhat see
extending one's definition of portable to mean "needs tuning for the
particular system, but for any given system, there's at least one way
to make it work". gnulib code is neither of these.

> It was tested using the recipe from
> http://lists.gnu.org/archive/html/bug-gnulib/2012-02/msg00022.html
> All tests pass, except for a bug in setvbuf or __fbufsize, that I have
> reported separately.
> 
> There are two areas where musl libc can be modified to reduce the size
> of this patch.
> 
> 1) Currently, gnulib has to go to a great length to detect musl.
>It uses the presence of __stdio_read and __stdio_write as an indicator;

That's not valid. These are internal functions that could be renamed
or disappear (e.g. be changed to visibility=hidden) at any time.

I hope it doesn't sound like I'm just being pedantic here. All along,
a big part of musl's design goal has been to avoid implementation
lock-in caused by exposing internals to applications. By keeping the
interface basically just the standard C and POSIX functions, arbitrary
stable struct definitions and constants, and a moderate subset of BSD
and GNU extension functions, musl keeps its options open to replace
the internals of almost any component with an improved design, without
breaking application binaries.

When applications start probing around for internals like
__stdio_read, this goal gets a lot harder. Not only do old interfaces
that have nothing to do with a new implementation have to be preserved
(and possibly emulated); the knowledge of what interfaces are being
(ab)used by applications, and what properties applications are
depending on, is not even readily available.

>   Why does musl make it so hard?

Why does gnulib make it so hard? No other code we've encountered is
full of so many ugly system-specific hacks. The majority of packages
tested with musl _just_ _work_, or have a few minor issues like
missing includes needed to get a type defined.

>Can't you just define a preprocessor symbol that indicates musl?
>Like so many other systems do?

No, because the intent is NOT to support broken code like what's
currently in gnulib that hard-codes implementation-internals of
particular implementations, or worse, hard-codes assumptions that a
particular implementation is missing certain interfaces. The latter
causes serious breakage when the implementation later adds said
functionality.

Even if __MUSL__ did exist and provided you with sufficient
information to know the version, using internals specific to a
particular version would break dynamic-linked binaries used with a
different version of the library.

>gnulib needs this because some implementations provide the same functions
>but they behave differently. E.g. __freading is broken in glibc < 2.7
>but not in musl. And __freading and __fwriting don't implement exactly
>the same semantics as glibc. That's why real-life programs need to
>#ifdef according to OS and libraries.

Then either test that __freading works, or use something like

#if defined(__GLIBC__) && __GLIBC__MAJOR__ == 2 && __GLIBC_MINOR__ < 7

to special-case the workaround only for particular known broken
versions, and otherwise assume it works. Really I prefer just making a
configure test to make sure it works...

> 2) Provide 4 primitive functions.

This is less offensive at least, but will configure even find them?
Eric Blake offered to make it work that way, but you seem to want to
test against macros that identify the implementation...? If gnulib is
willing to _detect_ working functions rather than trying to detect
musl, I'd be willing, albeit reluctant, to add them.

My favorite option would really be a #3 though: ensure that gnulib,
and gnulib-based apps, can work with a PURELY C/POSIX implementation
with no extensions and no poking at implementation internals. Then,
optionally do the ugly hacks if there's a compelling benefit, but make
sure the code doesn't break when you can't identify the system as one
you've got a set of hacks for.

Rich

Re: stdioext on musl

[Bruno Haible]

Rich Felker wrote:
> 1. freadahead is inherently non-portable and has no working portable
> fallback version. At some point in the discussions, it was suggested
> that this function should not be pulled in except on old broken
> systems where stdio doesn't work and needs replacement functions.
> However, at least some packages, notably GNU M4, seem to use it
> directly.

freadahead is one of the "extended" stdio functions that gnulib provides
portably. "Portable" sometimes means that some porting effort is needed.
Find attached a draft patch for gnulib, to support the stdioext functions
on musl.

It was tested using the recipe from
http://lists.gnu.org/archive/html/bug-gnulib/2012-02/msg00022.html
All tests pass, except for a bug in setvbuf or __fbufsize, that I have
reported separately.

There are two areas where musl libc can be modified to reduce the size
of this patch.

1) Currently, gnulib has to go to a great length to detect musl.
   It uses the presence of __stdio_read and __stdio_write as an indicator;
   no other system has these functions. Why does musl make it so hard?
   Can't you just define a preprocessor symbol that indicates musl?
   Like so many other systems do?
   
http://sourceforge.net/apps/mediawiki/predef/index.php?title=Operating_Systems
   http://sourceforge.net/apps/mediawiki/predef/index.php?title=Libraries
   The idea is that once a program includes  (or any other standard
   .h file) it can use a "#ifdef __MUSL__".

   gnulib needs this because some implementations provide the same functions
   but they behave differently. E.g. __freading is broken in glibc < 2.7
   but not in musl. And __freading and __fwriting don't implement exactly
   the same semantics as glibc. That's why real-life programs need to
   #ifdef according to OS and libraries.

2) Provide 4 primitive functions.

Specified in gnulib's freadahead.h:

/* Assuming the stream FP is open for reading:
   Return the number of bytes waiting in the input buffer of FP.
   This includes both the bytes that have been read from the underlying input
   source and the bytes that have been pushed back through 'ungetc'.
   If this number is 0 and the stream is not currently writing,
   fflush (FP) is known to be a no-op.
   FP must not be wide-character oriented.  */
size_t __freadahead (FILE *fp)
{
  if (!__freadable (fp))
return 0;
  return fp->rend - fp->rpos;
}

Specified in gnulib's freadptr.h:

/* Assuming the stream FP is open for reading:
   Return a pointer to the input buffer of FP, or NULL.
   If the returned pointer is non-NULL, *SIZEP is set to the (positive) size
   of the input buffer.
   If the returned pointer is NULL, you should use getc (FP),
   fgetc (FP), or fread (..., FP) to access the input from FP.
   The resulting pointer becomes invalid upon any operation on FP.
   FP must not be wide-character oriented.  */
const char * __freadptr (FILE *fp, size_t *sizep)
{
  if (fp->rpos >= fp->rend)
return NULL;
  *sizep = fp->rend - fp->rpos;
  return fp->rpos;
}

/* Increment the in-memory pointer.  INCREMENT must be at most the
   buffer size returned by freadptr().  */
void __freadptrinc (FILE *fp, size_t increment)
{
  fp->rpos += increment;
}

Specified in gnulib' fseterr.h:

/* Set the error indicator of the stream FP.
   The "error indicator" is set when an I/O operation on the stream fails, and
   is cleared (together with the "end-of-file" indicator) by clearerr (FP).  */
void __fseterr (FILE *fp)
{
  fp->flags |= F_ERR;
}

Rich Felker wrote on 2012-06-11:
> would adding __freadahead be sufficient to make it use
> __freadahead, or does gnulib hard-code knowledge about which systems
> have which stdio extension functions?

Adding __freadahead is one of the things needed for gnulib. Another thing
is to add an indicator such as __MUSL__, because gnulib hard-codes the
knowledge of which of these functions are known to work or known to be
buggy.

Rich Felker wrote on 2012-06-11:
> If you're willing
> to put in a test for it, I'm willing to add the function.

4 functions proposals are above. Please let us know what you put into
musl, so that I can reduce the size of the patch I put into gnulib.

Bruno


--- lib/fbufmode.c.orig Sun Jun 17 22:26:32 2012
+++ lib/fbufmode.c  Sun Jun 17 21:43:36 2012
@@ -79,6 +79,10 @@
   if (fp->__linebuf)
 return _IOLBF;
   return (fp->__bufsize > 0 ? _IOFBF : _IONBF);
+#elif defined __MUSL__  /* musl libc */
+  if (__flbf (fp))
+return _IOLBF;
+  return (__fbufsize (fp) > 0 ? _IOFBF : _IONBF);
 #elif defined EPLAN9/* Plan9 */
   if (fp->flags & 2 /* LINEBUF */)
 return _IOLBF;
--- lib/fpurge.c.orig   Sun Jun 17 22:26:32 2012
+++ lib/fpurge.cSun Jun 17 21:06:31 2012
@@ -29,7 +29,7 @@
 int
 fpurge (FILE *fp)
 {
-#if HAVE___FPURGE   /* glibc >= 2.2, Haiku, Solaris >= 7 */
+#if HAVE___FPURGE   /* glibc >= 2.2, Haiku, Solaris >= 7, musl 
libc */
 
   __fpurge (fp);
   /* The __fpurge function