from:"Romain Beauxis"

Re: [FFmpeg-devel] [PATCH 2/2] Add fate test for hls metadata update.

2024-04-12 Thread Romain Beauxis

Le jeu. 11 avr. 2024 à 09:17, Andreas Rheinhardt <
andreas.rheinha...@outlook.com> a écrit :

> Romain Beauxis:
> > This patch adds a FATE test for the new HLS metadata update. The fate
> > sample consists of a two segment AAC hls stream. The first segment has
> > test title 1 as title metadata while the second has test title 2.
> >
> > In the log, we can see that test title 2 is reported for the stream,
> > indicating that the metadata was updated with the second segment.
> >
> > Romain
> > ---
> >  tests/fate/demux.mak   |   3 +
> >  tests/ref/fate/hls-adts-meta-demux | 124 +
> >  2 files changed, 127 insertions(+)
> >  create mode 100644 tests/ref/fate/hls-adts-meta-demux
> >
> > diff --git a/tests/fate/demux.mak b/tests/fate/demux.mak
> > index d9b9045f0b..7243c2c163 100644
> > --- a/tests/fate/demux.mak
> > +++ b/tests/fate/demux.mak
> > @@ -160,6 +160,9 @@ fate-ts-demux: CMD = ffprobe_demux
> $(TARGET_SAMPLES)/ac3/mp3ac325-4864-small.ts
> >  FATE_FFPROBE_DEMUX-$(CONFIG_MPEGTS_DEMUXER) += fate-ts-timed-id3-demux
> >  fate-ts-timed-id3-demux: CMD = ffprobe_demux
> $(TARGET_SAMPLES)/mpegts/id3.ts
> >
> > +FATE_FFPROBE_DEMUX-$(CONFIG_MPEGTS_DEMUXER) += fate-hls-adts-meta-demux
> > +fate-fate-hls-adts-meta-demux: CMD = ffprobe_demux
> $(TARGET_SAMPLES)/hls-adts-meta/stream.m3u8
> > +
> >  FATE_SAMPLES_DEMUX += $(FATE_SAMPLES_DEMUX-yes)
> >  FATE_SAMPLES_FFMPEG += $(FATE_SAMPLES_DEMUX)
> >  FATE_FFPROBE_DEMUX   += $(FATE_FFPROBE_DEMUX-yes)
> > diff --git a/tests/ref/fate/hls-adts-meta-demux
> b/tests/ref/fate/hls-adts-meta-demux
> > new file mode 100644
> > index 00..ab944695fc
> > --- /dev/null
> > +++ b/tests/ref/fate/hls-adts-meta-demux
> > @@ -0,0 +1,124 @@
> >
> +packet|codec_type=audio|stream_index=0|pts=3416400|pts_time=37.96|dts=3416400|dts_time=37.96|duration=2090|duration_time=0.023222|size=368|pos=0|flags=K__|data_hash=CRC32:c371b0d9
> >
> +packet|codec_type=audio|stream_index=0|pts=3418490|pts_time=37.983222|dts=3418490|dts_time=37.983222|duration=2090|duration_time=0.023222|size=390|pos=368|flags=K__|data_hash=CRC32:950c52b2
> >
> +packet|codec_type=audio|stream_index=0|pts=3420580|pts_time=38.006444|dts=3420580|dts_time=38.006444|duration=2090|duration_time=0.023222|size=357|pos=758|flags=K__|data_hash=CRC32:3e672212
> >
> +packet|codec_type=audio|stream_index=0|pts=3422669|pts_time=38.029656|dts=3422669|dts_time=38.029656|duration=2090|duration_time=0.023222|size=426|pos=1115|flags=K__|data_hash=CRC32:817b6e4c
> >
> +packet|codec_type=audio|stream_index=0|pts=3424759|pts_time=38.052878|dts=3424759|dts_time=38.052878|duration=2090|duration_time=0.023222|size=368|pos=1541|flags=K__|data_hash=CRC32:c4c6e1ed
> >
> +packet|codec_type=audio|stream_index=0|pts=3426849|pts_time=38.076100|dts=3426849|dts_time=38.076100|duration=2090|duration_time=0.023222|size=389|pos=1909|flags=K__|data_hash=CRC32:67cb6dd9
> >
> +packet|codec_type=audio|stream_index=0|pts=3428939|pts_time=38.099322|dts=3428939|dts_time=38.099322|duration=2090|duration_time=0.023222|size=352|pos=2298|flags=K__|data_hash=CRC32:7a56ff53
> >
> +packet|codec_type=audio|stream_index=0|pts=3431029|pts_time=38.122544|dts=3431029|dts_time=38.122544|duration=2090|duration_time=0.023222|size=378|pos=2650|flags=K__|data_hash=CRC32:f8d5ef58
> >
> +packet|codec_type=audio|stream_index=0|pts=3433118|pts_time=38.145756|dts=3433118|dts_time=38.145756|duration=2090|duration_time=0.023222|size=384|pos=3028|flags=K__|data_hash=CRC32:73a4fb1c
> >
> +packet|codec_type=audio|stream_index=0|pts=3435208|pts_time=38.168978|dts=3435208|dts_time=38.168978|duration=2090|duration_time=0.023222|size=353|pos=3412|flags=K__|data_hash=CRC32:4ea999b5
> >
> +packet|codec_type=audio|stream_index=0|pts=3437298|pts_time=38.192200|dts=3437298|dts_time=38.192200|duration=2090|duration_time=0.023222|size=417|pos=3765|flags=K__|data_hash=CRC32:4540aec8
> >
> +packet|codec_type=audio|stream_index=0|pts=3439388|pts_time=38.215422|dts=3439388|dts_time=38.215422|duration=2090|duration_time=0.023222|size=361|pos=4182|flags=K__|data_hash=CRC32:635a04f4
> >
> +packet|codec_type=audio|stream_index=0|pts=3441478|pts_time=38.238644|dts=3441478|dts_time=38.238644|duration=2090|duration_time=0.023222|size=399|pos=4543|flags=K__|data_hash=CRC32:94583c18
> >
> +packet|codec_type=audio|stream_index=0|pts=3443567|pts_time=38.261856|dts=3443567|dts_time=38.261856|duration=2090|duration_time=0.023222|size=384|pos=4942|flags=K__|data_hash=CRC32:21070d79
> >
> +packet|codec_type=audio|stream_index=0|pts=3445657|pts_time=38.285078|dts=3445657|dts_time=38.285078|duration=2090|duration_ti

Re: [FFmpeg-devel] [PATCH 1/2] libavformat/hls.c: support in-stream ID3 metadata update.

2024-04-11 Thread Romain Beauxis

Hi all,

Le dim. 7 avr. 2024 à 09:46, Romain Beauxis  a
écrit :

>
>
> Le dim. 7 avr. 2024 à 05:44, Steven Liu  a
> écrit :
>
>> Romain Beauxis  于2024年3月26日周二 08:58写道：
>> >
>> > This patch adds support for updating HLS metadata passed as ID3 frames.
>> >
>> > This seems like a pretty straight-forward improvement. Updating the
>> > metadaata of the first stream seems to be the mechanism is other places
>> > in the code and works as expected.
>>
>>
Would it be possible to get this patch committed? My understanding is that
it has been reviewed so this should be the next step?

The second patch adding FATE tests could be committed separately if it
causes issues. The samples for it are here:
https://www.dropbox.com/scl/fo/1x74ztoa6yo9q49ignfnt/h?rlkey=xvg5nhgjr515gm6b375evm8n4=0
and
should be placed into a $FATE_SAMPLES/hls-adts-meta directory.

Thanks!


> > ---
>> >  libavformat/hls.c | 54 ---
>> >  1 file changed, 32 insertions(+), 22 deletions(-)
>> >
>> > diff --git a/libavformat/hls.c b/libavformat/hls.c
>> > index f6b44c2e35..ba6634d57a 100644
>> > --- a/libavformat/hls.c
>> > +++ b/libavformat/hls.c
>> > @@ -93,6 +93,12 @@ enum PlaylistType {
>> >  PLS_TYPE_VOD
>> >  };
>> >
>> > +#define ID3_PRIV_OWNER_TS
>> "com.apple.streaming.transportStreamTimestamp"
>> > +#define ID3_PRIV_OWNER_AUDIO_SETUP
>> "com.apple.streaming.audioDescription"
>> > +
>> > +#define ID3v2_PRIV_OWNER_TS ID3v2_PRIV_METADATA_PREFIX
>> ID3_PRIV_OWNER_TS
>> > +#define ID3v2_PRIV_OWNER_AUDIO_SETUP ID3v2_PRIV_METADATA_PREFIX
>> ID3_PRIV_OWNER_AUDIO_SETUP
>> > +
>> >  /*
>> >   * Each playlist has its own demuxer. If it currently is active,
>> >   * it has an open AVIOContext too, and potentially an AVPacket
>> > @@ -150,9 +156,7 @@ struct playlist {
>> >  int64_t id3_offset; /* in stream original tb */
>> >  uint8_t* id3_buf; /* temp buffer for id3 parsing */
>> >  unsigned int id3_buf_size;
>> > -AVDictionary *id3_initial; /* data from first id3 tag */
>> > -int id3_found; /* ID3 tag found at some point */
>> > -int id3_changed; /* ID3 tag data has changed at some point */
>> > +AVDictionary *last_id3; /* data from the last id3 tag */
>> >  ID3v2ExtraMeta *id3_deferred_extra; /* stored here until
>> subdemuxer is opened */
>> >
>> >  HLSAudioSetupInfo audio_setup_info;
>> > @@ -270,7 +274,7 @@ static void free_playlist_list(HLSContext *c)
>> >  av_freep(>main_streams);
>> >  av_freep(>renditions);
>> >  av_freep(>id3_buf);
>> > -av_dict_free(>id3_initial);
>> > +av_dict_free(>last_id3);
>> >  ff_id3v2_free_extra_meta(>id3_deferred_extra);
>> >  av_freep(>init_sec_buf);
>> >  av_packet_free(>pkt);
>> > @@ -1083,15 +1087,13 @@ static void parse_id3(AVFormatContext *s,
>> AVIOContext *pb,
>> >AVDictionary **metadata, int64_t *dts,
>> HLSAudioSetupInfo *audio_setup_info,
>> >ID3v2ExtraMetaAPIC **apic, ID3v2ExtraMeta
>> **extra_meta)
>> >  {
>> > -static const char id3_priv_owner_ts[] =
>> "com.apple.streaming.transportStreamTimestamp";
>> > -static const char id3_priv_owner_audio_setup[] =
>> "com.apple.streaming.audioDescription";
>> >  ID3v2ExtraMeta *meta;
>> >
>> >  ff_id3v2_read_dict(pb, metadata, ID3v2_DEFAULT_MAGIC, extra_meta);
>> >  for (meta = *extra_meta; meta; meta = meta->next) {
>> >  if (!strcmp(meta->tag, "PRIV")) {
>> >  ID3v2ExtraMetaPRIV *priv = >data.priv;
>> > -if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
>> id3_priv_owner_ts, 44)) {
>> > +if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
>> ID3_PRIV_OWNER_TS, strlen(ID3_PRIV_OWNER_TS))) {
>> >  /* 33-bit MPEG timestamp */
>> >  int64_t ts = AV_RB64(priv->data);
>> >  av_log(s, AV_LOG_DEBUG, "HLS ID3 audio timestamp
>> %"PRId64"\n", ts);
>> > @@ -1099,7 +1101,9 @@ static void parse_id3(AVFormatContext *s,
>> AVIOContext *pb,
>> >  *dts = ts;
>> >  else
>> >

Re: [FFmpeg-devel] Query from Reuters on XZ, open source, and Microsoft

2024-04-09 Thread Romain Beauxis

Le mar. 9 avr. 2024 à 18:46, Paul B Mahol  a écrit :

> On Tue, Apr 9, 2024 at 10:57 PM Romain Beauxis 
> wrote:
>
> > [Apologies for continuing the conversation, Rémi]
> >
> > Le mar. 9 avr. 2024 à 14:05, Tomas Härdin  a écrit :
> >
> > > mån 2024-04-08 klockan 13:13 -0500 skrev Romain Beauxis:
> > > > On Wed, Apr 3, 2024, 11:39 Kieran Kunhya via ffmpeg-devel <
> > > > ffmpeg-devel@ffmpeg.org> wrote:
> > > >
> > > > > Hi Raphael,
> > > > >
> > > > > I was the author of the tweet and I gave a short talk about this
> > > > > topic at
> > > > > Demuxed at a video conference last year:
> > > > > https://m.youtube.com/watch?v=OIyOEuQQsCQ=930s
> > > > >
> > > > > That said this is a community project and it would be best to
> > > > > continue the
> > > > > discussion on this mailing list unless agreed otherwise.
> > > > >
> > > >
> > > > Thank you for sharing your talk. It's indeed unfortunate that large
> > > > companies are not more generous with the projects they depend on
> > > > _heavily_
> > > >
> > > > I would like to offer a constructive feedback really not intended as
> > > > trolling. I believe that supporting a more modern development
> > > > workflow such
> > > > as the GitHub PR or gitlab MR would go very long way in helping
> > > > onboarding
> > > > new developers.
> > >
> > > Considering which company owns GitHub and which company was the target
> > > of Kieran's criticism, this seems ill-adviced
> > >
> >
> > Would you mind explaining what you mean exactly? I want to respond but
> I'm
> > not sure if I fully understand your point here.
> >
> >
> Kieran's criticism is trolling, as Kieran and rest of FFmpeg devs use
> regularly and passionately Various Big Corpo products all the time.
>
> Kieran's criticism is unfounded one. As I, originally 'volunteered' in that
> now 'famous' ticket about Certain Big Corpo bug report and kindly replied
> in friendly manner to bug reporter and give reporter free support, me still
> see no issues in that mine action.
>
> I strictly do make difference between collective and single specific
> person.
>


Thanks for the clarification, that's a level of nuance I did not grasp.

>
> > > Also as someone who had to maintain a Gitlab instance at uni for a
> > > couple of years, I agree with Rémi's points
> > >
> >
> > My initial contribution was motivated by the argument presented in the
> > original talk that bringing new blood is critical to the survival of the
> > project.
> >
>
> Projects go and die, and new ones rise up, all the time.
>
>
> >
> > If so, then I do believe that there must be a compromise to be made
> between
> > being easier to join for new developers and changing the existing
> workflow.
> > I'm also aware that changing the existing workflow has been discussed
> > before.
> >
> > I don't think that media is not cool anymore, as argued in the talk. I
> see
> > a _lot_ of interested developers in my other projects and all over the
> open
> > source landscape. That's why I believe that it's also important to
> consider
> > other reasons than the talk's argument.
> >
> > Being someone actually trying to contribute, with years of developers
> > experience and involvement in other communities, I was thinking that I
> may
> > be able to bring in more new context to the topic.
> >
> > If there's interest in continuing the discussion, could you or Rémi be
> > willing to explain what kind of burden gitlab would add?
> >
>
> Infrastructure maintenance, no volunteers for gitlab repo admin.
> Extra steps to setup account and X-factor authentication.
>
>
You have a little bit of a bias issue here I believe. :-)

If you restrict the contributors only to people who can do git send-email
and irc then, of course, you won't find many volunteers to maintain a
gitlab repo. And, of course, the majority of the people voicing their
opinion will be in favor of what they're already familiar with.

There's always a learning curve to adopting new tools. Matter of fact, I'm
old enough to remember myself complaining about git when it was taking over
SVN.. 


> >
> > Also, Rémi said this would make it harder to join the project, in which
> way
> > exactly?
> >
> > Otherwise, I'm happy to let this die and return to trying to get my patch
> > com

Re: [FFmpeg-devel] Query from Reuters on XZ, open source, and Microsoft

2024-04-09 Thread Romain Beauxis

[Apologies for continuing the conversation, Rémi]

Le mar. 9 avr. 2024 à 14:05, Tomas Härdin  a écrit :

> mån 2024-04-08 klockan 13:13 -0500 skrev Romain Beauxis:
> > On Wed, Apr 3, 2024, 11:39 Kieran Kunhya via ffmpeg-devel <
> > ffmpeg-devel@ffmpeg.org> wrote:
> >
> > > Hi Raphael,
> > >
> > > I was the author of the tweet and I gave a short talk about this
> > > topic at
> > > Demuxed at a video conference last year:
> > > https://m.youtube.com/watch?v=OIyOEuQQsCQ=930s
> > >
> > > That said this is a community project and it would be best to
> > > continue the
> > > discussion on this mailing list unless agreed otherwise.
> > >
> >
> > Thank you for sharing your talk. It's indeed unfortunate that large
> > companies are not more generous with the projects they depend on
> > _heavily_
> >
> > I would like to offer a constructive feedback really not intended as
> > trolling. I believe that supporting a more modern development
> > workflow such
> > as the GitHub PR or gitlab MR would go very long way in helping
> > onboarding
> > new developers.
>
> Considering which company owns GitHub and which company was the target
> of Kieran's criticism, this seems ill-adviced
>

Would you mind explaining what you mean exactly? I want to respond but I'm
not sure if I fully understand your point here.

> Also as someone who had to maintain a Gitlab instance at uni for a
> couple of years, I agree with Rémi's points
>

My initial contribution was motivated by the argument presented in the
original talk that bringing new blood is critical to the survival of the
project.

If so, then I do believe that there must be a compromise to be made between
being easier to join for new developers and changing the existing workflow.
I'm also aware that changing the existing workflow has been discussed
before.

I don't think that media is not cool anymore, as argued in the talk. I see
a _lot_ of interested developers in my other projects and all over the open
source landscape. That's why I believe that it's also important to consider
other reasons than the talk's argument.

Being someone actually trying to contribute, with years of developers
experience and involvement in other communities, I was thinking that I may
be able to bring in more new context to the topic.

If there's interest in continuing the discussion, could you or Rémi be
willing to explain what kind of burden gitlab would add?

Also, Rémi said this would make it harder to join the project, in which way
exactly?

Otherwise, I'm happy to let this die and return to trying to get my patch
committed.. 

Thanks,
-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] Query from Reuters on XZ, open source, and Microsoft

2024-04-08 Thread Romain Beauxis

On Wed, Apr 3, 2024, 11:39 Kieran Kunhya via ffmpeg-devel <
ffmpeg-devel@ffmpeg.org> wrote:

> Hi Raphael,
>
> I was the author of the tweet and I gave a short talk about this topic at
> Demuxed at a video conference last year:
> https://m.youtube.com/watch?v=OIyOEuQQsCQ=930s
>
> That said this is a community project and it would be best to continue the
> discussion on this mailing list unless agreed otherwise.
>

Thank you for sharing your talk. It's indeed unfortunate that large
companies are not more generous with the projects they depend on _heavily_

I would like to offer a constructive feedback really not intended as
trolling. I believe that supporting a more modern development workflow such
as the GitHub PR or gitlab MR would go very long way in helping onboarding
new developers.

I have been trying to contribute to the project for perhaps 18 months and,
while I think I'm getting better at it, I still find it incredibly complex
and hard to ramp up to.

The other open source community that I am involved with, the OCaml
compiler, moved to GitHub maybe 5/6 years ago and this has really done
wonders bringing in new contributors. And, you know, this is a niche
compiler that is also not getting much attention from AI/Machine Learning
people.

> Regards,
> Kieran Kunhya
>
> On Wed, 3 Apr 2024, 16:52 Satter, Raphael (Reuters) via ffmpeg-devel, <
> ffmpeg-devel@ffmpeg.org> wrote:
>
> > Dear FFMPEG community,
> >
> > This is Raphael Satter, a journalist with Reuters. I saw your thread on X
> > about your experience with Microsoft in the context of the XZ story and
> > would love to follow up with a few questions.
> >
> > Is there a good person to talk to? I’m reachable using the details below
> > or on Signal/WhatsApp at +1 202 430 9389.
> >
> > Raphael
> >
> >  raphael.sat...@tr.com
> >  raphaelsatter.com
> >  reuters.com/authors/raphael-satter
> >
> > Thomson Reuters
> > 1333 H Street NW
> > Washington, DC 20005
> >
> > ☎️ +1 202 843-6302
> >
> > ___
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
> >
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
>
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/2] libavformat/hls.c: support in-stream ID3 metadata update.

2024-04-07 Thread Romain Beauxis

Le dim. 7 avr. 2024 à 05:44, Steven Liu  a écrit :

> Romain Beauxis  于2024年3月26日周二 08:58写道：
> >
> > This patch adds support for updating HLS metadata passed as ID3 frames.
> >
> > This seems like a pretty straight-forward improvement. Updating the
> > metadaata of the first stream seems to be the mechanism is other places
> > in the code and works as expected.
> >
> > ---
> >  libavformat/hls.c | 54 ---
> >  1 file changed, 32 insertions(+), 22 deletions(-)
> >
> > diff --git a/libavformat/hls.c b/libavformat/hls.c
> > index f6b44c2e35..ba6634d57a 100644
> > --- a/libavformat/hls.c
> > +++ b/libavformat/hls.c
> > @@ -93,6 +93,12 @@ enum PlaylistType {
> >  PLS_TYPE_VOD
> >  };
> >
> > +#define ID3_PRIV_OWNER_TS "com.apple.streaming.transportStreamTimestamp"
> > +#define ID3_PRIV_OWNER_AUDIO_SETUP
> "com.apple.streaming.audioDescription"
> > +
> > +#define ID3v2_PRIV_OWNER_TS ID3v2_PRIV_METADATA_PREFIX ID3_PRIV_OWNER_TS
> > +#define ID3v2_PRIV_OWNER_AUDIO_SETUP ID3v2_PRIV_METADATA_PREFIX
> ID3_PRIV_OWNER_AUDIO_SETUP
> > +
> >  /*
> >   * Each playlist has its own demuxer. If it currently is active,
> >   * it has an open AVIOContext too, and potentially an AVPacket
> > @@ -150,9 +156,7 @@ struct playlist {
> >  int64_t id3_offset; /* in stream original tb */
> >  uint8_t* id3_buf; /* temp buffer for id3 parsing */
> >  unsigned int id3_buf_size;
> > -AVDictionary *id3_initial; /* data from first id3 tag */
> > -int id3_found; /* ID3 tag found at some point */
> > -int id3_changed; /* ID3 tag data has changed at some point */
> > +AVDictionary *last_id3; /* data from the last id3 tag */
> >  ID3v2ExtraMeta *id3_deferred_extra; /* stored here until subdemuxer
> is opened */
> >
> >  HLSAudioSetupInfo audio_setup_info;
> > @@ -270,7 +274,7 @@ static void free_playlist_list(HLSContext *c)
> >  av_freep(>main_streams);
> >  av_freep(>renditions);
> >  av_freep(>id3_buf);
> > -av_dict_free(>id3_initial);
> > +av_dict_free(>last_id3);
> >  ff_id3v2_free_extra_meta(>id3_deferred_extra);
> >  av_freep(>init_sec_buf);
> >  av_packet_free(>pkt);
> > @@ -1083,15 +1087,13 @@ static void parse_id3(AVFormatContext *s,
> AVIOContext *pb,
> >AVDictionary **metadata, int64_t *dts,
> HLSAudioSetupInfo *audio_setup_info,
> >ID3v2ExtraMetaAPIC **apic, ID3v2ExtraMeta
> **extra_meta)
> >  {
> > -static const char id3_priv_owner_ts[] =
> "com.apple.streaming.transportStreamTimestamp";
> > -static const char id3_priv_owner_audio_setup[] =
> "com.apple.streaming.audioDescription";
> >  ID3v2ExtraMeta *meta;
> >
> >  ff_id3v2_read_dict(pb, metadata, ID3v2_DEFAULT_MAGIC, extra_meta);
> >  for (meta = *extra_meta; meta; meta = meta->next) {
> >  if (!strcmp(meta->tag, "PRIV")) {
> >  ID3v2ExtraMetaPRIV *priv = >data.priv;
> > -if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
> id3_priv_owner_ts, 44)) {
> > +if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
> ID3_PRIV_OWNER_TS, strlen(ID3_PRIV_OWNER_TS))) {
> >  /* 33-bit MPEG timestamp */
> >  int64_t ts = AV_RB64(priv->data);
> >  av_log(s, AV_LOG_DEBUG, "HLS ID3 audio timestamp
> %"PRId64"\n", ts);
> > @@ -1099,7 +1101,9 @@ static void parse_id3(AVFormatContext *s,
> AVIOContext *pb,
> >  *dts = ts;
> >  else
> >  av_log(s, AV_LOG_ERROR, "Invalid HLS ID3 audio
> timestamp %"PRId64"\n", ts);
> > -} else if (priv->datasize >= 8 &&
> !av_strncasecmp(priv->owner, id3_priv_owner_audio_setup, 36)) {
> > +} else if (priv->datasize >= 8 &&
> > +   !av_strncasecmp(priv->owner,
> ID3_PRIV_OWNER_AUDIO_SETUP, 36) &&
> > +   audio_setup_info) {
> >  ff_hls_senc_read_audio_setup_info(audio_setup_info,
> priv->data, priv->datasize);
> >  }
> >  } else if (!strcmp(meta->tag, "APIC") && apic)
> > @@ -1113,9 +1117,10 @@ static int id3_has_changed_values(struct playlist
> *pls, AVDictionary *metadata,
&g

Re: [FFmpeg-devel] [PATCH 1/2] libavformat/hls.c: support in-stream ID3 metadata update.

2024-04-06 Thread Romain Beauxis

On Sun, Mar 31, 2024, 12:46 Romain Beauxis  wrote:

>
>
> On Sun, Mar 31, 2024, 05:52 Liu Steven  wrote:
>
>>
>>
>> > On Mar 29, 2024, at 06:51, Romain Beauxis  wrote:
>> >
>> > On Mon, Mar 25, 2024, 19:58 Romain Beauxis 
>> wrote:
>> >
>> >> This patch adds support for updating HLS metadata passed as ID3 frames.
>> >>
>> >> This seems like a pretty straight-forward improvement. Updating the
>> >> metadaata of the first stream seems to be the mechanism is other places
>> >> in the code and works as expected.
>> >>
>> >
>> >
>> > Hello!
>> >
>> > Any interest in reviewing this?
>>
>> Patchset looks good to me, looks better than before patch.
>>
>
> Great! Happy to provide the fate samples too.
>
>
Hi!

Any update on this?

Let me know if I can do anything to help!




> > ---
>> >> libavformat/hls.c | 54 ---
>> >> 1 file changed, 32 insertions(+), 22 deletions(-)
>> >>
>> >> diff --git a/libavformat/hls.c b/libavformat/hls.c
>> >> index f6b44c2e35..ba6634d57a 100644
>> >> --- a/libavformat/hls.c
>> >> +++ b/libavformat/hls.c
>> >> @@ -93,6 +93,12 @@ enum PlaylistType {
>> >> PLS_TYPE_VOD
>> >> };
>> >>
>> >> +#define ID3_PRIV_OWNER_TS
>> "com.apple.streaming.transportStreamTimestamp"
>> >> +#define ID3_PRIV_OWNER_AUDIO_SETUP
>> "com.apple.streaming.audioDescription"
>> >> +
>> >> +#define ID3v2_PRIV_OWNER_TS ID3v2_PRIV_METADATA_PREFIX
>> ID3_PRIV_OWNER_TS
>> >> +#define ID3v2_PRIV_OWNER_AUDIO_SETUP ID3v2_PRIV_METADATA_PREFIX
>> >> ID3_PRIV_OWNER_AUDIO_SETUP
>> >> +
>> >> /*
>> >>  * Each playlist has its own demuxer. If it currently is active,
>> >>  * it has an open AVIOContext too, and potentially an AVPacket
>> >> @@ -150,9 +156,7 @@ struct playlist {
>> >> int64_t id3_offset; /* in stream original tb */
>> >> uint8_t* id3_buf; /* temp buffer for id3 parsing */
>> >> unsigned int id3_buf_size;
>> >> -AVDictionary *id3_initial; /* data from first id3 tag */
>> >> -int id3_found; /* ID3 tag found at some point */
>> >> -int id3_changed; /* ID3 tag data has changed at some point */
>> >> +AVDictionary *last_id3; /* data from the last id3 tag */
>> >> ID3v2ExtraMeta *id3_deferred_extra; /* stored here until subdemuxer
>> >> is opened */
>> >>
>> >> HLSAudioSetupInfo audio_setup_info;
>> >> @@ -270,7 +274,7 @@ static void free_playlist_list(HLSContext *c)
>> >> av_freep(>main_streams);
>> >> av_freep(>renditions);
>> >> av_freep(>id3_buf);
>> >> -av_dict_free(>id3_initial);
>> >> +av_dict_free(>last_id3);
>> >> ff_id3v2_free_extra_meta(>id3_deferred_extra);
>> >> av_freep(>init_sec_buf);
>> >> av_packet_free(>pkt);
>> >> @@ -1083,15 +1087,13 @@ static void parse_id3(AVFormatContext *s,
>> >> AVIOContext *pb,
>> >>   AVDictionary **metadata, int64_t *dts,
>> >> HLSAudioSetupInfo *audio_setup_info,
>> >>   ID3v2ExtraMetaAPIC **apic, ID3v2ExtraMeta
>> >> **extra_meta)
>> >> {
>> >> -static const char id3_priv_owner_ts[] =
>> >> "com.apple.streaming.transportStreamTimestamp";
>> >> -static const char id3_priv_owner_audio_setup[] =
>> >> "com.apple.streaming.audioDescription";
>> >> ID3v2ExtraMeta *meta;
>> >>
>> >> ff_id3v2_read_dict(pb, metadata, ID3v2_DEFAULT_MAGIC, extra_meta);
>> >> for (meta = *extra_meta; meta; meta = meta->next) {
>> >> if (!strcmp(meta->tag, "PRIV")) {
>> >> ID3v2ExtraMetaPRIV *priv = >data.priv;
>> >> -if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
>> >> id3_priv_owner_ts, 44)) {
>> >> +if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
>> >> ID3_PRIV_OWNER_TS, strlen(ID3_PRIV_OWNER_TS))) {
>> >> /* 33-bit MPEG timestamp */
>> >> int64_t ts = AV_RB64(priv->data);
>> >>

Re: [FFmpeg-devel] [PATCH 1/2] libavformat/hls.c: support in-stream ID3 metadata update.

2024-03-31 Thread Romain Beauxis

On Sun, Mar 31, 2024, 05:52 Liu Steven  wrote:

>
>
> > On Mar 29, 2024, at 06:51, Romain Beauxis  wrote:
> >
> > On Mon, Mar 25, 2024, 19:58 Romain Beauxis  wrote:
> >
> >> This patch adds support for updating HLS metadata passed as ID3 frames.
> >>
> >> This seems like a pretty straight-forward improvement. Updating the
> >> metadaata of the first stream seems to be the mechanism is other places
> >> in the code and works as expected.
> >>
> >
> >
> > Hello!
> >
> > Any interest in reviewing this?
>
> Patchset looks good to me, looks better than before patch.
>

Great! Happy to provide the fate samples too.


> ---
> >> libavformat/hls.c | 54 ---
> >> 1 file changed, 32 insertions(+), 22 deletions(-)
> >>
> >> diff --git a/libavformat/hls.c b/libavformat/hls.c
> >> index f6b44c2e35..ba6634d57a 100644
> >> --- a/libavformat/hls.c
> >> +++ b/libavformat/hls.c
> >> @@ -93,6 +93,12 @@ enum PlaylistType {
> >> PLS_TYPE_VOD
> >> };
> >>
> >> +#define ID3_PRIV_OWNER_TS
> "com.apple.streaming.transportStreamTimestamp"
> >> +#define ID3_PRIV_OWNER_AUDIO_SETUP
> "com.apple.streaming.audioDescription"
> >> +
> >> +#define ID3v2_PRIV_OWNER_TS ID3v2_PRIV_METADATA_PREFIX
> ID3_PRIV_OWNER_TS
> >> +#define ID3v2_PRIV_OWNER_AUDIO_SETUP ID3v2_PRIV_METADATA_PREFIX
> >> ID3_PRIV_OWNER_AUDIO_SETUP
> >> +
> >> /*
> >>  * Each playlist has its own demuxer. If it currently is active,
> >>  * it has an open AVIOContext too, and potentially an AVPacket
> >> @@ -150,9 +156,7 @@ struct playlist {
> >> int64_t id3_offset; /* in stream original tb */
> >> uint8_t* id3_buf; /* temp buffer for id3 parsing */
> >> unsigned int id3_buf_size;
> >> -AVDictionary *id3_initial; /* data from first id3 tag */
> >> -int id3_found; /* ID3 tag found at some point */
> >> -int id3_changed; /* ID3 tag data has changed at some point */
> >> +AVDictionary *last_id3; /* data from the last id3 tag */
> >> ID3v2ExtraMeta *id3_deferred_extra; /* stored here until subdemuxer
> >> is opened */
> >>
> >> HLSAudioSetupInfo audio_setup_info;
> >> @@ -270,7 +274,7 @@ static void free_playlist_list(HLSContext *c)
> >> av_freep(>main_streams);
> >> av_freep(>renditions);
> >> av_freep(>id3_buf);
> >> -av_dict_free(>id3_initial);
> >> +av_dict_free(>last_id3);
> >> ff_id3v2_free_extra_meta(>id3_deferred_extra);
> >> av_freep(>init_sec_buf);
> >> av_packet_free(>pkt);
> >> @@ -1083,15 +1087,13 @@ static void parse_id3(AVFormatContext *s,
> >> AVIOContext *pb,
> >>   AVDictionary **metadata, int64_t *dts,
> >> HLSAudioSetupInfo *audio_setup_info,
> >>   ID3v2ExtraMetaAPIC **apic, ID3v2ExtraMeta
> >> **extra_meta)
> >> {
> >> -static const char id3_priv_owner_ts[] =
> >> "com.apple.streaming.transportStreamTimestamp";
> >> -static const char id3_priv_owner_audio_setup[] =
> >> "com.apple.streaming.audioDescription";
> >> ID3v2ExtraMeta *meta;
> >>
> >> ff_id3v2_read_dict(pb, metadata, ID3v2_DEFAULT_MAGIC, extra_meta);
> >> for (meta = *extra_meta; meta; meta = meta->next) {
> >> if (!strcmp(meta->tag, "PRIV")) {
> >> ID3v2ExtraMetaPRIV *priv = >data.priv;
> >> -if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
> >> id3_priv_owner_ts, 44)) {
> >> +if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
> >> ID3_PRIV_OWNER_TS, strlen(ID3_PRIV_OWNER_TS))) {
> >> /* 33-bit MPEG timestamp */
> >> int64_t ts = AV_RB64(priv->data);
> >> av_log(s, AV_LOG_DEBUG, "HLS ID3 audio timestamp
> >> %"PRId64"\n", ts);
> >> @@ -1099,7 +1101,9 @@ static void parse_id3(AVFormatContext *s,
> >> AVIOContext *pb,
> >> *dts = ts;
> >> else
> >> av_log(s, AV_LOG_ERROR, "Invalid HLS ID3 audio
> >> timestamp %"PRId64"\n", ts);
> >> -} else if (priv->datasize >=

Re: [FFmpeg-devel] [PATCH 1/2] libavformat/hls.c: support in-stream ID3 metadata update.

2024-03-28 Thread Romain Beauxis

On Mon, Mar 25, 2024, 19:58 Romain Beauxis  wrote:

> This patch adds support for updating HLS metadata passed as ID3 frames.
>
> This seems like a pretty straight-forward improvement. Updating the
> metadaata of the first stream seems to be the mechanism is other places
> in the code and works as expected.
>


Hello!

Any interest in reviewing this?


---
>  libavformat/hls.c | 54 ---
>  1 file changed, 32 insertions(+), 22 deletions(-)
>
> diff --git a/libavformat/hls.c b/libavformat/hls.c
> index f6b44c2e35..ba6634d57a 100644
> --- a/libavformat/hls.c
> +++ b/libavformat/hls.c
> @@ -93,6 +93,12 @@ enum PlaylistType {
>  PLS_TYPE_VOD
>  };
>
> +#define ID3_PRIV_OWNER_TS "com.apple.streaming.transportStreamTimestamp"
> +#define ID3_PRIV_OWNER_AUDIO_SETUP "com.apple.streaming.audioDescription"
> +
> +#define ID3v2_PRIV_OWNER_TS ID3v2_PRIV_METADATA_PREFIX ID3_PRIV_OWNER_TS
> +#define ID3v2_PRIV_OWNER_AUDIO_SETUP ID3v2_PRIV_METADATA_PREFIX
> ID3_PRIV_OWNER_AUDIO_SETUP
> +
>  /*
>   * Each playlist has its own demuxer. If it currently is active,
>   * it has an open AVIOContext too, and potentially an AVPacket
> @@ -150,9 +156,7 @@ struct playlist {
>  int64_t id3_offset; /* in stream original tb */
>  uint8_t* id3_buf; /* temp buffer for id3 parsing */
>  unsigned int id3_buf_size;
> -AVDictionary *id3_initial; /* data from first id3 tag */
> -int id3_found; /* ID3 tag found at some point */
> -int id3_changed; /* ID3 tag data has changed at some point */
> +AVDictionary *last_id3; /* data from the last id3 tag */
>  ID3v2ExtraMeta *id3_deferred_extra; /* stored here until subdemuxer
> is opened */
>
>  HLSAudioSetupInfo audio_setup_info;
> @@ -270,7 +274,7 @@ static void free_playlist_list(HLSContext *c)
>  av_freep(>main_streams);
>  av_freep(>renditions);
>  av_freep(>id3_buf);
> -av_dict_free(>id3_initial);
> +av_dict_free(>last_id3);
>  ff_id3v2_free_extra_meta(>id3_deferred_extra);
>  av_freep(>init_sec_buf);
>  av_packet_free(>pkt);
> @@ -1083,15 +1087,13 @@ static void parse_id3(AVFormatContext *s,
> AVIOContext *pb,
>AVDictionary **metadata, int64_t *dts,
> HLSAudioSetupInfo *audio_setup_info,
>ID3v2ExtraMetaAPIC **apic, ID3v2ExtraMeta
> **extra_meta)
>  {
> -static const char id3_priv_owner_ts[] =
> "com.apple.streaming.transportStreamTimestamp";
> -static const char id3_priv_owner_audio_setup[] =
> "com.apple.streaming.audioDescription";
>  ID3v2ExtraMeta *meta;
>
>  ff_id3v2_read_dict(pb, metadata, ID3v2_DEFAULT_MAGIC, extra_meta);
>  for (meta = *extra_meta; meta; meta = meta->next) {
>  if (!strcmp(meta->tag, "PRIV")) {
>  ID3v2ExtraMetaPRIV *priv = >data.priv;
> -if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
> id3_priv_owner_ts, 44)) {
> +if (priv->datasize == 8 && !av_strncasecmp(priv->owner,
> ID3_PRIV_OWNER_TS, strlen(ID3_PRIV_OWNER_TS))) {
>  /* 33-bit MPEG timestamp */
>  int64_t ts = AV_RB64(priv->data);
>  av_log(s, AV_LOG_DEBUG, "HLS ID3 audio timestamp
> %"PRId64"\n", ts);
> @@ -1099,7 +1101,9 @@ static void parse_id3(AVFormatContext *s,
> AVIOContext *pb,
>  *dts = ts;
>  else
>  av_log(s, AV_LOG_ERROR, "Invalid HLS ID3 audio
> timestamp %"PRId64"\n", ts);
> -} else if (priv->datasize >= 8 &&
> !av_strncasecmp(priv->owner, id3_priv_owner_audio_setup, 36)) {
> +} else if (priv->datasize >= 8 &&
> +   !av_strncasecmp(priv->owner,
> ID3_PRIV_OWNER_AUDIO_SETUP, 36) &&
> +   audio_setup_info) {
>  ff_hls_senc_read_audio_setup_info(audio_setup_info,
> priv->data, priv->datasize);
>  }
>  } else if (!strcmp(meta->tag, "APIC") && apic)
> @@ -1113,9 +1117,10 @@ static int id3_has_changed_values(struct playlist
> *pls, AVDictionary *metadata,
>  {
>  const AVDictionaryEntry *entry = NULL;
>  const AVDictionaryEntry *oldentry;
> +
>  /* check that no keys have changed values */
>  while ((entry = av_dict_iterate(metadata, entry))) {
> -oldentry = av_dict_get(pls->id3_initial, entry->key, NULL,
> AV_DICT_MATCH_CASE);
> +oldentry = av_dict_get(pls->last_id3, entry-

[FFmpeg-devel] [PATCH 2/2] Add fate test for hls metadata update.

2024-03-25 Thread Romain Beauxis

This patch adds a FATE test for the new HLS metadata update. The fate
sample consists of a two segment AAC hls stream. The first segment has
test title 1 as title metadata while the second has test title 2.

In the log, we can see that test title 2 is reported for the stream,
indicating that the metadata was updated with the second segment.

Romain
---
 tests/fate/demux.mak   |   3 +
 tests/ref/fate/hls-adts-meta-demux | 124 +
 2 files changed, 127 insertions(+)
 create mode 100644 tests/ref/fate/hls-adts-meta-demux

diff --git a/tests/fate/demux.mak b/tests/fate/demux.mak
index d9b9045f0b..7243c2c163 100644
--- a/tests/fate/demux.mak
+++ b/tests/fate/demux.mak
@@ -160,6 +160,9 @@ fate-ts-demux: CMD = ffprobe_demux 
$(TARGET_SAMPLES)/ac3/mp3ac325-4864-small.ts
 FATE_FFPROBE_DEMUX-$(CONFIG_MPEGTS_DEMUXER) += fate-ts-timed-id3-demux
 fate-ts-timed-id3-demux: CMD = ffprobe_demux $(TARGET_SAMPLES)/mpegts/id3.ts
 
+FATE_FFPROBE_DEMUX-$(CONFIG_MPEGTS_DEMUXER) += fate-hls-adts-meta-demux
+fate-fate-hls-adts-meta-demux: CMD = ffprobe_demux 
$(TARGET_SAMPLES)/hls-adts-meta/stream.m3u8
+
 FATE_SAMPLES_DEMUX += $(FATE_SAMPLES_DEMUX-yes)
 FATE_SAMPLES_FFMPEG += $(FATE_SAMPLES_DEMUX)
 FATE_FFPROBE_DEMUX   += $(FATE_FFPROBE_DEMUX-yes)
diff --git a/tests/ref/fate/hls-adts-meta-demux 
b/tests/ref/fate/hls-adts-meta-demux
new file mode 100644
index 00..ab944695fc
--- /dev/null
+++ b/tests/ref/fate/hls-adts-meta-demux
@@ -0,0 +1,124 @@
+packet|codec_type=audio|stream_index=0|pts=3416400|pts_time=37.96|dts=3416400|dts_time=37.96|duration=2090|duration_time=0.023222|size=368|pos=0|flags=K__|data_hash=CRC32:c371b0d9
+packet|codec_type=audio|stream_index=0|pts=3418490|pts_time=37.983222|dts=3418490|dts_time=37.983222|duration=2090|duration_time=0.023222|size=390|pos=368|flags=K__|data_hash=CRC32:950c52b2
+packet|codec_type=audio|stream_index=0|pts=3420580|pts_time=38.006444|dts=3420580|dts_time=38.006444|duration=2090|duration_time=0.023222|size=357|pos=758|flags=K__|data_hash=CRC32:3e672212
+packet|codec_type=audio|stream_index=0|pts=3422669|pts_time=38.029656|dts=3422669|dts_time=38.029656|duration=2090|duration_time=0.023222|size=426|pos=1115|flags=K__|data_hash=CRC32:817b6e4c
+packet|codec_type=audio|stream_index=0|pts=3424759|pts_time=38.052878|dts=3424759|dts_time=38.052878|duration=2090|duration_time=0.023222|size=368|pos=1541|flags=K__|data_hash=CRC32:c4c6e1ed
+packet|codec_type=audio|stream_index=0|pts=3426849|pts_time=38.076100|dts=3426849|dts_time=38.076100|duration=2090|duration_time=0.023222|size=389|pos=1909|flags=K__|data_hash=CRC32:67cb6dd9
+packet|codec_type=audio|stream_index=0|pts=3428939|pts_time=38.099322|dts=3428939|dts_time=38.099322|duration=2090|duration_time=0.023222|size=352|pos=2298|flags=K__|data_hash=CRC32:7a56ff53
+packet|codec_type=audio|stream_index=0|pts=3431029|pts_time=38.122544|dts=3431029|dts_time=38.122544|duration=2090|duration_time=0.023222|size=378|pos=2650|flags=K__|data_hash=CRC32:f8d5ef58
+packet|codec_type=audio|stream_index=0|pts=3433118|pts_time=38.145756|dts=3433118|dts_time=38.145756|duration=2090|duration_time=0.023222|size=384|pos=3028|flags=K__|data_hash=CRC32:73a4fb1c
+packet|codec_type=audio|stream_index=0|pts=3435208|pts_time=38.168978|dts=3435208|dts_time=38.168978|duration=2090|duration_time=0.023222|size=353|pos=3412|flags=K__|data_hash=CRC32:4ea999b5
+packet|codec_type=audio|stream_index=0|pts=3437298|pts_time=38.192200|dts=3437298|dts_time=38.192200|duration=2090|duration_time=0.023222|size=417|pos=3765|flags=K__|data_hash=CRC32:4540aec8
+packet|codec_type=audio|stream_index=0|pts=3439388|pts_time=38.215422|dts=3439388|dts_time=38.215422|duration=2090|duration_time=0.023222|size=361|pos=4182|flags=K__|data_hash=CRC32:635a04f4
+packet|codec_type=audio|stream_index=0|pts=3441478|pts_time=38.238644|dts=3441478|dts_time=38.238644|duration=2090|duration_time=0.023222|size=399|pos=4543|flags=K__|data_hash=CRC32:94583c18
+packet|codec_type=audio|stream_index=0|pts=3443567|pts_time=38.261856|dts=3443567|dts_time=38.261856|duration=2090|duration_time=0.023222|size=384|pos=4942|flags=K__|data_hash=CRC32:21070d79
+packet|codec_type=audio|stream_index=0|pts=3445657|pts_time=38.285078|dts=3445657|dts_time=38.285078|duration=2090|duration_time=0.023222|size=378|pos=5326|flags=K__|data_hash=CRC32:bd5beb97
+packet|codec_type=audio|stream_index=0|pts=3447747|pts_time=38.308300|dts=3447747|dts_time=38.308300|duration=2090|duration_time=0.023222|size=362|pos=5704|flags=K__|data_hash=CRC32:8bb15fb7
+packet|codec_type=audio|stream_index=0|pts=3449837|pts_time=38.331522|dts=3449837|dts_time=38.331522|duration=2090|duration_time=0.023222|size=365|pos=6066|flags=K__|data_hash=CRC32:a1801ece
+packet|codec_type=audio|stream_index=0|pts=3451927|pts_time=38.354744|dts=3451927|dts_time=38.354744|duration=2090|duration_time=0.023222|size=390|pos=6431|flags=K__|data_hash=CRC32:8eb3880b

[FFmpeg-devel] [PATCH 1/2] libavformat/hls.c: support in-stream ID3 metadata update.

2024-03-25 Thread Romain Beauxis

This patch adds support for updating HLS metadata passed as ID3 frames.

This seems like a pretty straight-forward improvement. Updating the
metadaata of the first stream seems to be the mechanism is other places
in the code and works as expected.

---
 libavformat/hls.c | 54 ---
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/libavformat/hls.c b/libavformat/hls.c
index f6b44c2e35..ba6634d57a 100644
--- a/libavformat/hls.c
+++ b/libavformat/hls.c
@@ -93,6 +93,12 @@ enum PlaylistType {
 PLS_TYPE_VOD
 };
 
+#define ID3_PRIV_OWNER_TS "com.apple.streaming.transportStreamTimestamp"
+#define ID3_PRIV_OWNER_AUDIO_SETUP "com.apple.streaming.audioDescription"
+
+#define ID3v2_PRIV_OWNER_TS ID3v2_PRIV_METADATA_PREFIX ID3_PRIV_OWNER_TS
+#define ID3v2_PRIV_OWNER_AUDIO_SETUP ID3v2_PRIV_METADATA_PREFIX 
ID3_PRIV_OWNER_AUDIO_SETUP
+
 /*
  * Each playlist has its own demuxer. If it currently is active,
  * it has an open AVIOContext too, and potentially an AVPacket
@@ -150,9 +156,7 @@ struct playlist {
 int64_t id3_offset; /* in stream original tb */
 uint8_t* id3_buf; /* temp buffer for id3 parsing */
 unsigned int id3_buf_size;
-AVDictionary *id3_initial; /* data from first id3 tag */
-int id3_found; /* ID3 tag found at some point */
-int id3_changed; /* ID3 tag data has changed at some point */
+AVDictionary *last_id3; /* data from the last id3 tag */
 ID3v2ExtraMeta *id3_deferred_extra; /* stored here until subdemuxer is 
opened */
 
 HLSAudioSetupInfo audio_setup_info;
@@ -270,7 +274,7 @@ static void free_playlist_list(HLSContext *c)
 av_freep(>main_streams);
 av_freep(>renditions);
 av_freep(>id3_buf);
-av_dict_free(>id3_initial);
+av_dict_free(>last_id3);
 ff_id3v2_free_extra_meta(>id3_deferred_extra);
 av_freep(>init_sec_buf);
 av_packet_free(>pkt);
@@ -1083,15 +1087,13 @@ static void parse_id3(AVFormatContext *s, AVIOContext 
*pb,
   AVDictionary **metadata, int64_t *dts, HLSAudioSetupInfo 
*audio_setup_info,
   ID3v2ExtraMetaAPIC **apic, ID3v2ExtraMeta **extra_meta)
 {
-static const char id3_priv_owner_ts[] = 
"com.apple.streaming.transportStreamTimestamp";
-static const char id3_priv_owner_audio_setup[] = 
"com.apple.streaming.audioDescription";
 ID3v2ExtraMeta *meta;
 
 ff_id3v2_read_dict(pb, metadata, ID3v2_DEFAULT_MAGIC, extra_meta);
 for (meta = *extra_meta; meta; meta = meta->next) {
 if (!strcmp(meta->tag, "PRIV")) {
 ID3v2ExtraMetaPRIV *priv = >data.priv;
-if (priv->datasize == 8 && !av_strncasecmp(priv->owner, 
id3_priv_owner_ts, 44)) {
+if (priv->datasize == 8 && !av_strncasecmp(priv->owner, 
ID3_PRIV_OWNER_TS, strlen(ID3_PRIV_OWNER_TS))) {
 /* 33-bit MPEG timestamp */
 int64_t ts = AV_RB64(priv->data);
 av_log(s, AV_LOG_DEBUG, "HLS ID3 audio timestamp %"PRId64"\n", 
ts);
@@ -1099,7 +1101,9 @@ static void parse_id3(AVFormatContext *s, AVIOContext *pb,
 *dts = ts;
 else
 av_log(s, AV_LOG_ERROR, "Invalid HLS ID3 audio timestamp 
%"PRId64"\n", ts);
-} else if (priv->datasize >= 8 && !av_strncasecmp(priv->owner, 
id3_priv_owner_audio_setup, 36)) {
+} else if (priv->datasize >= 8 &&
+   !av_strncasecmp(priv->owner, 
ID3_PRIV_OWNER_AUDIO_SETUP, 36) &&
+   audio_setup_info) {
 ff_hls_senc_read_audio_setup_info(audio_setup_info, 
priv->data, priv->datasize);
 }
 } else if (!strcmp(meta->tag, "APIC") && apic)
@@ -1113,9 +1117,10 @@ static int id3_has_changed_values(struct playlist *pls, 
AVDictionary *metadata,
 {
 const AVDictionaryEntry *entry = NULL;
 const AVDictionaryEntry *oldentry;
+
 /* check that no keys have changed values */
 while ((entry = av_dict_iterate(metadata, entry))) {
-oldentry = av_dict_get(pls->id3_initial, entry->key, NULL, 
AV_DICT_MATCH_CASE);
+oldentry = av_dict_get(pls->last_id3, entry->key, NULL, 
AV_DICT_MATCH_CASE);
 if (!oldentry || strcmp(oldentry->value, entry->value) != 0)
 return 1;
 }
@@ -1143,35 +1148,40 @@ static void handle_id3(AVIOContext *pb, struct playlist 
*pls)
 ID3v2ExtraMetaAPIC *apic = NULL;
 ID3v2ExtraMeta *extra_meta = NULL;
 int64_t timestamp = AV_NOPTS_VALUE;
+// Only set audio_setup_info on first id3 chunk.
+HLSAudioSetupInfo *audio_setup_info = pls->last_id3 ? NULL : 
>audio_setup_info;
 
-parse_id3(pls->ctx, pb, , , >audio_setup_info, 
, _meta);
+parse_id3(pls->ctx, pb, , , audio_setup_info, , 
_meta);
 
-if (timestamp != AV_NOPTS_VALUE) {
+if (pls->id3_mpegts_timestamp == AV_NOPTS_VALUE && timestamp != 
AV_NOPTS_VALUE) {
 pls->id3_mpegts_timestamp = timestamp;
 pls->id3_offset = 0;

Re: [FFmpeg-devel] [PATCH] Set native order for wav channel layouts up until 8 channels.

2024-02-27 Thread Romain Beauxis

Le sam. 24 févr. 2024 à 19:27, Michael Niedermayer
 a écrit :
>
> On Fri, Feb 23, 2024 at 02:41:06PM -0600, Romain Beauxis wrote:
> > The new default channel layout for the various RIFF/WAV decoders is not
> > backward compatible.
> >
> > Historically, most decoders will expect the channel layouts to follow
> > the native layout up-to a reasonable number of channels.
> >
> > Additionally, non-native layouts are causing troubles with filters
> > chaining.
> >
> > This PR changes the default channel layout reported by RIFF/WAV decoders
> > to default to the native layout when the number of channels is up-to 8.
> >
> > The logic for these changes is the same as the logic for the vorbis/opus
> > decoders.
> >
> > Romain
>
> breaks fate
> make -j32 fate-flcl1905
> TESTflcl1905
> --- ./tests/ref/fate/flcl1905   2024-02-09 03:32:32.540199565 +0100
> +++ tests/data/fate/flcl19052024-02-25 02:26:51.079111678 +0100
> @@ -1,192 +1,192 @@
>  
> packet|codec_type=audio|stream_index=0|pts=0|pts_time=0.00|dts=0|dts_time=0.00|duration=22528|duration_time=0.510839|size=4092|pos=56|flags=K__
> -frame|media_type=audio|stream_index=0|key_frame=1|pts=N/A|pts_time=N/A|pkt_dts=N/A|pkt_dts_time=N/A|best_effort_timestamp=N/A|best_effort_timestamp_time=N/A|pkt_duration=22528|pkt_duration_time=0.510839|duration=22528|duration_time=0.510839|pkt_pos=56|pkt_size=4092|sample_fmt=fltp|nb_samples=2048|channels=2|channel_layout=unknown
> -frame|media_type=audio|stream_index=0|key_frame=1|pts=N/A|pts_time=N/A|pkt_dts=N/A|pkt_dts_time=N/A|best_effort_timestamp=N/A|best_effort_timestamp_time=N/A|pkt_duration=22528|pkt_duration_time=0.510839|duration=22528|duration_time=0.510839|pkt_pos=56|pkt_size=4092|sample_fmt=fltp|nb_samples=2048|channels=2|channel_layout=unknown
> -frame|media_type=audio|stream_index=0|key_frame=1|pts=N/A|pts_time=N/A|pkt_dts=N/A|pkt_dts_time=N/A|best_effort_timestamp=N/A|best_effort_timestamp_time=N/A|pkt_duration=22528|pkt_duration_time=0.510839|duration=22528|duration_time=0.510839|pkt_pos=56|pkt_size=4092|sample_fmt=fltp|nb_samples=2048|channels=2|channel_layout=unknown
> ...

Thanks.

I did more backward compatibility tests and I might have jumped the
gun on it as well.

I'll check again and will come back to it if I can confirm anything.

-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] Set native order for wav channel layouts up until 8 channels.

2024-02-23 Thread Romain Beauxis

Le ven. 23 févr. 2024 à 15:11, Marton Balint  a écrit :
>
>
>
> On Fri, 23 Feb 2024, Romain Beauxis wrote:
>
> > The new default channel layout for the various RIFF/WAV decoders is not
> > backward compatible.
> >
> > Historically, most decoders will expect the channel layouts to follow
> > the native layout up-to a reasonable number of channels.
> >
> > Additionally, non-native layouts are causing troubles with filters
> > chaining.
> >
> > This PR changes the default channel layout reported by RIFF/WAV decoders
> > to default to the native layout when the number of channels is up-to 8.
> >
> > The logic for these changes is the same as the logic for the vorbis/opus
> > decoders.
>
> For Vorbis the channel layout is in the actual Vorbis specification. So
> you should follow the specification, simple guessing in the demuxer likely
> won't be acceptable.

I would argue that even though there is no official specification on
channel layout for wav/riff, the de-facto assumption that _most_ users
of the library would expect is the native layout.

Typically, 1 and 2 channels would be assumed to be mono and stereo by
most users.

It's great that the API does provide flexibility but the default
should be set to satisfy most users and, in that regard, it seems that
assuming a native layout is what the vast majority of library's users
will expect.

This choice also implies at least two ABI breakage for applications
using the deprecated API but running on a the new ABI:

1: With the updated API, the library is not able to provide a backward
compatible `channel_layout` for AVFrame when the new channel order is
AV_CHANNEL_ORDER_UNSPEC which breaks ABI compatibility as the field is
reported as `0`.

2. AV_CHANNEL_ORDER_UNSPEC channel order also breaks filters chaining.
The issue appears to be that the unspec channel order implicitly sets
the the filter to accept all channel order, which breaks compatibility
here:

if (link->incfg.channel_layouts->all_layouts) {
av_log(link->src, AV_LOG_ERROR, "Cannot select channel layout for"
   " the link between filters %s and %s.\n", link->src->name,
   link->dst->name);
if (!link->incfg.channel_layouts->all_counts)
av_log(link->src, AV_LOG_ERROR, "Unknown channel layouts not "
   "supported, try specifying a channel layout using "
   "'aformat=channel_layouts=something'.\n");
return AVERROR(EINVAL);
}

-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH] Set native order for wav channel layouts up until 8 channels.

2024-02-23 Thread Romain Beauxis

The new default channel layout for the various RIFF/WAV decoders is not
backward compatible.

Historically, most decoders will expect the channel layouts to follow
the native layout up-to a reasonable number of channels.

Additionally, non-native layouts are causing troubles with filters
chaining.

This PR changes the default channel layout reported by RIFF/WAV decoders
to default to the native layout when the number of channels is up-to 8.

The logic for these changes is the same as the logic for the vorbis/opus
decoders.

Romain

---
 libavformat/riff.h|  1 +
 libavformat/riffdec.c | 31 ---
 libavformat/wavdec.c  |  4 +---
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/libavformat/riff.h b/libavformat/riff.h
index a93eadfeca..f474efdce9 100644
--- a/libavformat/riff.h
+++ b/libavformat/riff.h
@@ -67,6 +67,7 @@ void ff_put_bmp_header(AVIOContext *pb, AVCodecParameters 
*par, int for_asf, int
 int ff_put_wav_header(AVFormatContext *s, AVIOContext *pb, AVCodecParameters 
*par, int flags);
 
 enum AVCodecID ff_wav_codec_get_id(unsigned int tag, int bps);
+void ff_get_wav_ch_layout(AVChannelLayout *ch_layout, int channels);
 int ff_get_wav_header(void *logctx, AVIOContext *pb, AVCodecParameters *par,
   int size, int big_endian);
 
diff --git a/libavformat/riffdec.c b/libavformat/riffdec.c
index 0fe4e02b7b..bb7c01c5f5 100644
--- a/libavformat/riffdec.c
+++ b/libavformat/riffdec.c
@@ -90,6 +90,33 @@ static void parse_waveformatex(void *logctx, AVIOContext 
*pb, AVCodecParameters
 }
 }
 
+const AVChannelLayout ff_wav_demux_ch_layouts[9] = {
+AV_CHANNEL_LAYOUT_MONO,
+AV_CHANNEL_LAYOUT_STEREO,
+AV_CHANNEL_LAYOUT_SURROUND,
+AV_CHANNEL_LAYOUT_QUAD,
+AV_CHANNEL_LAYOUT_5POINT0_BACK,
+AV_CHANNEL_LAYOUT_5POINT1_BACK,
+{
+.nb_channels = 7,
+.order   = AV_CHANNEL_ORDER_NATIVE,
+.u.mask  = AV_CH_LAYOUT_5POINT1 | AV_CH_BACK_CENTER,
+},
+AV_CHANNEL_LAYOUT_7POINT1,
+{ 0 }
+};
+
+void ff_get_wav_ch_layout(AVChannelLayout *ch_layout, int channels)
+{
+av_channel_layout_uninit(ch_layout);
+if (channels > 8) {
+ch_layout->order   = AV_CHANNEL_ORDER_UNSPEC;
+ch_layout->nb_channels = channels;
+} else {
+av_channel_layout_copy(ch_layout, _wav_demux_ch_layouts[channels - 
1]);
+}
+}
+
 /* "big_endian" values are needed for RIFX file format */
 int ff_get_wav_header(void *logctx, AVIOContext *pb,
   AVCodecParameters *par, int size, int big_endian)
@@ -195,9 +222,7 @@ int ff_get_wav_header(void *logctx, AVIOContext *pb,
 
 /* ignore WAVEFORMATEXTENSIBLE layout if different from channel count */
 if (channels != par->ch_layout.nb_channels) {
-av_channel_layout_uninit(>ch_layout);
-par->ch_layout.order   = AV_CHANNEL_ORDER_UNSPEC;
-par->ch_layout.nb_channels = channels;
+ff_get_wav_ch_layout(>ch_layout, channels);
 }
 
 return 0;
diff --git a/libavformat/wavdec.c b/libavformat/wavdec.c
index 0c6629b157..282e1ae017 100644
--- a/libavformat/wavdec.c
+++ b/libavformat/wavdec.c
@@ -222,9 +222,7 @@ static int wav_parse_xma2_tag(AVFormatContext *s, int64_t 
size, AVStream *st)
 channels += avio_r8(pb);
 avio_skip(pb, 3);
 }
-av_channel_layout_uninit(>codecpar->ch_layout);
-st->codecpar->ch_layout.order   = AV_CHANNEL_ORDER_UNSPEC;
-st->codecpar->ch_layout.nb_channels = channels;
+ff_get_wav_ch_layout(>codecpar->ch_layout, channels);
 
 if (st->codecpar->ch_layout.nb_channels <= 0 || st->codecpar->sample_rate 
<= 0)
 return AVERROR_INVALIDDATA;
-- 
2.39.3 (Apple Git-145)

___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] FFmpeg 6.1.1

2024-01-05 Thread Romain Beauxis

Le jeu. 21 déc. 2023 à 13:17, Michael Niedermayer
 a écrit :
>
> Hi all
>
> I will probably make a 6.1.1 release in maybe 1-3 weeks due to bug/fixes
> in it.
> if you want something in it, please backport it now!

https://ffmpeg.org/pipermail/ffmpeg-devel/2024-January/319173.html
could be a good candidate.

> thx
>
> --
> Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
>
> If you drop bombs on a foreign country and kill a hundred thousand
> innocent people, expect your government to call the consequence
> "unprovoked inhuman terrorist attacks" and use it to justify dropping
> more bombs and killing more people. The technology changed, the idea is old.
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] libavformat/hlsenc.c: Populate OTI using AAC profile in write_codec_attr.

2024-01-01 Thread Romain Beauxis

Le sam. 30 déc. 2023 à 16:25, David Johansen  a écrit :
>
> On Sat, Dec 30, 2023 at 8:23 AM Romain Beauxis  wrote:
>>
>> Le jeu. 28 déc. 2023 à 17:26, David Johansen  a 
>> écrit :
>> >>
>> >> I love this change, but it appears that st->codecpar->profile is always 
>> >> AV_PROFILE_UNKNOWN when using libfdk_aac as the encoder. Any indications 
>> >> where I should look for fix that so this can be used with that encoder?
>> >
>> >
>> > It appears that the issue is that profile doesn't default to what's being 
>> > used so `--profile:a` has to be set explicitly with libfdk_aac and then it 
>> > works. Not sure if that's an issue worth fixing, but if someone points me 
>> > to where it needs to be done, then I'd be glad to take a look at fixing it
>>
>> This feels like a second, separate issue to me?
>>
>> Maybe we could get these changes in first and then tackle it?
>
>
> But this is technically a breaking change, because it takes commands/uses 
> that currently work and changes them to no longer include CODECS since the 
> profile value is unknown by default

Ha gotcha.

Yes, in this case we do need to send the previous value as fallback
when the profile is unknown.

Just sent an updated patch!

-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v2] libavformat/hlsenc.c: Populate OTI using AAC profile in write_codec_attr.

2024-01-01 Thread Romain Beauxis

This patch populates the third entry for HLS codec attribute using the
AAC profile.

The HLS specifications[1] require this value to be the Object Type ID as
referred to in table 1.3 of ISO/IEC 14496-3:2009[2].

The numerical constants in the code refer to these OTIs minus one, as
documented in commit 372597e[3], confirmed by comparing the values in the
code with the values in the table mentioned above.

Links:
1: https://datatracker.ietf.org/doc/html/rfc6381#section-3.3
2: https://csclub.uwaterloo.ca/~ehashman/ISO14496-3-2009.pdf
3: 
https://github.com/FFmpeg/FFmpeg/commit/372597e5381c097455a7b73849254d56083eb056

Changes in this version:
- Default value set to "mp4a.40.2" when profile is unknown for backward
  compatibility.

---
 libavformat/hlsenc.c | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
index 7049956dd7..55123d2297 100644
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@@ -418,8 +418,11 @@ static void write_codec_attr(AVStream *st, VariantStream 
*vs)
 } else if (st->codecpar->codec_id == AV_CODEC_ID_MP3) {
 snprintf(attr, sizeof(attr), "mp4a.40.34");
 } else if (st->codecpar->codec_id == AV_CODEC_ID_AAC) {
-/* TODO : For HE-AAC, HE-AACv2, the last digit needs to be set to 5 
and 29 respectively */
-snprintf(attr, sizeof(attr), "mp4a.40.2");
+if (st->codecpar->profile != AV_PROFILE_UNKNOWN)
+snprintf(attr, sizeof(attr), "mp4a.40.%d", 
st->codecpar->profile+1);
+else
+// This is for backward compatibility with the previous 
implementation.
+snprintf(attr, sizeof(attr), "mp4a.40.2");
 } else if (st->codecpar->codec_id == AV_CODEC_ID_AC3) {
 snprintf(attr, sizeof(attr), "ac-3");
 } else if (st->codecpar->codec_id == AV_CODEC_ID_EAC3) {
-- 
2.39.3 (Apple Git-145)

___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] libavformat/hlsenc.c: Populate OTI using AAC profile in write_codec_attr.

2023-12-30 Thread Romain Beauxis

Le jeu. 28 déc. 2023 à 17:26, David Johansen  a écrit :
>>
>> I love this change, but it appears that st->codecpar->profile is always 
>> AV_PROFILE_UNKNOWN when using libfdk_aac as the encoder. Any indications 
>> where I should look for fix that so this can be used with that encoder?
>
>
> It appears that the issue is that profile doesn't default to what's being 
> used so `--profile:a` has to be set explicitly with libfdk_aac and then it 
> works. Not sure if that's an issue worth fixing, but if someone points me to 
> where it needs to be done, then I'd be glad to take a look at fixing it

This feels like a second, separate issue to me?

Maybe we could get these changes in first and then tackle it?

-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] libavformat/hlsenc.c: Populate OTI using AAC profile in write_codec_attr.

2023-12-28 Thread Romain Beauxis

Hey there!

Le ven. 22 déc. 2023 à 09:09, Romain Beauxis  a écrit :
>
> This patch populates the third entry for HLS codec attribute using the
> AAC profile.
>
> The HLS specifications[1] require this digit to be the Object Type ID as
> referred to in table 1.3 of ISO/IEC 14496-3:2009[2].
>
> The numerical constants in the code refer to these OTIs minus one, as
> documented in commit 372597e[3], confirmed by comparing the values in the
> code with the values in the table mentioned above.
>
> Links:
> 1: https://datatracker.ietf.org/doc/html/rfc6381#section-3.3
> 2: https://csclub.uwaterloo.ca/~ehashman/ISO14496-3-2009.pdf
> 3: 
> https://github.com/FFmpeg/FFmpeg/commit/372597e5381c097455a7b73849254d56083eb056

Anyone interested? I think that this is a pretty straight-forward
change that could potentially qualify as a bugfix for 6.1.1, after
all, this generates incorrect HLS playlist descriptions..

-- Romain

> ---
>  libavformat/hlsenc.c | 6 --
>  1 file changed, 4 insertions(+), 2 deletions(-)
>
> diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
> index 7049956dd7..2551bac6ae 100644
> --- a/libavformat/hlsenc.c
> +++ b/libavformat/hlsenc.c
> @@ -418,8 +418,10 @@ static void write_codec_attr(AVStream *st, VariantStream 
> *vs)
>  } else if (st->codecpar->codec_id == AV_CODEC_ID_MP3) {
>  snprintf(attr, sizeof(attr), "mp4a.40.34");
>  } else if (st->codecpar->codec_id == AV_CODEC_ID_AAC) {
> -/* TODO : For HE-AAC, HE-AACv2, the last digit needs to be set to 5 
> and 29 respectively */
> -snprintf(attr, sizeof(attr), "mp4a.40.2");
> +if (st->codecpar->profile != AV_PROFILE_UNKNOWN)
> +snprintf(attr, sizeof(attr), "mp4a.40.%d", 
> st->codecpar->profile+1);
> +else
> +goto fail;
>  } else if (st->codecpar->codec_id == AV_CODEC_ID_AC3) {
>  snprintf(attr, sizeof(attr), "ac-3");
>  } else if (st->codecpar->codec_id == AV_CODEC_ID_EAC3) {
> --
> 2.39.3 (Apple Git-145)
>
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH] libavformat/hlsenc.c: Populate OTI using AAC profile in write_codec_attr.

2023-12-22 Thread Romain Beauxis

This patch populates the third entry for HLS codec attribute using the
AAC profile.

The HLS specifications[1] require this digit to be the Object Type ID as
referred to in table 1.3 of ISO/IEC 14496-3:2009[2].

The numerical constants in the code refer to these OTIs minus one, as
documeted in commit 372597e[3], confirmed by comparing the values in the
code with the values in the table mentioned above.

Links:
1: https://datatracker.ietf.org/doc/html/rfc6381#section-3.3
2: https://csclub.uwaterloo.ca/~ehashman/ISO14496-3-2009.pdf
3: 
https://github.com/FFmpeg/FFmpeg/commit/372597e5381c097455a7b73849254d56083eb056

---
 libavformat/hlsenc.c | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
index 7049956dd7..2551bac6ae 100644
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@@ -418,8 +418,10 @@ static void write_codec_attr(AVStream *st, VariantStream 
*vs)
 } else if (st->codecpar->codec_id == AV_CODEC_ID_MP3) {
 snprintf(attr, sizeof(attr), "mp4a.40.34");
 } else if (st->codecpar->codec_id == AV_CODEC_ID_AAC) {
-/* TODO : For HE-AAC, HE-AACv2, the last digit needs to be set to 5 
and 29 respectively */
-snprintf(attr, sizeof(attr), "mp4a.40.2");
+if (st->codecpar->profile != AV_PROFILE_UNKNOWN)
+snprintf(attr, sizeof(attr), "mp4a.40.%d", 
st->codecpar->profile+1);
+else
+goto fail;
 } else if (st->codecpar->codec_id == AV_CODEC_ID_AC3) {
 snprintf(attr, sizeof(attr), "ac-3");
 } else if (st->codecpar->codec_id == AV_CODEC_ID_EAC3) {
-- 
2.39.3 (Apple Git-145)

___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v2 2/6] libavformat/sdp: remove whitespaces in fmtp

2023-11-14 Thread Romain Beauxis

Le mar. 14 nov. 2023 à 09:47, Tomas Härdin  a écrit :
>
> tis 2023-11-07 klockan 15:12 +0100 skrev Michael Riedl:
> > Whitespaces after semicolon breaks some servers
>
> Which servers? If the spec allows whitespace then the onus is on them
> to fix their implementations.

The logic could be inverted: if the specs allow for both but a
majority of users do not accept white space, it would make sense to
change the implementation to maximize compatibility.

> /Tomas
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v2] Extract av_hls_codec_attr

2023-11-03 Thread Romain Beauxis

Le ven. 3 nov. 2023 à 03:00, Andreas Rheinhardt
 a écrit :
>
> Romain Beauxis:
> > The logic for extracting HLS codec attribute strings is very useful and
> > can be re-used in many different situations when working with HLS
> > streams using libavcodec/libavformat.
> >
> > This patch extracts the function's code and places it into a publicly
> > available function.
> >
> > Differences since v1:
> > - ff_nal_unit_extract_rbsp renamed into avpriv_nal_unit_extract_rbsp to
> >   follow the appropriate library cross-linking convention.
> >
> > ---
> >  libavcodec/Makefile  |   2 +
> >  libavcodec/hls.c | 105 +++
> >  libavcodec/hls.h |  42 +
> >  libavformat/avc.c|   4 +-
> >  libavformat/avc.h|   2 +-
> >  libavformat/hevc.c   |   2 +-
> >  libavformat/hlsenc.c |  83 +++---
> >  7 files changed, 159 insertions(+), 81 deletions(-)
> >  create mode 100644 libavcodec/hls.c
> >  create mode 100644 libavcodec/hls.h
> >
> > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> > index 580a8d6b54..b3b2b18980 100644
> > --- a/libavcodec/Makefile
> > +++ b/libavcodec/Makefile
> > @@ -16,6 +16,7 @@ HEADERS = ac3_parser.h
> >   \
> >dirac.h   \
> >dv_profile.h  \
> >dxva2.h   \
> > +  hls.h \
> >jni.h \
> >mediacodec.h  \
> >packet.h  \
> > @@ -47,6 +48,7 @@ OBJS = ac3_parser.o   
> >   \
> > get_buffer.o \
> > imgconvert.o \
> > jni.o\
> > +   hls.o\
> > mathtables.o \
> > mediacodec.o \
> > mpeg12framerate.o\
> > diff --git a/libavcodec/hls.c b/libavcodec/hls.c
> > new file mode 100644
> > index 00..05a6277dbc
> > --- /dev/null
> > +++ b/libavcodec/hls.c
> > @@ -0,0 +1,105 @@
> > +/*
> > + * HLS public API
> > + *
> > + * This file is part of FFmpeg.
> > + *
> > + * FFmpeg is free software; you can redistribute it and/or
> > + * modify it under the terms of the GNU Lesser General Public
> > + * License as published by the Free Software Foundation; either
> > + * version 2.1 of the License, or (at your option) any later version.
> > + *
> > + * FFmpeg is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> > + * Lesser General Public License for more details.
> > + *
> > + * You should have received a copy of the GNU Lesser General Public
> > + * License along with FFmpeg; if not, write to the Free Software
> > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 
> > 02110-1301 USA
> > + */
> > +
> > +#include "hls.h"
> > +#include "libavutil/intreadwrite.h"
> > +#include "libavformat/avc.h"
> > +
> > +int av_hls_codec_attr(AVStream *st, char *attr, size_t attr_len) {
> > +  if (st->codecpar->codec_id == AV_CODEC_ID_H264) {
> > +  uint8_t *data = st->codecpar->extradata;
> > +  if (data) {
> > +  const uint8_t *p;
> > +
> > +  if (AV_RB32(data) == 0x01 && (data[4] & 0x1F) == 7)
> > +  p = [5];
> > +  else if (AV_RB24(data) == 0x01 && (data[3] & 0x1F) == 7)
> > +  p = [4];
> > +  else if (data[0] == 0x01)  /* avcC */
> > +  p = [1];
> > +  else
> > +  return AVERROR_INVALIDDATA;
> > +  snprintf(attr, attr_len,
> > +   "avc1.%02x%02x%02x", p[0], p[1], p[2]);
> &

[FFmpeg-devel] [PATCH v3] Extract av_hls_codec_attr

2023-11-03 Thread Romain Beauxis

The logic for extracting HLS codec attribute strings is very useful and
can be re-used in many different situations when working with HLS
streams using libavcodec/libavformat.

This patch extracts the function's code and places it into a publicly
available function.

Differences since v2:
- Shared function moved to libavformat, ff_nal_unit_extract_rbsp
  function kept identical.

---
 libavformat/Makefile|   2 +
 libavformat/hls_utils.c | 105 
 libavformat/hls_utils.h |  42 
 libavformat/hlsenc.c|  83 +++
 4 files changed, 155 insertions(+), 77 deletions(-)
 create mode 100644 libavformat/hls_utils.c
 create mode 100644 libavformat/hls_utils.h

diff --git a/libavformat/Makefile b/libavformat/Makefile
index 329055ccfd..5e148e5bbc 100644
--- a/libavformat/Makefile
+++ b/libavformat/Makefile
@@ -3,6 +3,7 @@ DESC = FFmpeg container format library
 
 HEADERS = avformat.h\
   avio.h\
+  hls_utils.h   \
   version.h \
   version_major.h   \
 
@@ -15,6 +16,7 @@ OBJS = allformats.o \
dump.o   \
dv.o \
format.o \
+   hls_utils.o  \
id3v1.o  \
id3v2.o  \
isom_tags.o  \
diff --git a/libavformat/hls_utils.c b/libavformat/hls_utils.c
new file mode 100644
index 00..df45fbc1a4
--- /dev/null
+++ b/libavformat/hls_utils.c
@@ -0,0 +1,105 @@
+/*
+ * HLS public API
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "hls_utils.h"
+#include "libavutil/intreadwrite.h"
+#include "avc.h"
+
+int av_hls_codec_attr(AVStream *st, char *attr, size_t attr_len) {
+  if (st->codecpar->codec_id == AV_CODEC_ID_H264) {
+  uint8_t *data = st->codecpar->extradata;
+  if (data) {
+  const uint8_t *p;
+
+  if (AV_RB32(data) == 0x01 && (data[4] & 0x1F) == 7)
+  p = [5];
+  else if (AV_RB24(data) == 0x01 && (data[3] & 0x1F) == 7)
+  p = [4];
+  else if (data[0] == 0x01)  /* avcC */
+  p = [1];
+  else
+  return AVERROR_INVALIDDATA;
+  snprintf(attr, attr_len,
+   "avc1.%02x%02x%02x", p[0], p[1], p[2]);
+  } else {
+  return AVERROR_INVALIDDATA;
+  }
+  } else if (st->codecpar->codec_id == AV_CODEC_ID_HEVC) {
+  uint8_t *data = st->codecpar->extradata;
+  int profile = AV_PROFILE_UNKNOWN;
+  int level = AV_LEVEL_UNKNOWN;
+
+  if (st->codecpar->profile != AV_PROFILE_UNKNOWN)
+  profile = st->codecpar->profile;
+  if (st->codecpar->level != AV_LEVEL_UNKNOWN)
+  level = st->codecpar->level;
+
+  /* check the boundary of data which from current position is small than 
extradata_size */
+  while (data && (data - st->codecpar->extradata + 19) < 
st->codecpar->extradata_size) {
+  /* get HEVC SPS NAL and seek to profile_tier_level */
+  if (!(data[0] | data[1] | data[2]) && data[3] == 1 && ((data[4] & 
0x7E) == 0x42)) {
+  uint8_t *rbsp_buf;
+  int remain_size = 0;
+  int rbsp_size = 0;
+  /* skip start code + nalu header */
+  data += 6;
+  /* process by reference General NAL unit syntax */
+  remain_size = st->codecpar->extradata_size - (data - 
st->codecpar->extradata);
+  rbsp_buf = ff_nal_unit_extract_rbsp(data, remain_size, 
_size, 0);
+  if (!rbsp_buf)
+  return AVERROR_INVALIDDATA;
+  if (rbsp_size < 13) {
+  av_freep(_buf);
+  break;
+  }
+  /* skip sps_video_parameter_set_id   u(4),
+   *  sps_max_sub_layers_minus1u(3),
+   *  and sps_temporal_id_nesting_flag u(1) */
+  profile = rbsp_buf[1] & 0x1f;
+  /* skip 8 + 8 + 32 + 4 + 43 + 1 bit */
+  level =

Re: [FFmpeg-devel] [PATCH] Extract av_hls_codec_attr

2023-11-02 Thread Romain Beauxis

Le mar. 31 oct. 2023 à 11:47, Zhao Zhili  a écrit :
>
>
> > From: ffmpeg-devel  On Behalf Of Romain 
> > Beauxis
> > Sent: 2023年10月30日 9:06
> > To: ffmpeg-devel@ffmpeg.org
> > Cc: Romain Beauxis 
> > Subject: [FFmpeg-devel] [PATCH] Extract av_hls_codec_attr
> >
> > The logic for extracting HLS codec attribute strings is very useful and
> > can be re-used in many different situations when working with HLS
> > streams using libavcodec/libavformat.
> >
> > This patch extracts the function's code and places it into a publicly
> > available function.
>
> I don't think the implementation is complete enough to be exported as
> an API. And the ad-hoc API needs some design too.

I am not in a position to dispute this assessment but I would say it
could be a "build it and they will come" kind of situation (or chicken
and egg too).

> >
> > ---
> >  libavcodec/Makefile  |   2 +
> >  libavcodec/hls.c | 105 +++
> >  libavcodec/hls.h |  42 +
> >  libavformat/hlsenc.c |  83 +++---
> >  4 files changed, 155 insertions(+), 77 deletions(-)
> >  create mode 100644 libavcodec/hls.c
> >  create mode 100644 libavcodec/hls.h
> >
>
>
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] Extract av_hls_codec_attr

2023-11-02 Thread Romain Beauxis

Le mar. 31 oct. 2023 à 08:28, Michael Niedermayer
 a écrit :
>
> On Sun, Oct 29, 2023 at 08:05:50PM -0500, Romain Beauxis wrote:
> > The logic for extracting HLS codec attribute strings is very useful and
> > can be re-used in many different situations when working with HLS
> > streams using libavcodec/libavformat.
> >
> > This patch extracts the function's code and places it into a publicly
> > available function.
> >
> > ---
> >  libavcodec/Makefile  |   2 +
> >  libavcodec/hls.c | 105 +++
> >  libavcodec/hls.h |  42 +
> >  libavformat/hlsenc.c |  83 +++---
>
> you cannot call ff_* functions across libs
>
> they need to start with av* / avpriv*

Thanks for the review! Updated patch submitted.

> [...]
> > +  rbsp_buf = ff_nal_unit_extract_rbsp(data, remain_size, 
> > _size, 0);
>
> libavcodec/libavcodec.so: undefined reference to `ff_nal_unit_extract_rbsp'
> clang: error: linker command failed with exit code 1 (use -v to see 
> invocation)
> Makefile:133: recipe for target 'ffmpeg_g' failed
> make: *** [ffmpeg_g] Error 1
> make: *** Waiting for unfinished jobs
> libavcodec/libavcodec.so: undefined reference to `ff_nal_unit_extract_rbsp'
> clang: error: linker command failed with exit code 1 (use -v to see 
> invocation)
> Makefile:133: recipe for target 'ffplay_g' failed
> make: *** [ffplay_g] Error 1
>
> thx
>
> [...]
> --
> Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
>
> If a bugfix only changes things apparently unrelated to the bug with no
> further explanation, that is a good sign that the bugfix is wrong.
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v2] Extract av_hls_codec_attr

2023-11-02 Thread Romain Beauxis

The logic for extracting HLS codec attribute strings is very useful and
can be re-used in many different situations when working with HLS
streams using libavcodec/libavformat.

This patch extracts the function's code and places it into a publicly
available function.

Differences since v1:
- ff_nal_unit_extract_rbsp renamed into avpriv_nal_unit_extract_rbsp to
  follow the appropriate library cross-linking convention.

---
 libavcodec/Makefile  |   2 +
 libavcodec/hls.c | 105 +++
 libavcodec/hls.h |  42 +
 libavformat/avc.c|   4 +-
 libavformat/avc.h|   2 +-
 libavformat/hevc.c   |   2 +-
 libavformat/hlsenc.c |  83 +++---
 7 files changed, 159 insertions(+), 81 deletions(-)
 create mode 100644 libavcodec/hls.c
 create mode 100644 libavcodec/hls.h

diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 580a8d6b54..b3b2b18980 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -16,6 +16,7 @@ HEADERS = ac3_parser.h
  \
   dirac.h   \
   dv_profile.h  \
   dxva2.h   \
+  hls.h \
   jni.h \
   mediacodec.h  \
   packet.h  \
@@ -47,6 +48,7 @@ OBJS = ac3_parser.o   
  \
get_buffer.o \
imgconvert.o \
jni.o\
+   hls.o\
mathtables.o \
mediacodec.o \
mpeg12framerate.o\
diff --git a/libavcodec/hls.c b/libavcodec/hls.c
new file mode 100644
index 00..05a6277dbc
--- /dev/null
+++ b/libavcodec/hls.c
@@ -0,0 +1,105 @@
+/*
+ * HLS public API
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "hls.h"
+#include "libavutil/intreadwrite.h"
+#include "libavformat/avc.h"
+
+int av_hls_codec_attr(AVStream *st, char *attr, size_t attr_len) {
+  if (st->codecpar->codec_id == AV_CODEC_ID_H264) {
+  uint8_t *data = st->codecpar->extradata;
+  if (data) {
+  const uint8_t *p;
+
+  if (AV_RB32(data) == 0x01 && (data[4] & 0x1F) == 7)
+  p = [5];
+  else if (AV_RB24(data) == 0x01 && (data[3] & 0x1F) == 7)
+  p = [4];
+  else if (data[0] == 0x01)  /* avcC */
+  p = [1];
+  else
+  return AVERROR_INVALIDDATA;
+  snprintf(attr, attr_len,
+   "avc1.%02x%02x%02x", p[0], p[1], p[2]);
+  } else {
+  return AVERROR_INVALIDDATA;
+  }
+  } else if (st->codecpar->codec_id == AV_CODEC_ID_HEVC) {
+  uint8_t *data = st->codecpar->extradata;
+  int profile = AV_PROFILE_UNKNOWN;
+  int level = AV_LEVEL_UNKNOWN;
+
+  if (st->codecpar->profile != AV_PROFILE_UNKNOWN)
+  profile = st->codecpar->profile;
+  if (st->codecpar->level != AV_LEVEL_UNKNOWN)
+  level = st->codecpar->level;
+
+  /* check the boundary of data which from current position is small than 
extradata_size */
+  while (data && (data - st->codecpar->extradata + 19) < 
st->codecpar->extradata_size) {
+  /* get HEVC SPS NAL and seek to profile_tier_level */
+  if (!(data[0] | data[1] | data[2]) && data[3] == 1 && ((data[4] & 
0x7E) == 0x42)) {
+  uint8_t *rbsp_buf;
+  int remain_size = 0;
+  int rbsp_size = 0;
+  /* skip start code + nalu header */
+  data += 6;
+  /* process by reference General NAL unit syntax */
+  remain_size =

[FFmpeg-devel] [PATCH] Extract av_hls_codec_attr

2023-10-29 Thread Romain Beauxis

The logic for extracting HLS codec attribute strings is very useful and
can be re-used in many different situations when working with HLS
streams using libavcodec/libavformat.

This patch extracts the function's code and places it into a publicly
available function.

---
 libavcodec/Makefile  |   2 +
 libavcodec/hls.c | 105 +++
 libavcodec/hls.h |  42 +
 libavformat/hlsenc.c |  83 +++---
 4 files changed, 155 insertions(+), 77 deletions(-)
 create mode 100644 libavcodec/hls.c
 create mode 100644 libavcodec/hls.h

diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 580a8d6b54..b3b2b18980 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -16,6 +16,7 @@ HEADERS = ac3_parser.h
  \
   dirac.h   \
   dv_profile.h  \
   dxva2.h   \
+  hls.h \
   jni.h \
   mediacodec.h  \
   packet.h  \
@@ -47,6 +48,7 @@ OBJS = ac3_parser.o   
  \
get_buffer.o \
imgconvert.o \
jni.o\
+   hls.o\
mathtables.o \
mediacodec.o \
mpeg12framerate.o\
diff --git a/libavcodec/hls.c b/libavcodec/hls.c
new file mode 100644
index 00..59ab0c4819
--- /dev/null
+++ b/libavcodec/hls.c
@@ -0,0 +1,105 @@
+/*
+ * HLS public API
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "hls.h"
+#include "libavutil/intreadwrite.h"
+#include "libavformat/avc.h"
+
+int av_hls_codec_attr(AVStream *st, char *attr, size_t attr_len) {
+  if (st->codecpar->codec_id == AV_CODEC_ID_H264) {
+  uint8_t *data = st->codecpar->extradata;
+  if (data) {
+  const uint8_t *p;
+
+  if (AV_RB32(data) == 0x01 && (data[4] & 0x1F) == 7)
+  p = [5];
+  else if (AV_RB24(data) == 0x01 && (data[3] & 0x1F) == 7)
+  p = [4];
+  else if (data[0] == 0x01)  /* avcC */
+  p = [1];
+  else
+  return AVERROR_INVALIDDATA;
+  snprintf(attr, attr_len,
+   "avc1.%02x%02x%02x", p[0], p[1], p[2]);
+  } else {
+  return AVERROR_INVALIDDATA;
+  }
+  } else if (st->codecpar->codec_id == AV_CODEC_ID_HEVC) {
+  uint8_t *data = st->codecpar->extradata;
+  int profile = AV_PROFILE_UNKNOWN;
+  int level = AV_LEVEL_UNKNOWN;
+
+  if (st->codecpar->profile != AV_PROFILE_UNKNOWN)
+  profile = st->codecpar->profile;
+  if (st->codecpar->level != AV_LEVEL_UNKNOWN)
+  level = st->codecpar->level;
+
+  /* check the boundary of data which from current position is small than 
extradata_size */
+  while (data && (data - st->codecpar->extradata + 19) < 
st->codecpar->extradata_size) {
+  /* get HEVC SPS NAL and seek to profile_tier_level */
+  if (!(data[0] | data[1] | data[2]) && data[3] == 1 && ((data[4] & 
0x7E) == 0x42)) {
+  uint8_t *rbsp_buf;
+  int remain_size = 0;
+  int rbsp_size = 0;
+  /* skip start code + nalu header */
+  data += 6;
+  /* process by reference General NAL unit syntax */
+  remain_size = st->codecpar->extradata_size - (data - 
st->codecpar->extradata);
+  rbsp_buf = ff_nal_unit_extract_rbsp(data, remain_size, 
_size, 0);
+  if (!rbsp_buf)
+  return AVERROR_INVALIDDATA;
+  if (rbsp_size < 13)

Re: [FFmpeg-devel] [PATCH] libavformat/mpegts.c: fix hardcoded 5-bytes skip for metadata streams.

2023-06-26 Thread Romain Beauxis

Le sam. 24 juin 2023 à 05:51, Thilo Borgmann  a
écrit :
>
> Am 24.06.23 um 12:43 schrieb Anton Khirnov:
> > Quoting Romain Beauxis (2023-06-22 16:19:36)
> >> commit ca0472eeebe478b7eb6e7d1dc4351037f8811728
> >> Author: Romain Beauxis 
> >> Date:   Thu Jun 22 09:14:18 2023 -0500
> >>
> >>  Add FATE test for timed id3 demux.
> >>
> >> diff --git a/tests/fate/demux.mak b/tests/fate/demux.mak
> >> index d8fc68af88..ace8fa0b52 100644
> >> --- a/tests/fate/demux.mak
> >> +++ b/tests/fate/demux.mak
> >> @@ -157,6 +157,9 @@ fate-xwma-demux: CMD = crc -i
$(TARGET_SAMPLES)/xwma/ergon.xwma -c:a copy
> >>   FATE_FFPROBE_DEMUX-$(CONFIG_MPEGTS_DEMUXER) += fate-ts-demux
> >>   fate-ts-demux: CMD = ffprobe_demux
$(TARGET_SAMPLES)/ac3/mp3ac325-4864-small.ts
> >>
> >> +FATE_FFPROBE_DEMUX-$(CONFIG_MPEGTS_DEMUXER) += fate-ts-timed-id3-demux
> >> +fate-ts-timed-id3-demux: CMD = ffprobe_demux
$(TARGET_SAMPLES)/mpegts/id3.ts
> >> +
> >>   FATE_SAMPLES_DEMUX += $(FATE_SAMPLES_DEMUX-yes)
> >>   FATE_SAMPLES_FFMPEG += $(FATE_SAMPLES_DEMUX)
> >>   FATE_FFPROBE_DEMUX   += $(FATE_FFPROBE_DEMUX-yes)
> >> diff --git a/tests/ref/fate/ts-timed-id3-demux
b/tests/ref/fate/ts-timed-id3-demux
> >> new file mode 100644
> >> index 00..922ca17d15
> >> --- /dev/null
> >> +++ b/tests/ref/fate/ts-timed-id3-demux
> >> @@ -0,0 +1,6 @@
> >>
+packet|codec_type=data|stream_index=0|pts=126000|pts_time=1.40|dts=126000|dts_time=1.40|duration=N/A|duration_time=N/A|size=26|pos=564|flags=K__|data_hash=CRC32:469f474b|side_data|side_data_type=MPEGTS
Stream ID|id=189
> >> +
> >>
+packet|codec_type=data|stream_index=0|pts=577350|pts_time=6.415000|dts=577350|dts_time=6.415000|duration=N/A|duration_time=N/A|size=26|pos=1316|flags=K__|data_hash=CRC32:469f474b|side_data|side_data_type=MPEGTS
Stream ID|id=189
> >> +
> >>
+stream|index=0|codec_name=timed_id3|profile=unknown|codec_type=data|codec_tag_string=ID3
|codec_tag=0x20334449|ts_packetsize=188|id=0x100|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/9|start_pts=126000|start_time=1.40|duration_ts=451350|duration=5.015000|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=2|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0
> >>
+format|filename=id3.ts|nb_streams=1|nb_programs=1|format_name=mpegts|start_time=1.40|duration=5.015000|size=1504|bit_rate=2399|probe_score=2
> >
> > Looks good, can someone please put the sample in place?
>
> Done.
>
> 4200d5ad77c2495255742248cbfd69c3  fate-suite/mpegts/id3.ts

Thanks! I guess we still need someone's help pushing the patch adding the
test to the main repo.

> -Thilo
>
> ___
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] libavformat/mpegts.c: fix hardcoded 5-bytes skip for metadata streams.

2023-06-22 Thread Romain Beauxis

Le mer. 21 juin 2023 à 03:32, Anton Khirnov  a écrit :
>
> Quoting to...@rastageeks.org (2023-06-20 07:09:33)
> > From: Romain Beauxis 
> >
> > Before the introduction of AV_CODEC_ID_TIMED_ID3 for timed_id3 metadata
streams
> > in mpegts (commit 4a4437c0fbc8f7afe0c533070395a42e56b4ee75),
AV_CODEC_ID_SMPTE_KLV
> > was the only existing codec for metadata.
> >
> > It seems that this codec has a 5-bytes metadata header[1] that, for
some reason,
> > was always skipped when decoding data packets.
> >
> > However, when working with a AV_CODEC_ID_TIMED_ID3 streams, this
results in the
> > 5 first bytes of the payload being cut-off, which includes essential
informations
> > such as the ID3 tag version.
> >
> > This patch fixes the issue by keeping the 5-bytes skip only for
AV_CODEC_ID_SMPTE_KLV
> > streams.
> >
> > To test:
> > 1. download this file:
https://www.dropbox.com/s/jy8sih3pe8qskxb/bla.ts?dl=1
> >
> > This file was download from:
http://playertest.longtailvideo.com/adaptive/wowzaid3/playlist.m3u8
> >
> > 2. run this command:
> >   ffprobe -show_streams -select_streams 0 -show_packets
-show_private_data \
> >   -show_data /path/to/bla.ts
> >
> > Before:
> > [PACKET]
> > codec_type=data
> > stream_index=0
> > pts=494646418
> > pts_time=5496.071311
> > dts=494646418
> > dts_time=5496.071311
> > duration=N/A
> > duration_time=N/A
> > size=21
> > pos=482784
> > flags=K__
> > data=
> > :   1054 4954 3200  0600 0003  .TIT2...
> > 0010: 7465 7374 00 test.
> >
> > After:
> > [PACKET]
> > codec_type=data
> > stream_index=0
> > pts=494646418
> > pts_time=5496.071311
> > dts=494646418
> > dts_time=5496.071311
> > duration=N/A
> > duration_time=N/A
> > size=26
> > pos=482784
> > flags=K__
> > data=
> > : 4944 3304   0010 5449 5432   ID3...TIT2..
> > 0010: 0006  0374 6573 7400 .test.
>
> A FATE test for this would be nice.

Here's a patch attached, along with the sample that should go into
fate-suite/mpegts/id3.ts

The output has been verified with a local ffprobe build:

packet|codec_type=data|stream_index=0|pts=126000|pts_time=1.40|dts=126000|dts_time=1.40|duration=N/A|duration_time=N/A|size=26|pos=564|flags=K__|data=\n:
4944 3304   0010 5449 5432   ID3...TIT2..\n0010: 0006
 0374 6573 7400
.test.\n|side_data|side_data_type=MPEGTS Stream ID|id=189

packet|codec_type=data|stream_index=0|pts=577350|pts_time=6.415000|dts=577350|dts_time=6.415000|duration=N/A|duration_time=N/A|size=26|pos=1316|flags=K__|data=\n:
4944 3304   0010 5449 5432   ID3...TIT2..\n0010: 0006
 0374 6573 7400
.test.\n|side_data|side_data_type=MPEGTS Stream ID|id=189

stream|index=0|codec_name=timed_id3|profile=unknown|codec_type=data|codec_tag_string=ID3
|codec_tag=0x20334449|ts_packetsize=188|id=0x100|r_frame_rate=0/0|avg_frame_rate=0/0|time_base=1/9|start_pts=126000|start_time=1.40|duration_ts=451350|duration=5.015000|bit_rate=N/A|max_bit_rate=N/A|bits_per_raw_sample=N/A|nb_frames=N/A|nb_read_frames=N/A|nb_read_packets=2|extradata=\n|disposition:default=0|disposition:dub=0|disposition:original=0|disposition:comment=0|disposition:lyrics=0|disposition:karaoke=0|disposition:forced=0|disposition:hearing_impaired=0|disposition:visual_impaired=0|disposition:clean_effects=0|disposition:attached_pic=0|disposition:timed_thumbnails=0|disposition:captions=0|disposition:descriptions=0|disposition:metadata=0|disposition:dependent=0|disposition:still_image=0
format|filename=id3.ts|nb_streams=1|nb_programs=1|format_name=mpegts|start_time=1.40|duration=5.015000|size=1504|bit_rate=2399|probe_score=2

> --
> Anton Khirnov


timed_id3_fate_test.patch
Description: Binary data


id3.ts
Description: Binary data
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] libavformat/mpegts.c: fix hardcoded 5-bytes skip for metadata streams.

2023-06-20 Thread Romain Beauxis

Le mar. 20 juin 2023 à 02:10, Paul B Mahol  a écrit :
>
>
>
> On Tue, Jun 20, 2023 at 7:19 AM  wrote:
>>
>> From: Romain Beauxis 
>>
>> Before the introduction of AV_CODEC_ID_TIMED_ID3 for timed_id3 metadata
streams
>> in mpegts (commit 4a4437c0fbc8f7afe0c533070395a42e56b4ee75),
AV_CODEC_ID_SMPTE_KLV
>> was the only existing codec for metadata.
>>
>> It seems that this codec has a 5-bytes metadata header[1] that, for some
reason,
>> was always skipped when decoding data packets.
>>
>> However, when working with a AV_CODEC_ID_TIMED_ID3 streams, this results
in the
>> 5 first bytes of the payload being cut-off, which includes essential
informations
>> such as the ID3 tag version.
>>
>> This patch fixes the issue by keeping the 5-bytes skip only for
AV_CODEC_ID_SMPTE_KLV
>> streams.
>>
>> To test:
>> 1. download this file:
https://www.dropbox.com/s/jy8sih3pe8qskxb/bla.ts?dl=1
>>
>> This file was download from:
http://playertest.longtailvideo.com/adaptive/wowzaid3/playlist.m3u8
>>
>> 2. run this command:
>>   ffprobe -show_streams -select_streams 0 -show_packets
-show_private_data \
>>   -show_data /path/to/bla.ts
>>
>> Before:
>> [PACKET]
>> codec_type=data
>> stream_index=0
>> pts=494646418
>> pts_time=5496.071311
>> dts=494646418
>> dts_time=5496.071311
>> duration=N/A
>> duration_time=N/A
>> size=21
>> pos=482784
>> flags=K__
>> data=
>> :   1054 4954 3200  0600 0003  .TIT2...
>> 0010: 7465 7374 00 test.
>>
>> After:
>> [PACKET]
>> codec_type=data
>> stream_index=0
>> pts=494646418
>> pts_time=5496.071311
>> dts=494646418
>> dts_time=5496.071311
>> duration=N/A
>> duration_time=N/A
>> size=26
>> pos=482784
>> flags=K__
>> data=
>> : 4944 3304   0010 5449 5432   ID3...TIT2..
>> 0010: 0006  0374 6573 7400 .test.
>>
>> ---
>>  libavformat/mpegts.c | 2 +-
>>  1 file changed, 1 insertion(+), 1 deletion(-)
>>
>> diff --git a/libavformat/mpegts.c b/libavformat/mpegts.c
>> index fb8b0bf8fd..0b3edda817 100644
>> --- a/libavformat/mpegts.c
>> +++ b/libavformat/mpegts.c
>> @@ -1305,7 +1305,7 @@ skip:
>>  p += sl_header_bytes;
>>  buf_size -= sl_header_bytes;
>>  }
>> -if (pes->stream_type == 0x15 && buf_size >= 5) {
>> +if (pes->st->codecpar->codec_id ==
AV_CODEC_ID_SMPTE_KLV && buf_size >= 5) {
>>  /* skip metadata access unit header */
>>  pes->pes_header_size += 5;
>>  p += 5;
>> --
>> 2.39.2 (Apple Git-143)
>
>
> LGTM

Great, thanks!

Anything I can do to help move this to the finish line?

Thanks,
-- Romain

>>
>>
>> ___
>> ffmpeg-devel mailing list
>> ffmpeg-devel@ffmpeg.org
>> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>>
>> To unsubscribe, visit link above, or email
>> ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 2/2] libavformat/oggparseopus: Clear existing stream metadata before parsing potentially new one. Fixes: #10363

2023-05-18 Thread Romain Beauxis

Hello again!

I wanted to see if there was any interest in this patch and the other one
adding metadata decoding for chained ogg bitstream. These two feel like
easy bugfixes and features to add to the next release.

Reproduction steps for this one are detailed here:
https://trac.ffmpeg.org/ticket/10363

-- Romain

Le dim. 14 mai 2023 à 16:42,  a écrit :

> From: Romain Beauxis 
>
> This is the third version of a series of patches improving metadata
> support in
> chained ogg streams.
>
> Previous versions of this patch were including changes that were later
> identified as issues from another encoded and fixed there. See:
> https://github.com/savonet/liquidsoap/pull/3062
>
> The remaining changes address a memory leak in chained ogg/opus stream
> metadata. Reproduction steps for the issue are detailed in:
> https://trac.ffmpeg.org/ticket/10363
>
> ---
>  libavformat/oggparseopus.c | 1 +
>  1 file changed, 1 insertion(+)
>
> diff --git a/libavformat/oggparseopus.c b/libavformat/oggparseopus.c
> index 54aa725be6..86977b41db 100644
> --- a/libavformat/oggparseopus.c
> +++ b/libavformat/oggparseopus.c
> @@ -80,6 +80,7 @@ static int opus_header(AVFormatContext *avf, int idx)
>  if (priv->need_comments) {
>  if (os->psize < 8 || memcmp(packet, "OpusTags", 8))
>  return AVERROR_INVALIDDATA;
> +av_dict_free(>metadata);
>  ff_vorbis_stream_comment(avf, st, packet + 8, os->psize - 8);
>  priv->need_comments--;
>  return 1;
> --
> 2.37.1 (Apple Git-137.1)
>
>
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v3 1/2] libavformat/oggparseflac: Decode metadata packets. Fixes: #10364

2023-05-14 Thread Romain Beauxis

Le dim. 14 mai 2023 à 16:40,  a écrit :
>
> From: Romain Beauxis 
>
> This is the third version on a series of patches improving ffmpeg support for
> ogg chained streams.
>
> Reproduction steps for the issue fixed with patch are included in this bug
> report: https://trac.ffmpeg.org/ticket/10363

Sorry correct link is: https://trac.ffmpeg.org/ticket/10364

>
> ---
>  libavformat/oggparseflac.c | 20 
>  1 file changed, 20 insertions(+)
>
> diff --git a/libavformat/oggparseflac.c b/libavformat/oggparseflac.c
> index eef6e09927..1dd292483d 100644
> --- a/libavformat/oggparseflac.c
> +++ b/libavformat/oggparseflac.c
> @@ -126,10 +126,30 @@ fail:
>  return ret;
>  }
>
> +static int flac_packet(AVFormatContext *s, int idx)
> +{
> +struct ogg *ogg = s->priv_data;
> +struct ogg_stream *os = ogg->streams + idx;
> +int ret;
> +
> +if (os->psize > 4 && (*(os->buf + os->pstart) & 0x7F) == 
> FLAC_METADATA_TYPE_VORBIS_COMMENT) {
> +AVStream *st = s->streams[idx];
> +av_dict_free(>metadata);
> +ret = ff_vorbis_stream_comment(s, st, os->buf + os->pstart + 4,
> +   os->psize - 4);
> +
> +if (ret < 0) return ret;
> +}
> +
> +return 0;
> +}
> +
> +
>  const struct ogg_codec ff_flac_codec = {
>  .magic = "\177FLAC",
>  .magicsize = 5,
>  .header = flac_header,
> +.packet = flac_packet,
>  .nb_header = 2,
>  };
>
> --
> 2.37.1 (Apple Git-137.1)
>
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/4] Use appropriate method for device discovery, fix crash with bogus device index.

2022-01-31 Thread Romain Beauxis

Le lun. 31 janv. 2022 à 04:11, Thilo Borgmann  a
écrit :
>
> Hi,
>
> Am 30.01.22 um 18:30 schrieb to...@rastageeks.org:
> > From: Romain Beauxis 
> >
> > This updates the code for avfoundation to use modern device lookup APIs
and also adds a check to avoid querying the video devices array beyound its
maximum size.
> >
> > ---
> >   libavdevice/avfoundation.m | 71 ++
> >   1 file changed, 56 insertions(+), 15 deletions(-)
> >
> > diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
> > index 0cd6e646d5..d8bcd98f81 100644
> > --- a/libavdevice/avfoundation.m
> > +++ b/libavdevice/avfoundation.m
> > @@ -27,6 +27,7 @@
> >
> >   #import 
> >   #include 
> > +#include 
> >
> >   #include "libavutil/channel_layout.h"
> >   #include "libavutil/pixdesc.h"
> > @@ -764,8 +765,34 @@ static int avf_read_header(AVFormatContext *s)
> [...]
>
> the patch doesn't appear to be broken any more, though it does not apply
to current HEAD
>
> Thilos-Mac-mini:FFmpeg borgmann$ git apply
../patches/avfoundation/\[PATCH\ 1_4\]\ Use\ appropriate\ method\ for\
device\ discovery\,\ fix\ crash\ with\ bogus\ device\ index.\ -\ toots\@
rastageeks.org\ -\ 2022-01-30\ 1830.eml
> error: patch failed: libavdevice/avfoundation.m:27
> error: libavdevice/avfoundation.m: patch does not apply
>
> I don't see why it does not apply as the part around line 27 looks sane.
Did you try to apply to HEAD?

It does seem to apply here:
https://patchwork.ffmpeg.org/project/ffmpeg/patch/20220130173045.32690-2-to...@rastageeks.org/
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 5/5] Add AudioToolbox audio input device.

2022-01-29 Thread Romain Beauxis

Le sam. 29 janv. 2022 à 14:42, Andreas Rheinhardt
 a écrit :
>
> Romain Beauxis:
> > Le mer. 19 janv. 2022 à 10:59, Marvin Scholz  a écrit :
> >>
> >>
> >>
> >> On 19 Jan 2022, at 15:42, Romain Beauxis wrote:
> >>
> >> Hi, thanks for the patch. I've not done a full code review yet, just a
> >> few
> >> initial remarks below:
> >>
> >>> This patch adds support for a new, audio-specific input device using
> >>> the documented and battle-tested AUHAL input. This provides a pendant
> >>> to the AudioToolbox audio-only output.
> >>>
> >>> A couple of advantages for this:
> >>> * It avoids a lot of the complexity of supporting audio and video in a
> >>> single input
> >>> * The AUHAL API seems tested, documented and robust
> >>> * This implementation hopefully gives good control over audio latency
> >>> and also minimizes data copy
> >>> From: Romain Beauxis 
> >>> To: ffmpeg-devel@ffmpeg.org
> >>> Subject: [PATCH] Add AudioToolbox audio input device.
> >>> Date: 18. January 2022 at 23:29
> >>> Signed-off-by: Romain Beauxis 
> >>> ---
> >>>  configure  |   5 +
> >>>  doc/indevs.texi|  44 
> >>>  libavdevice/Makefile   |   1 +
> >>>  libavdevice/alldevices.c   |   1 +
> >>>  libavdevice/audiotoolbox_dec.m | 466
> >>> +
> >>>  5 files changed, 517 insertions(+)
> >>>  create mode 100644 libavdevice/audiotoolbox_dec.m
> >>>
> >>> diff --git a/configure b/configure
> >>> index 1413122d87..80e39aae44 100755
> >>> --- a/configure
> >>> +++ b/configure
> >>> @@ -204,6 +204,7 @@ External library support:
> >>>--disable-avfoundation   disable Apple AVFoundation framework
> >>> [autodetect]
> >>>--enable-avisynthenable reading of AviSynth script files
> >>> [no]
> >>>--disable-bzlib  disable bzlib [autodetect]
> >>> +  --disable-coremedia  disable Apple CoreMedia framework
> >>> [autodetect]
> >>>--disable-coreimage  disable Apple CoreImage framework
> >>> [autodetect]
> >>>--enable-chromaprint enable audio fingerprinting with
> >>> chromaprint [no]
> >>>--enable-frei0r  enable frei0r video filtering [no]
> >>> @@ -1750,6 +1751,7 @@ EXTERNAL_AUTODETECT_LIBRARY_LIST="
> >>>  appkit
> >>>  avfoundation
> >>>  bzlib
> >>> +coremedia
> >>>  coreimage
> >>>  iconv
> >>>  libxcb
> >>> @@ -3493,6 +3495,8 @@ alsa_outdev_deps="alsa"
> >>>  avfoundation_indev_deps="avfoundation corevideo coremedia pthreads"
> >>>  avfoundation_indev_suggest="coregraphics applicationservices"
> >>>  avfoundation_indev_extralibs="-framework Foundation"
> >>> +audiotoolbox_indev_deps="coremedia audiotoolbox"
> >>> +audiotoolbox_indev_extralibs="-framework CoreMedia -framework
> >>> AudioToolbox"
> >>>  audiotoolbox_outdev_deps="audiotoolbox pthreads"
> >>>  audiotoolbox_outdev_extralibs="-framework AudioToolbox -framework
> >>> CoreAudio"
> >>>  bktr_indev_deps_any="dev_bktr_ioctl_bt848_h machine_ioctl_bt848_h
> >>> dev_video_bktr_ioctl_bt848_h dev_ic_bt8xx_h"
> >>> @@ -6340,6 +6344,7 @@ check_lib camera2ndk "stdbool.h stdint.h
> >>> camera/NdkCameraManager.h" ACameraManag
> >>>  enabled appkit   && check_apple_framework AppKit
> >>>  enabled audiotoolbox && check_apple_framework AudioToolbox
> >>>  enabled avfoundation && check_apple_framework AVFoundation
> >>> +enabled coremedia&& check_apple_framework CoreMedia
> >>>  enabled coreimage&& check_apple_framework CoreImage
> >>>  enabled metal&& check_apple_framework Metal
> >>>  enabled videotoolbox && check_apple_framework VideoToolbox
> >>> diff --git a/doc/indevs.texi b/doc/indevs.texi
> >>> index 858c0fa4e4..30a91d304f 100644
> >>> --- a/doc/indevs.texi
> >>> +++ b/doc/indevs.texi
> >>> @@ -103,6 +103,50 @@ Set the maximum number of frames to buffer.
> &

Re: [FFmpeg-devel] [PATCH 5/5] Add AudioToolbox audio input device.

2022-01-29 Thread Romain Beauxis

Le mer. 19 janv. 2022 à 10:59, Marvin Scholz  a écrit :
>
>
>
> On 19 Jan 2022, at 15:42, Romain Beauxis wrote:
>
> Hi, thanks for the patch. I've not done a full code review yet, just a
> few
> initial remarks below:
>
> > This patch adds support for a new, audio-specific input device using
> > the documented and battle-tested AUHAL input. This provides a pendant
> > to the AudioToolbox audio-only output.
> >
> > A couple of advantages for this:
> > * It avoids a lot of the complexity of supporting audio and video in a
> > single input
> > * The AUHAL API seems tested, documented and robust
> > * This implementation hopefully gives good control over audio latency
> > and also minimizes data copy
> > From: Romain Beauxis 
> > To: ffmpeg-devel@ffmpeg.org
> > Subject: [PATCH] Add AudioToolbox audio input device.
> > Date: 18. January 2022 at 23:29
> > Signed-off-by: Romain Beauxis 
> > ---
> >  configure  |   5 +
> >  doc/indevs.texi|  44 
> >  libavdevice/Makefile   |   1 +
> >  libavdevice/alldevices.c   |   1 +
> >  libavdevice/audiotoolbox_dec.m | 466
> > +
> >  5 files changed, 517 insertions(+)
> >  create mode 100644 libavdevice/audiotoolbox_dec.m
> >
> > diff --git a/configure b/configure
> > index 1413122d87..80e39aae44 100755
> > --- a/configure
> > +++ b/configure
> > @@ -204,6 +204,7 @@ External library support:
> >--disable-avfoundation   disable Apple AVFoundation framework
> > [autodetect]
> >--enable-avisynthenable reading of AviSynth script files
> > [no]
> >--disable-bzlib  disable bzlib [autodetect]
> > +  --disable-coremedia  disable Apple CoreMedia framework
> > [autodetect]
> >--disable-coreimage  disable Apple CoreImage framework
> > [autodetect]
> >--enable-chromaprint enable audio fingerprinting with
> > chromaprint [no]
> >--enable-frei0r  enable frei0r video filtering [no]
> > @@ -1750,6 +1751,7 @@ EXTERNAL_AUTODETECT_LIBRARY_LIST="
> >  appkit
> >  avfoundation
> >  bzlib
> > +coremedia
> >  coreimage
> >  iconv
> >  libxcb
> > @@ -3493,6 +3495,8 @@ alsa_outdev_deps="alsa"
> >  avfoundation_indev_deps="avfoundation corevideo coremedia pthreads"
> >  avfoundation_indev_suggest="coregraphics applicationservices"
> >  avfoundation_indev_extralibs="-framework Foundation"
> > +audiotoolbox_indev_deps="coremedia audiotoolbox"
> > +audiotoolbox_indev_extralibs="-framework CoreMedia -framework
> > AudioToolbox"
> >  audiotoolbox_outdev_deps="audiotoolbox pthreads"
> >  audiotoolbox_outdev_extralibs="-framework AudioToolbox -framework
> > CoreAudio"
> >  bktr_indev_deps_any="dev_bktr_ioctl_bt848_h machine_ioctl_bt848_h
> > dev_video_bktr_ioctl_bt848_h dev_ic_bt8xx_h"
> > @@ -6340,6 +6344,7 @@ check_lib camera2ndk "stdbool.h stdint.h
> > camera/NdkCameraManager.h" ACameraManag
> >  enabled appkit   && check_apple_framework AppKit
> >  enabled audiotoolbox && check_apple_framework AudioToolbox
> >  enabled avfoundation && check_apple_framework AVFoundation
> > +enabled coremedia&& check_apple_framework CoreMedia
> >  enabled coreimage&& check_apple_framework CoreImage
> >  enabled metal&& check_apple_framework Metal
> >  enabled videotoolbox && check_apple_framework VideoToolbox
> > diff --git a/doc/indevs.texi b/doc/indevs.texi
> > index 858c0fa4e4..30a91d304f 100644
> > --- a/doc/indevs.texi
> > +++ b/doc/indevs.texi
> > @@ -103,6 +103,50 @@ Set the maximum number of frames to buffer.
> > Default is 5.
> >
> >  @end table
> >
> > +@section AudioToolbox
> > +
> > +AudioToolbox input device.
> > +
> > +Allows native input from CoreAudio devices on OSX.
> Nit: Nowadays it's macOS instead of OSX
>
> > +
> > +All available devices can be enumerated by using
> > @option{-list_devices true}, listing
> > +all device names, and corresponding unique ID.
>
> Instead of adding another device that uses a custom list-devices option,
> could you
> instead implement the .get_device_list callback? (See alsa or pulse
> modules for an
> example) Then listing would work properly with the `ffmpeg -sources`
> command and
> devices could be iterated over using the avdevic

Re: [FFmpeg-devel] [PATCH 5/5] Add AudioToolbox audio input device.

2022-01-24 Thread Romain Beauxis

Le lun. 24 janv. 2022 à 10:19, Marvin Scholz  a écrit :
>
>
>
> On 24 Jan 2022, at 16:42, Romain Beauxis wrote:
>
> > Hi Marvin,
> >
> > Le mer. 19 janv. 2022 à 10:59, Marvin Scholz  a
> > écrit :
> >> On 19 Jan 2022, at 15:42, Romain Beauxis wrote:
> >>
> >> Hi, thanks for the patch. I've not done a full code review yet, just
> >> a
> >> few
> >> initial remarks below:
> >>
> >>> This patch adds support for a new, audio-specific input device using
> >>> the documented and battle-tested AUHAL input. This provides a
> >>> pendant
> >>> to the AudioToolbox audio-only output.
> >>>
> >>> A couple of advantages for this:
> >>> * It avoids a lot of the complexity of supporting audio and video in
> >>> a
> >>> single input
> >>> * The AUHAL API seems tested, documented and robust
> >>> * This implementation hopefully gives good control over audio
> >>> latency
> >>> and also minimizes data copy
> >
> > Thanks!
> >
> > Do you need more time for this review? We discovered another issue
> > with the dynamic array of video devices and I'd like to send a revised
> > series soon.
>
> Well if you want to address the things I mentioned last time
> it will anyway change a bunch of things probably, so it
> might be easier if I just wait for the new patch.
>
> Additionally it might be useful to send it independent of the
> whole patchset as IIUC it does not really depend on any of the
> previous patches.

Will do, thanks!

> > Also, I think I'll reorder the patches so that the most
> > trivial one (unique device name, probing API call updates) can be
> > applied right away.
> >
> > Let me know!
> > Romain
> >
> >>> From: Romain Beauxis 
> >>> To: ffmpeg-devel@ffmpeg.org
> >>> Subject: [PATCH] Add AudioToolbox audio input device.
> >>> Date: 18. January 2022 at 23:29
> >>> Signed-off-by: Romain Beauxis 
> >>> ---
> >>>  configure  |   5 +
> >>>  doc/indevs.texi|  44 
> >>>  libavdevice/Makefile   |   1 +
> >>>  libavdevice/alldevices.c   |   1 +
> >>>  libavdevice/audiotoolbox_dec.m | 466
> >>> +
> >>>  5 files changed, 517 insertions(+)
> >>>  create mode 100644 libavdevice/audiotoolbox_dec.m
> >>>
> >>> diff --git a/configure b/configure
> >>> index 1413122d87..80e39aae44 100755
> >>> --- a/configure
> >>> +++ b/configure
> >>> @@ -204,6 +204,7 @@ External library support:
> >>>--disable-avfoundation   disable Apple AVFoundation framework
> >>> [autodetect]
> >>>--enable-avisynthenable reading of AviSynth script files
> >>> [no]
> >>>--disable-bzlib  disable bzlib [autodetect]
> >>> +  --disable-coremedia  disable Apple CoreMedia framework
> >>> [autodetect]
> >>>--disable-coreimage  disable Apple CoreImage framework
> >>> [autodetect]
> >>>--enable-chromaprint enable audio fingerprinting with
> >>> chromaprint [no]
> >>>--enable-frei0r  enable frei0r video filtering [no]
> >>> @@ -1750,6 +1751,7 @@ EXTERNAL_AUTODETECT_LIBRARY_LIST="
> >>>  appkit
> >>>  avfoundation
> >>>  bzlib
> >>> +coremedia
> >>>  coreimage
> >>>  iconv
> >>>  libxcb
> >>> @@ -3493,6 +3495,8 @@ alsa_outdev_deps="alsa"
> >>>  avfoundation_indev_deps="avfoundation corevideo coremedia pthreads"
> >>>  avfoundation_indev_suggest="coregraphics applicationservices"
> >>>  avfoundation_indev_extralibs="-framework Foundation"
> >>> +audiotoolbox_indev_deps="coremedia audiotoolbox"
> >>> +audiotoolbox_indev_extralibs="-framework CoreMedia -framework
> >>> AudioToolbox"
> >>>  audiotoolbox_outdev_deps="audiotoolbox pthreads"
> >>>  audiotoolbox_outdev_extralibs="-framework AudioToolbox -framework
> >>> CoreAudio"
> >>>  bktr_indev_deps_any="dev_bktr_ioctl_bt848_h machine_ioctl_bt848_h
> >>> dev_video_bktr_ioctl_bt848_h dev_ic_bt8xx_h"
> >>> @@ -6340,6 +6344,7 @@ check_lib camera2ndk "stdbool.h stdint.h
> >&g

Re: [FFmpeg-devel] [PATCH 5/5] Add AudioToolbox audio input device.

2022-01-24 Thread Romain Beauxis

Hi Marvin,

Le mer. 19 janv. 2022 à 10:59, Marvin Scholz  a écrit :
> On 19 Jan 2022, at 15:42, Romain Beauxis wrote:
>
> Hi, thanks for the patch. I've not done a full code review yet, just a
> few
> initial remarks below:
>
> > This patch adds support for a new, audio-specific input device using
> > the documented and battle-tested AUHAL input. This provides a pendant
> > to the AudioToolbox audio-only output.
> >
> > A couple of advantages for this:
> > * It avoids a lot of the complexity of supporting audio and video in a
> > single input
> > * The AUHAL API seems tested, documented and robust
> > * This implementation hopefully gives good control over audio latency
> > and also minimizes data copy

Thanks!

Do you need more time for this review? We discovered another issue
with the dynamic array of video devices and I'd like to send a revised
series soon. Also, I think I'll reorder the patches so that the most
trivial one (unique device name, probing API call updates) can be
applied right away.

Let me know!
Romain

> > From: Romain Beauxis 
> > To: ffmpeg-devel@ffmpeg.org
> > Subject: [PATCH] Add AudioToolbox audio input device.
> > Date: 18. January 2022 at 23:29
> > Signed-off-by: Romain Beauxis 
> > ---
> >  configure  |   5 +
> >  doc/indevs.texi|  44 
> >  libavdevice/Makefile   |   1 +
> >  libavdevice/alldevices.c   |   1 +
> >  libavdevice/audiotoolbox_dec.m | 466
> > +
> >  5 files changed, 517 insertions(+)
> >  create mode 100644 libavdevice/audiotoolbox_dec.m
> >
> > diff --git a/configure b/configure
> > index 1413122d87..80e39aae44 100755
> > --- a/configure
> > +++ b/configure
> > @@ -204,6 +204,7 @@ External library support:
> >--disable-avfoundation   disable Apple AVFoundation framework
> > [autodetect]
> >--enable-avisynthenable reading of AviSynth script files
> > [no]
> >--disable-bzlib  disable bzlib [autodetect]
> > +  --disable-coremedia  disable Apple CoreMedia framework
> > [autodetect]
> >--disable-coreimage  disable Apple CoreImage framework
> > [autodetect]
> >--enable-chromaprint enable audio fingerprinting with
> > chromaprint [no]
> >--enable-frei0r  enable frei0r video filtering [no]
> > @@ -1750,6 +1751,7 @@ EXTERNAL_AUTODETECT_LIBRARY_LIST="
> >  appkit
> >  avfoundation
> >  bzlib
> > +coremedia
> >  coreimage
> >  iconv
> >  libxcb
> > @@ -3493,6 +3495,8 @@ alsa_outdev_deps="alsa"
> >  avfoundation_indev_deps="avfoundation corevideo coremedia pthreads"
> >  avfoundation_indev_suggest="coregraphics applicationservices"
> >  avfoundation_indev_extralibs="-framework Foundation"
> > +audiotoolbox_indev_deps="coremedia audiotoolbox"
> > +audiotoolbox_indev_extralibs="-framework CoreMedia -framework
> > AudioToolbox"
> >  audiotoolbox_outdev_deps="audiotoolbox pthreads"
> >  audiotoolbox_outdev_extralibs="-framework AudioToolbox -framework
> > CoreAudio"
> >  bktr_indev_deps_any="dev_bktr_ioctl_bt848_h machine_ioctl_bt848_h
> > dev_video_bktr_ioctl_bt848_h dev_ic_bt8xx_h"
> > @@ -6340,6 +6344,7 @@ check_lib camera2ndk "stdbool.h stdint.h
> > camera/NdkCameraManager.h" ACameraManag
> >  enabled appkit   && check_apple_framework AppKit
> >  enabled audiotoolbox && check_apple_framework AudioToolbox
> >  enabled avfoundation && check_apple_framework AVFoundation
> > +enabled coremedia&& check_apple_framework CoreMedia
> >  enabled coreimage&& check_apple_framework CoreImage
> >  enabled metal&& check_apple_framework Metal
> >  enabled videotoolbox && check_apple_framework VideoToolbox
> > diff --git a/doc/indevs.texi b/doc/indevs.texi
> > index 858c0fa4e4..30a91d304f 100644
> > --- a/doc/indevs.texi
> > +++ b/doc/indevs.texi
> > @@ -103,6 +103,50 @@ Set the maximum number of frames to buffer.
> > Default is 5.
> >
> >  @end table
> >
> > +@section AudioToolbox
> > +
> > +AudioToolbox input device.
> > +
> > +Allows native input from CoreAudio devices on OSX.
> Nit: Nowadays it's macOS instead of OSX
>
> > +
> > +All available devices can be enumerated by using
> > @option{-list_devices true}, listing
> > +all device names, and corresponding unique ID.
>
> Instea

Re: [FFmpeg-devel] [PATCH 1/5] libavdevice/avfoundation.m: use setAudioSettings, extend supported formats

2022-01-19 Thread Romain Beauxis

Le mer. 19 janv. 2022 à 09:45, Gyan Doshi  a écrit :
> On 2022-01-19 08:51 pm, Romain Beauxis wrote:
> > Le mer. 19 janv. 2022 à 09:19, Gyan Doshi  a écrit :
> >> On 2022-01-19 08:44 pm, Romain Beauxis wrote:
> >>> Le mer. 19 janv. 2022 à 08:31, Gyan Doshi  a écrit :
> >>>> On 2022-01-19 07:53 pm, Romain Beauxis wrote:
> >>>>> This patch switches the logic around audio settings to let the caller 
> >>>>> drive the format.
> >>>>>
> >>>>> After experimenting with the AudioConverter, we realized that, even 
> >>>>> when adhering to a strict implementation of the documented API, we were 
> >>>>> still getting errors during conversions. The input device would 
> >>>>> randomly change from e.g. s32le to s24le between restarts and error out 
> >>>>> on conversion (using a freshly initialized converter).
> >>>> At present, the code uses the first frame to set attributes. If you
> >>>> wait for a few frames and then probe, the attributes are stable.
> >>> How is that supposed to work to get a full A/V stream? Discarding
> >>> initial audio frames results in data loss in audio-only input and
> >>> corrupted initial audio in A/V inputs.
> >> We're talking about around 5-6 packets, so ~100 ms. The streaming
> >> scenarios I worked on weren't sensitive to that amount of initial loss.
> >> YMMV.
> > I see thanks. And what advantages does this method provide aside from
> > supporting 24 bit sample formats which are currently excluded from
> > these changes?
>
> I was remarking on a way to avoid format changes post-initialization.
> Not a comment on your patches.

For sure, and I appreciate the feedback.
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/5] libavdevice/avfoundation.m: use setAudioSettings, extend supported formats

2022-01-19 Thread Romain Beauxis

Le mer. 19 janv. 2022 à 09:19, Gyan Doshi  a écrit :
> On 2022-01-19 08:44 pm, Romain Beauxis wrote:
> > Le mer. 19 janv. 2022 à 08:31, Gyan Doshi  a écrit :
> >> On 2022-01-19 07:53 pm, Romain Beauxis wrote:
> >>> This patch switches the logic around audio settings to let the caller 
> >>> drive the format.
> >>>
> >>> After experimenting with the AudioConverter, we realized that, even when 
> >>> adhering to a strict implementation of the documented API, we were still 
> >>> getting errors during conversions. The input device would randomly change 
> >>> from e.g. s32le to s24le between restarts and error out on conversion 
> >>> (using a freshly initialized converter).
> >>At present, the code uses the first frame to set attributes. If you
> >> wait for a few frames and then probe, the attributes are stable.
> > How is that supposed to work to get a full A/V stream? Discarding
> > initial audio frames results in data loss in audio-only input and
> > corrupted initial audio in A/V inputs.
>
> We're talking about around 5-6 packets, so ~100 ms. The streaming
> scenarios I worked on weren't sensitive to that amount of initial loss.
> YMMV.

I see thanks. And what advantages does this method provide aside from
supporting 24 bit sample formats which are currently excluded from
these changes?
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH 1/5] libavdevice/avfoundation.m: use setAudioSettings, extend supported formats

2022-01-19 Thread Romain Beauxis

Le mer. 19 janv. 2022 à 08:31, Gyan Doshi  a écrit :
> On 2022-01-19 07:53 pm, Romain Beauxis wrote:
> > This patch switches the logic around audio settings to let the caller drive 
> > the format.
> >
> > After experimenting with the AudioConverter, we realized that, even when 
> > adhering to a strict implementation of the documented API, we were still 
> > getting errors during conversions. The input device would randomly change 
> > from e.g. s32le to s24le between restarts and error out on conversion 
> > (using a freshly initialized converter).
>   At present, the code uses the first frame to set attributes. If you
> wait for a few frames and then probe, the attributes are stable.

How is that supposed to work to get a full A/V stream? Discarding
initial audio frames results in data loss in audio-only input and
corrupted initial audio in A/V inputs.
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH 5/5] Add AudioToolbox audio input device.

2022-01-19 Thread Romain Beauxis

This patch adds support for a new, audio-specific input device using the 
documented and battle-tested AUHAL input. This provides a pendant to the 
AudioToolbox audio-only output.

A couple of advantages for this:
* It avoids a lot of the complexity of supporting audio and video in a single 
input
* The AUHAL API seems tested, documented and robust
* This implementation hopefully gives good control over audio latency and also 
minimizes data copy

--- Begin Message ---
From 803e1aa52018bdac0a448be255f645ca447273e9 Mon Sep 17 00:00:00 2001
From: Romain Beauxis 
Date: Tue, 18 Jan 2022 16:29:59 -0600
Subject: [PATCH] Add AudioToolbox audio input device.
X-Unsent: 1
To: ffmpeg-devel@ffmpeg.org

Signed-off-by: Romain Beauxis 
---
 configure  |   5 +
 doc/indevs.texi|  44 
 libavdevice/Makefile   |   1 +
 libavdevice/alldevices.c   |   1 +
 libavdevice/audiotoolbox_dec.m | 466 +
 5 files changed, 517 insertions(+)
 create mode 100644 libavdevice/audiotoolbox_dec.m

diff --git a/configure b/configure
index 1413122d87..80e39aae44 100755
--- a/configure
+++ b/configure
@@ -204,6 +204,7 @@ External library support:
   --disable-avfoundation   disable Apple AVFoundation framework [autodetect]
   --enable-avisynthenable reading of AviSynth script files [no]
   --disable-bzlib  disable bzlib [autodetect]
+  --disable-coremedia  disable Apple CoreMedia framework [autodetect]
   --disable-coreimage  disable Apple CoreImage framework [autodetect]
   --enable-chromaprint enable audio fingerprinting with chromaprint [no]
   --enable-frei0r  enable frei0r video filtering [no]
@@ -1750,6 +1751,7 @@ EXTERNAL_AUTODETECT_LIBRARY_LIST="
 appkit
 avfoundation
 bzlib
+coremedia
 coreimage
 iconv
 libxcb
@@ -3493,6 +3495,8 @@ alsa_outdev_deps="alsa"
 avfoundation_indev_deps="avfoundation corevideo coremedia pthreads"
 avfoundation_indev_suggest="coregraphics applicationservices"
 avfoundation_indev_extralibs="-framework Foundation"
+audiotoolbox_indev_deps="coremedia audiotoolbox"
+audiotoolbox_indev_extralibs="-framework CoreMedia -framework AudioToolbox"
 audiotoolbox_outdev_deps="audiotoolbox pthreads"
 audiotoolbox_outdev_extralibs="-framework AudioToolbox -framework CoreAudio"
 bktr_indev_deps_any="dev_bktr_ioctl_bt848_h machine_ioctl_bt848_h 
dev_video_bktr_ioctl_bt848_h dev_ic_bt8xx_h"
@@ -6340,6 +6344,7 @@ check_lib camera2ndk "stdbool.h stdint.h 
camera/NdkCameraManager.h" ACameraManag
 enabled appkit   && check_apple_framework AppKit
 enabled audiotoolbox && check_apple_framework AudioToolbox
 enabled avfoundation && check_apple_framework AVFoundation
+enabled coremedia&& check_apple_framework CoreMedia
 enabled coreimage&& check_apple_framework CoreImage
 enabled metal&& check_apple_framework Metal
 enabled videotoolbox && check_apple_framework VideoToolbox
diff --git a/doc/indevs.texi b/doc/indevs.texi
index 858c0fa4e4..30a91d304f 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -103,6 +103,50 @@ Set the maximum number of frames to buffer. Default is 5.
 
 @end table
 
+@section AudioToolbox
+
+AudioToolbox input device.
+
+Allows native input from CoreAudio devices on OSX.
+
+All available devices can be enumerated by using @option{-list_devices true}, 
listing
+all device names, and corresponding unique ID.
+
+@subsection Options
+
+AudioToolbox supports the following options:
+
+@table @option
+
+@item channels
+Set the number of channels. Default is device's default.
+
+@item frames_queue_length
+Maximum of buffers in the input queue
+
+@item buffer_frame_size
+Buffer frame size, gouverning internal latency
+
+@item big_endian
+Return big endian samples
+
+@item sample_format
+Sample format
+
+@end table
+
+@subsection Examples
+
+@itemize
+
+@item
+Print the list of supported devices
+@example
+$ ffmpeg -f audiotoolbox -list_devices true -i ""
+@end example
+
+@end itemize
+
 @section avfoundation
 
 AVFoundation input device.
diff --git a/libavdevice/Makefile b/libavdevice/Makefile
index 53efda0514..0c73255a21 100644
--- a/libavdevice/Makefile
+++ b/libavdevice/Makefile
@@ -14,6 +14,7 @@ OBJS-$(HAVE_LIBC_MSVCRT) += file_open.o
 OBJS-$(CONFIG_ALSA_INDEV)+= alsa_dec.o alsa.o timefilter.o
 OBJS-$(CONFIG_ALSA_OUTDEV)   += alsa_enc.o alsa.o
 OBJS-$(CONFIG_ANDROID_CAMERA_INDEV)  += android_camera.o
+OBJS-$(CONFIG_AUDIOTOOLBOX_INDEV)+= audiotoolbox_dec.o
 OBJS-$(CONFIG_AUDIOTOOLBOX_OUTDEV)   += audiotoolbox.o
 OBJS-$(CONFIG_AVFOUNDATION_INDEV)+= avfoundation.o
 OBJS-$(CONFIG_BKTR_INDEV)+= bktr.o
diff --git a/libavdevice/alldevices.c b/libavdevice/alldevices.c
index 22323a0a44..fbecdbb0b2 100644
--- a

[FFmpeg-devel] [PATCH 4/5] Use appropriate method for device discovery.

2022-01-19 Thread Romain Beauxis

This adds support for the non-deprecated methods for selecting devices, when 
available.

--- Begin Message ---
From c42612b455289622edb638436b1892e43279d8ac Mon Sep 17 00:00:00 2001
From: Romain Beauxis 
Date: Fri, 14 Jan 2022 10:06:08 -0600
Subject: [PATCH] Use appropriate method for device discovery.
X-Unsent: 1
To: ffmpeg-devel@ffmpeg.org

Signed-off-by: Romain Beauxis 
---
 libavdevice/avfoundation.m | 77 --
 1 file changed, 57 insertions(+), 20 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 5d013cc0eb..03ec49df86 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -27,6 +27,7 @@
 
 #import 
 #import 
+#include 
 
 #include "libavutil/channel_layout.h"
 #include "libavutil/pixdesc.h"
@@ -698,8 +699,34 @@ static int avf_read_header(AVFormatContext *s)
 AVCaptureDevice *video_device = nil;
 AVCaptureDevice *audio_device = nil;
 // Find capture device
-NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
-NSArray *devices_muxed = [AVCaptureDevice 
devicesWithMediaType:AVMediaTypeMuxed];
+#if defined(__MAC_10_15) || (TARGET_OS_IPHONE && defined(__IPHONE_10_0))
+   AVCaptureDeviceDiscoverySession *discoverySession =
+[AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[
+#if TARGET_OS_IPHONE
+ 
AVCaptureDeviceTypeBuiltInDualCamera,
+ 
AVCaptureDeviceTypeBuiltInDualWideCamera,
+ 
AVCaptureDeviceTypeBuiltInUltraWideCamera,
+ 
AVCaptureDeviceTypeBuiltInTrueDepthCamera,
+ 
AVCaptureDeviceTypeBuiltInTelephotoCamera,
+#endif
+ 
AVCaptureDeviceTypeBuiltInWideAngleCamera,
+ 
AVCaptureDeviceTypeExternalUnknown
+ ]
+ mediaType:NULL
+ 
position:AVCaptureDevicePositionUnspecified];
+
+   NSMutableArray *devices   = [NSMutableArray array];
+   NSMutableArray *devices_muxed = [NSMutableArray array];
+   for (AVCaptureDevice *device in [discoverySession devices]) {
+   if ([device hasMediaType:AVMediaTypeVideo])
+   [devices addObject:device];
+   else if ([device hasMediaType:AVMediaTypeMuxed])
+   [devices_muxed addObject:device];
+   }
+#else
+   NSArray *devices = [AVCaptureDevice 
devicesWithMediaType:AVMediaTypeVideo];
+   NSArray *devices_muxed = [AVCaptureDevice 
devicesWithMediaType:AVMediaTypeMuxed];
+#endif
 
 ctx->num_video_devices = [devices count] + [devices_muxed count];
 
@@ -707,6 +734,21 @@ static int avf_read_header(AVFormatContext *s)
 CGGetActiveDisplayList(0, NULL, _screens);
 #endif
 
+NSArray *audio_devices;
+#if defined(__MAC_10_15) || (TARGET_OS_IPHONE && defined(__IPHONE_10_0))
+discoverySession =
+[AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[
+ 
AVCaptureDeviceTypeBuiltInMicrophone,
+ AVCaptureDeviceTypeExternalUnknown
+ ]
+ mediaType:AVMediaTypeAudio
+ 
position:AVCaptureDevicePositionUnspecified];
+
+audio_devices = [discoverySession devices];
+#else
+audio_devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
+#endif
+
 // List devices if requested
 if (ctx->list_devices) {
 int index = 0;
@@ -734,8 +776,7 @@ static int avf_read_header(AVFormatContext *s)
 #endif
 
 av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
-devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
-for (AVCaptureDevice *device in devices) {
+for (AVCaptureDevice *device in audio_devices) {
 const char *name = [[device localizedName] UTF8String];
 const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
 int index= [devices indexOfObject:device];
@@ -885,9 +926,7 @@ static int avf_read_header(AVFormatContext *s)
 
 // get audio device
 if (ctx->audio_device_index >= 0) {
-NSArray *devices = [AVCaptureDevice 
devicesWithMediaType:AVMediaTypeAudio];
-
-if (ctx->audio_device_index >= [devices count]) {
+if (ctx->audio_device_index >= [audio_devices count]) {
 av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
 goto fail;
 }
@@ -898,22 +937,20 @@ static int avf_read_he

[FFmpeg-devel] [PATCH 3/5] libavdevice/avfoundation.m: Allow to select devices by unique ID

2022-01-19 Thread Romain Beauxis

This patch changes the logic around device selection to allow to use a static, 
machine-readable unique ID when selecting devices. Device names depends on 
locale settings and device index can change when plugging/unplugging devices.

--- Begin Message ---
From bf72d48c846f5116866ec588fc0ee54a2c354e87 Mon Sep 17 00:00:00 2001
From: Romain Beauxis 
Date: Mon, 13 Dec 2021 09:14:50 -0600
Subject: [PATCH] libavdevice/avfoundation.m: Allow to select devices by unique
 ID
X-Unsent: 1
To: ffmpeg-devel@ffmpeg.org

Signed-off-by: Romain Beauxis 
---
 doc/indevs.texi|  6 ++--
 libavdevice/avfoundation.m | 72 +-
 2 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 9d8020311a..858c0fa4e4 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
 -i "[[VIDEO]:[AUDIO]]"
 @end example
 The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or ID as shown by the 
device list.
 Alternatively, the video and/or audio input device can be chosen by index 
using the
 @option{
 -video_device_index 
@@ -127,7 +127,9 @@ and/or
 device name or index given in the input filename.
 
 All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name 
can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.
 
 There are two device name aliases:
 @table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index e6f64b35b8..5d013cc0eb 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -39,6 +39,8 @@
 #include "libavutil/imgutils.h"
 #include "avdevice.h"
 
+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" 
withString:@"."] UTF8String]
+
 static void av_log_avfoundation(void *s, int lvl, const char *str, OSStatus 
err) {
 NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
 av_log(s, lvl, "AVFoundation: %s, %s\n", str,
@@ -710,21 +712,23 @@ static int avf_read_header(AVFormatContext *s)
 int index = 0;
 av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
 for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
 }
 for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [devices_muxed 
indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices count] + [devices_muxed 
indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
 }
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
 if (num_screens > 0) {
 CGDirectDisplayID screens[num_screens];
 CGGetActiveDisplayList(num_screens, screens, _screens);
 for (int i = 0; i < num_screens; i++) {
-av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", 
ctx->num_video_devices + i, i);
+av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d (ID: 
AvfilterAvfoundationCaptureScreen%d)\n", ctx->num_video_devices + i, i, 
screens[i]);
 }
 }
 #endif
@@ -732,9 +736,10 @@ static int avf_read_header(AVFormatContext *s)
 av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
 devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
 for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-int index  = [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, nam

[FFmpeg-devel] [PATCH 2/5] libavdevice/avfoundation.m: Replace mutex-based concurrency by a thread-safe fifo queue with maximum length

2022-01-19 Thread Romain Beauxis

The existing implementation of avdevice input has issues in its concurrent 
model as it only allows for one shared frame between writing and reading 
threads. This means that, if reading thread gets late, frames get dropped, 
resulting in corrupted input.

This patch changes the concurrency logic to use a single shared queue for both 
video and audio frames. Previous version of the patch used separate queues for 
audio and video but this could cause synchronization issues.

In order to avoid dropping initial audio frames, the video configuration logic 
is also changed to assume height/width as configured when opening the input 
device so as to not depend on the first video frame for it.

--- Begin Message ---
From d1a4c6e74ff589d9e59e1310a9afc9bc185382a1 Mon Sep 17 00:00:00 2001
From: Romain Beauxis 
Date: Sun, 12 Dec 2021 17:29:27 -0600
Subject: [PATCH] libavdevice/avfoundation.m: Replace mutex-based concurrency
 handling in avfoundation.m by a thread-safe fifo queue with maximum length
X-Unsent: 1
To: ffmpeg-devel@ffmpeg.org

* Use a shared CMSimpleQueueEnqueue with maximum length to queue and process 
incoming audio and video frames.
* Simplify video configuration to avoid consuming first frame.
* Log avfoundation errors.
* Use AVERROR_EXTERNAL instead of AVERROR(EIO) in avfoundation errors.

Signed-off-by: Romain Beauxis 
---
 libavdevice/avfoundation.m | 227 +
 1 file changed, 101 insertions(+), 126 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 77c6e68763..e6f64b35b8 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
  */
 
 #import 
-#include 
+#import 
 
 #include "libavutil/channel_layout.h"
 #include "libavutil/pixdesc.h"
@@ -39,6 +39,13 @@
 #include "libavutil/imgutils.h"
 #include "avdevice.h"
 
+static void av_log_avfoundation(void *s, int lvl, const char *str, OSStatus 
err) {
+NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
+av_log(s, lvl, "AVFoundation: %s, %s\n", str,
+[[[NSError errorWithDomain:NSOSStatusErrorDomain code:err 
userInfo:nil] localizedDescription] UTF8String]);
+[pool release];
+}
+
 static const int avf_time_base = 100;
 
 static const AVRational avf_time_base_q = {
@@ -84,9 +91,6 @@
 {
 AVClass*class;
 
-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
 id  avf_delegate;
 id  avf_audio_delegate;
 
@@ -121,8 +125,9 @@
 AVCaptureSession *capture_session;
 AVCaptureVideoDataOutput *video_output;
 AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+
+CMSimpleQueueRef  frames_queue;
+int   max_frames;
 
 AVCaptureDevice  *observed_device;
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -131,16 +136,6 @@
 int  observed_quit;
 } AVFContext;
 
-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
 /** FrameReciever class - delegate for AVCaptureSession
  */
 @interface AVFFrameReceiver : NSObject
@@ -218,17 +213,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
   didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
  fromConnection:(AVCaptureConnection *)connection
 {
-lock_frames(_context);
+OSStatus ret = CMSimpleQueueEnqueue(_context->frames_queue, videoFrame);
 
-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
+if (ret != noErr) {
+  av_log_avfoundation(_context, AV_LOG_DEBUG, "Error while queueing video 
frame", ret);
 }
 
-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
 }
 
 @end
@@ -262,17 +253,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
   didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
  fromConnection:(AVCaptureConnection *)connection
 {
-lock_frames(_context);
+OSStatus ret = CMSimpleQueueEnqueue(_context->frames_queue, audioFrame);
 
-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
+if (ret != noErr) {
+  av_log_avfoundation(_context, AV_LOG_DEBUG, "Error while queueing audio 
frame", ret);
 }
 
-_context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
-
-unlock_frames(_context);
-
-++_context->audio_frames_captured;
+CFRetain(audioFrame);
 }
 
 @end
@@ -287,6 +274,19 @@ static void destroy_context(AVFContext* ctx)
 [ctx->avf_de

[FFmpeg-devel] [PATCH 1/5] libavdevice/avfoundation.m: use setAudioSettings, extend supported formats

2022-01-19 Thread Romain Beauxis

This patch switches the logic around audio settings to let the caller drive the 
format.

After experimenting with the AudioConverter, we realized that, even when 
adhering to a strict implementation of the documented API, we were still 
getting errors during conversions. The input device would randomly change from 
e.g. s32le to s24le between restarts and error out on conversion (using a 
freshly initialized converter).

Using setAudioSettings allow the OS to drive audio conversion internally and 
pick whatever appropriate settings for the audio device. This has been working 
very well and is also the way AVFoundation audio input is setup in videolan.

--- Begin Message ---
From fd30f651bdaafe812b5cdc022ef3e4ebd74b6727 Mon Sep 17 00:00:00 2001
From: Romain Beauxis 
Date: Mon, 29 Nov 2021 08:46:05 -0600
Subject: [PATCH] libavdevice/avfoundation.m: use setAudioSettings, extend
 supported formats
X-Unsent: 1
To: ffmpeg-devel@ffmpeg.org

This fixes: https://trac.ffmpeg.org/ticket/9502

Signed-off-by: Romain Beauxis 
---
 libavdevice/avfoundation.m | 206 -
 1 file changed, 63 insertions(+), 143 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..77c6e68763 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -93,6 +93,11 @@
 AVRational  framerate;
 int width, height;
 
+int channels;
+int big_endian;
+int sample_rate;
+enum AVSampleFormat sample_format;
+
 int capture_cursor;
 int capture_mouse_clicks;
 int capture_raw_data;
@@ -111,17 +116,6 @@
 
 int num_video_devices;
 
-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
-
 enum AVPixelFormat pixel_format;
 
 AVCaptureSession *capture_session;
@@ -298,14 +292,6 @@ static void destroy_context(AVFContext* ctx)
 ctx->audio_output= NULL;
 ctx->avf_delegate= NULL;
 ctx->avf_audio_delegate = NULL;
-
-av_freep(>audio_buffer);
-
-pthread_mutex_destroy(>frame_lock);
-
-if (ctx->current_frame) {
-CFRelease(ctx->current_frame);
-}
 }
 
 static void parse_device_name(AVFormatContext *s)
@@ -671,88 +657,62 @@ static int get_video_config(AVFormatContext *s)
 static int get_audio_config(AVFormatContext *s)
 {
 AVFContext *ctx = (AVFContext*)s->priv_data;
-CMFormatDescriptionRef format_desc;
-AVStream* stream = avformat_new_stream(s, NULL);
+AVStream* stream;
+int bits_per_sample, is_float;
 
-if (!stream) {
-return 1;
-}
+enum AVCodecID codec_id = av_get_pcm_codec(ctx->sample_format, 
ctx->big_endian);
 
-// Take stream info from the first frame.
-while (ctx->audio_frames_captured < 1) {
-CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
+if (codec_id == AV_CODEC_ID_NONE) {
+   av_log(ctx, AV_LOG_ERROR, "Error: invalid sample format!\n");
+   return AVERROR(EINVAL);
 }
 
-lock_frames(ctx);
-
-ctx->audio_stream_index = stream->index;
-
-avpriv_set_pts_info(stream, 64, 1, avf_time_base);
-
-format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+switch (ctx->sample_format) {
+case AV_SAMPLE_FMT_S16:
+bits_per_sample = 16;
+is_float = 0;
+break;
+case AV_SAMPLE_FMT_S32:
+bits_per_sample = 32;
+is_float = 0;
+break;
+case AV_SAMPLE_FMT_FLT:
+bits_per_sample = 32;
+is_float = 1;
+break;
+default:
+av_log(ctx, AV_LOG_ERROR, "Error: invalid sample format!\n");
+unlock_frames(ctx);
+return AVERROR(EINVAL);
+}
 
-if (!basic_desc) {
+[ctx->audio_output setAudioSettings:@{
+AVFormatIDKey:   @(kAudioFormatLinearPCM),
+AVLinearPCMBitDepthKey:  @(bits_per_sample),
+AVLinearPCMIsFloatKey:   @(is_float),
+AVLinearPCMIsBigEndianKey:   @(ctx->big_endian),
+AVNumberOfChannelsKey:   @(ctx->channels),
+AVLinearPCMIsNonInterleaved: @NO,
+AVSampleRateKey: @(ctx->sample_rate)
+}];
+
+stream = avformat_new_stream(s, NULL);
+if (!stream) {
 unlock_frames(ctx);
-av_log(s, AV_LOG_ERROR, "audio format not available\n");
-return 1;
+return -1;
 }
 
+

[FFmpeg-devel] [PATCH 0/5] macos avdevice fixes and improvements

2022-01-19 Thread Romain Beauxis

This is a follow-up from a previous series of patches that fix, enhance and
cleanup support for audio and video input on macos in libavdevice. Due to
some important recent refactoring and addition, version is reset.

Patches:
  libavdevice/avfoundation.m: use setAudioSettings, extend supported formats
  libavdevice/avfoundation.m: Replace mutex-based concurrency handling in
avfoundation.m by a thread-safe fifo queue with maximum length
  libavdevice/avfoundation.m: Allow to select devices by unique ID
  Use appropriate method for device discovery.
  Add AudioToolbox audio input device.

 configure|   5 ++
 doc/indevs.texi  |  50 +++-
 libavdevice/Makefile |   1 +
 libavdevice/alldevices.c |   1 +
 libavdevice/audiotoolbox_dec.m (new) | 466
+++
 libavdevice/avfoundation.m   | 548
-
 6 files changed, 781 insertions(+), 290 deletions(-)
 create mode 100644 libavdevice/audiotoolbox_dec.m
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v9 1/3] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2022-01-14 Thread Romain Beauxis

Le ven. 14 janv. 2022 à 11:47, Thilo Borgmann  a écrit :
>
> Am 14.01.22 um 13:57 schrieb Marvin Scholz:
> >
> >
> > On 6 Jan 2022, at 15:24, Romain Beauxis wrote:
> >
> >> * Implement support for AudioConverter
> >> * Switch to AudioConverter's API to convert unsupported PCM
> >>   formats (non-interleaved, non-packed) to supported formats
> >> * Minimize data copy.
> >>
> >> This fixes: https://trac.ffmpeg.org/ticket/9502
> >>
> >> API ref:
> >> https://developer.apple.com/documentation/audiotoolbox/audio_converter_services
> >>
> >> Signed-off-by: Romain Beauxis 
> >> ---
> >> This is the first patch of a series of 3 that fix, cleanup and enhance the
> >> avfoundation implementation for libavdevice.
> >>
> >> These patches come from an actual user-facing application relying on
> >> libavdevice’s implementation of avfoundation audio input. Without them,
> >> Avfoundation is practically unusable as it will:
> >> * Refuse to process certain specific audio input format that are actually
> >> returned by the OS for some users (packed PCM audio)
> >> * Drop audio frames, resulting in corrupted audio input. This might have 
> >> been
> >> unnoticed with video frames but this makes avfoundation essentially 
> >> unusable
> >> for audio.
> >>
> >> The patches are now being included in our production build so they are 
> >> tested
> >> and usable in production.
> >>
> >
> > Hi,
> >
> > the patches are still corrupt and do not apply.
> > As stated earlier, please either use git send-email or attach the patch
> > to the mail instead of putting its contents in it, as apparently Mail.app
> > messes them up.
>
> Still the same for me. Do you use git send-email or git format-patch?

Thanks for checking on this y'all and sorry about these complications.

I used git format-patches. I might try git send-email or the github PR
bridge, that seems like a neat trick.

I'm working on a new revision of the patches, I discovered more issues
with audio conversion, possibly linked to bugs with the AudioConverter
API.

I also discovered an API to do the conversion internally without
having to deal with manually reconverting. Hopefully, this also fixes
the issues we uncovered.

Will post once we have done more testing. All in all, macos sound APIs
are pretty confusing and buggy around the edges it seems.

Thanks again!
-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] 5.0 blocking issues

2022-01-08 Thread Romain Beauxis

Le sam. 8 janv. 2022 à 10:30, Michael Niedermayer
 a écrit :
>
> Hi all
>
> This is a simple go/no go call
> if you know of something that still should go into 5.0 please reply here
> with a list of what you are working on and a timelimit until when you
> will be done with it
>
> if you think everything is ready for the release, then too feel free to
> reply (assuming few others said ok yet) this is not supposed to become a
> 100 reply thread with oks, just maybe 2-3 people confirming that noone
> is aware of things missing.
>
> I intend to do the release within 2-3 days of all "no go" things being
> resolved or timeouting

Hi,

Not sure if that should be blocking but the series of patches starting
here: http://ffmpeg.org/pipermail/ffmpeg-devel/2022-January/290940.html
do fix avfoundation devices, which are currently unusable. It would be
nice to get it out as part of a new release.

I'm available in the coming days for quick review/changes/fixes.
-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH v8 2/3] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length

2022-01-06 Thread Romain Beauxis

Le mer. 5 janv. 2022 à 08:50, Marvin Scholz  a écrit :
>
> On 31 Dec 2021, at 18:43, Romain Beauxis wrote:
>
> > * Use a CMSimpleQueueEnqueue with maximum length to queue and process
> > incoming audio and video frames.
> > * Log avfoundation errors.
> > * Use AVERROR_EXTERNAL instead of AVERROR(EIO) in avfoundation errors.
> >
> > Signed-off-by: Romain Beauxis 
> > —
> > [Sorry for the noise but an issue came up with the previous set]
> >
> > This is the second patch of a series of 3 that fix, cleanup and
> > enhance the
> > avfoundation implementation for libavdevice.
> >
> > These patches come from an actual user-facing application relying on
> > libavdevice’s implementation of avfoundation audio input. Without
> > them,
> > Avfoundation is practically unusable as it will:
> > * Refuse to process certain specific audio input format that are
> > actually
> > returned by the OS for some users (packed PCM audio)
> > * Drop audio frames, resulting in corrupted audio input. This might
> > have been
> > unnoticed with video frames but this makes avfoundation essentially
> > unusable
> > for audio.
> >
> > The patches are now being included in our production build so they are
> > tested
> > and usable in production.
> >
> > Changelog for this patch:
> > * v2: None
> > * v3: None
> > * v4: None
> > * v5: Fix indentation/wrapping
> > * v6: None
> > * v7: Removed use of kAudioConverterPropertyCalculateOutputBufferSize
> > to calculate output buffer size. The calculation is trivial and this
> > call was
> > randomly failing for no reason
> > * v8: Fix memory leak when video or audio queue is full
> >
> > libavdevice/avfoundation.m | 194 +++--
> > 1 file changed, 100 insertions(+), 94 deletions(-)
> >
> > diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
> > index 738cd93375..36f9fdc53d 100644
> > --- a/libavdevice/avfoundation.m
> > +++ b/libavdevice/avfoundation.m
> > @@ -26,7 +26,7 @@
> >  */
> >
> > #import 
> > -#include 
> > +#import 
> >
> > #include "libavutil/channel_layout.h"
> > #include "libavutil/pixdesc.h"
> > @@ -39,6 +39,11 @@
> > #include "libavutil/imgutils.h"
> > #include "avdevice.h"
> >
> > +#define av_log_avfoundation_error(s, str, err) \
> > +   av_log(s, AV_LOG_ERROR, "Avfoundation: %s, %s\n", str, \
>
> nitpick: should probably be AVFoundation, no?

Done!

> > + [[[NSError errorWithDomain:NSOSStatusErrorDomain code:err
> > userInfo:nil] localizedDescription] UTF8String] \
> > +  )
> > +
>
> The errorWithDomain: returns an autorelease NSError, however there is no
> autorelease pool.
> Either make this a function with an @autorelease pool or use [[…
> alloc] init…] instead, and
> release the NSError.

That's right, thanks for pointing that out. Just sent a v9 version of
the patchset fixing that and also dropping the log level for the error
returned when the queue is full. In practice, the queue can become
full pretty often, for instance when waiting on a remote output
connection so these logs end up polluting the output in situations
that are not really problematic.

Thanks for looking into this!
-- Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

[FFmpeg-devel] [PATCH v9 3/3] libavdevice/avfoundation.m: Allow to select devices by unique ID

2022-01-06 Thread Romain Beauxis

Signed-off-by: Romain Beauxis 
---
This is the third patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
unnoticed with video frames but this makes avfoundation essentially unusable
for audio.

The patches are now being included in our production build so they are tested
and usable in production.

Changes:
v2: None
v3:
 * Switched unique ID to use system-prodvided unique ID
 * Implemented unique IDs for screen capture
v4: Cleanup
v5: Fix indentation/wrapping
v6: None
v7: None
V8: None
v9: None

doc/indevs.texi|  6 ++--
libavdevice/avfoundation.m | 72 +-
2 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 9d8020311a..858c0fa4e4 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
-i "[[VIDEO]:[AUDIO]]"
@end example
The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or ID as shown by the 
device list.
Alternatively, the video and/or audio input device can be chosen by index using 
the
@option{
-video_device_index 
@@ -127,7 +127,9 @@ and/or
device name or index given in the input filename.

All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name 
can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.

There are two device name aliases:
@table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 5ee19f4863..c4a4272965 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -46,6 +46,8 @@ static inline void av_log_avfoundation(void *s, int lvl, 
const char *str, OSStat
  [pool release];
}

+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" 
withString:@"."] UTF8String]
+
static const int avf_time_base = 100;

static const AVRational avf_time_base_q = {
@@ -817,21 +819,23 @@ static int avf_read_header(AVFormatContext *s)
int index = 0;
av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [devices_muxed 
indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices count] + [devices_muxed 
indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
if (num_screens > 0) {
CGDirectDisplayID screens[num_screens];
CGGetActiveDisplayList(num_screens, screens, _screens);
for (int i = 0; i < num_screens; i++) {
-av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", 
ctx->num_video_devices + i, i);
+av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d (ID: 
AvfilterAvfoundationCaptureScreen%d)\n", ctx->num_video_devices + i, i, 
screens[i]);
}
}
#endif
@@ -839,9 +843,10 @@ static int avf_read_header(AVFormatContext *s)
av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
devices = [AVCaptureDevice devicesWithMediaType:AVM

[FFmpeg-devel] [PATCH v9 2/3] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length

2022-01-06 Thread Romain Beauxis

* Use a CMSimpleQueueEnqueue with maximum length to queue and process incoming 
audio and video frames.
* Log avfoundation errors.
* Use AVERROR_EXTERNAL instead of AVERROR(EIO) in avfoundation errors.

Signed-off-by: Romain Beauxis 
---
This is the first patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
unnoticed with video frames but this makes avfoundation essentially unusable
for audio.

The patches are now being included in our production build so they are tested
and usable in production.

Changes:
* v2: None
* v3: Switched queue implementation to CMSimpleQueue
* v4: None
* v5: Fix indentation/wrapping
* v6: Fix audio/video frame queue cleanup logic
* v7: Enhance avfoundation error reporting: add human-readable description,
 use AVERROR_EXTERNAL instead of AVERROR(EIO)
* v8: Fix memory leak when video or audio queue is full
* v9: Use auto-release pool for av_log_avfoundation, make it an inline
 function, drop log level to AV_LOG_DEBUG when frame queueing fails
 as it is quite likely to happen while setting up a processing
 pipeline. 

libavdevice/avfoundation.m | 196 +++--
1 file changed, 102 insertions(+), 94 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 738cd93375..5ee19f4863 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
 */

#import 
-#include 
+#import 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -39,6 +39,13 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+static inline void av_log_avfoundation(void *s, int lvl, const char *str, 
OSStatus err) {
+  NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
+  NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:err 
userInfo:nil];
+  av_log(s, lvl, "AVFoundation: %s, %s\n", str, [[error localizedDescription] 
UTF8String]);
+  [pool release];
+}
+
static const int avf_time_base = 100;

static const AVRational avf_time_base_q = {
@@ -80,13 +87,12 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define MAX_QUEUED_FRAMES 10
+
typedef struct
{
AVClass*class;

-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
id  avf_delegate;
id  avf_audio_delegate;

@@ -122,8 +128,8 @@
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+CMSimpleQueueRef  audio_frames_queue;
+CMSimpleQueueRef  video_frames_queue;

AVCaptureDevice  *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -132,16 +138,6 @@
int  observed_quit;
} AVFContext;

-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
@@ -219,17 +215,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
+OSStatus ret = CMSimpleQueueEnqueue(_context->video_frames_queue, 
videoFrame);

-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
+if (ret != noErr) {
+  av_log_avfoundation(_context, AV_LOG_DEBUG, "Error while queueing video 
frame", ret);
}

-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
}

@end
@@ -263,17 +255,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
+OSStatus ret = CMSimpleQueueEnqueue(_context->audio_frames_queue, 
audioFrame);

-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
+if (ret != noErr) {
+  av_log_avfoundation(_context, AV_LOG_DEBUG, "Error while queueing audio 
frame", ret);
}

-_context->current_audio_frame =

[FFmpeg-devel] [PATCH v9 1/3] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2022-01-06 Thread Romain Beauxis

* Implement support for AudioConverter
* Switch to AudioConverter's API to convert unsupported PCM
  formats (non-interleaved, non-packed) to supported formats
* Minimize data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref:
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services

Signed-off-by: Romain Beauxis 
---
This is the first patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
unnoticed with video frames but this makes avfoundation essentially unusable
for audio.

The patches are now being included in our production build so they are tested
and usable in production.

Changelog for this patch:
* v2: None
* v3: None
* v4: None
* v5: Fix indentation/wrapping
* v6: None
* v7: Removed use of kAudioConverterPropertyCalculateOutputBufferSize
to calculate output buffer size. The calculation is trivial and this call was 
randomly failing for no reason
* v8: None
* v9: None

libavdevice/avfoundation.m | 255 +
1 file changed, 145 insertions(+), 110 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..738cd93375 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,11 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32input_bytes_per_sample;
+UInt32output_bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +294,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +671,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +692,97 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;
-ctx->audio_packed  = basic_desc->mFormatFlags &

[FFmpeg-devel] [PATCH v8 3/3] libavdevice/avfoundation.m: Allow to select devices by unique ID

2021-12-31 Thread Romain Beauxis

Signed-off-by: Romain Beauxis 
—
[Sorry for the noise but an issue came up with the previous set]

This is the third patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
unnoticed with video frames but this makes avfoundation essentially unusable
for audio.

The patches are now being included in our production build so they are tested
and usable in production

Changes:
v2: None
v3:
  * Switched unique ID to use system-prodvided unique ID
  * Implemented unique IDs for screen capture
v4: Cleanup
v5: Fix indentation/wrapping
v6: None
v7: None
V8: None

This patch adds a unique ID to avfoundation devices. This is needed
because device index can change while the machine is running when
devices are plugged or unplugged and device names can be tricky to use
with localization and etc.

Example of output:
./ffmpeg -f avfoundation -list_devices true -i ""
[...]
[AVFoundation indev @ 0x158705230] AVFoundation video devices:
[AVFoundation indev @ 0x158705230] [0] FaceTime HD Camera (ID: 
47B4B64B70674B9CAD2BAE273A71F4B5)
[AVFoundation indev @ 0x158705230] [1] Capture screen 0 (ID: 
AvfilterAvfoundationCaptureScreen1)
[AVFoundation indev @ 0x158705230] AVFoundation audio devices:
[AVFoundation indev @ 0x158705230] [0] Loopback Audio (ID: 
com.rogueamoeba.Loopback.A5668B36-711E-4DF5-8A8D-7148508C735B)
[AVFoundation indev @ 0x158705230] [1] MacBook Pro Microphone 
(ID:BuiltInMicrophoneDevice)

Notes:
* Unique names do not seem to follow any specific pattern. I have used
one similar to the builtin microphone for screen capture
* The : substitution is actually required. The loopback device above did
have it in its name.

doc/indevs.texi|  6 ++--
libavdevice/avfoundation.m | 72 +-
2 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 9d8020311a..858c0fa4e4 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
-i "[[VIDEO]:[AUDIO]]"
@end example
The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or ID as shown by the 
device list.
Alternatively, the video and/or audio input device can be chosen by index using 
the
@option{
-video_device_index 
@@ -127,7 +127,9 @@ and/or
device name or index given in the input filename.

All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name 
can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.

There are two device name aliases:
@table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 36f9fdc53d..d09a81cb3b 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -39,6 +39,8 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" 
withString:@"."] UTF8String]
+
#define av_log_avfoundation_error(s, str, err) \
   av_log(s, AV_LOG_ERROR, "Avfoundation: %s, %s\n", str, \
 [[[NSError errorWithDomain:NSOSStatusErrorDomain code:err userInfo:nil] 
localizedDescription] UTF8String] \
@@ -815,21 +817,23 @@ static int avf_read_header(AVFormatContext *s)
int index = 0;
av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [

[FFmpeg-devel] [PATCH v8 2/3] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length

2021-12-31 Thread Romain Beauxis

* Use a CMSimpleQueueEnqueue with maximum length to queue and process incoming 
audio and video frames.
* Log avfoundation errors.
* Use AVERROR_EXTERNAL instead of AVERROR(EIO) in avfoundation errors.

Signed-off-by: Romain Beauxis 
—
[Sorry for the noise but an issue came up with the previous set]

This is the second patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
unnoticed with video frames but this makes avfoundation essentially unusable
for audio.

The patches are now being included in our production build so they are tested
and usable in production.

Changelog for this patch:
* v2: None
* v3: None
* v4: None
* v5: Fix indentation/wrapping
* v6: None
* v7: Removed use of kAudioConverterPropertyCalculateOutputBufferSize
to calculate output buffer size. The calculation is trivial and this call was 
randomly failing for no reason
* v8: Fix memory leak when video or audio queue is full

libavdevice/avfoundation.m | 194 +++--
1 file changed, 100 insertions(+), 94 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 738cd93375..36f9fdc53d 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
 */

#import 
-#include 
+#import 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -39,6 +39,11 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+#define av_log_avfoundation_error(s, str, err) \
+   av_log(s, AV_LOG_ERROR, "Avfoundation: %s, %s\n", str, \
+ [[[NSError errorWithDomain:NSOSStatusErrorDomain code:err userInfo:nil] 
localizedDescription] UTF8String] \
+  )
+
static const int avf_time_base = 100;

static const AVRational avf_time_base_q = {
@@ -80,13 +85,12 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define MAX_QUEUED_FRAMES 10
+
typedef struct
{
AVClass*class;

-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
id  avf_delegate;
id  avf_audio_delegate;

@@ -122,8 +126,8 @@
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+CMSimpleQueueRef  audio_frames_queue;
+CMSimpleQueueRef  video_frames_queue;

AVCaptureDevice  *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -132,16 +136,6 @@
int  observed_quit;
} AVFContext;

-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
@@ -219,17 +213,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
+OSStatus ret = CMSimpleQueueEnqueue(_context->video_frames_queue, 
videoFrame);

-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
+if (ret != noErr) {
+  av_log_avfoundation_error(_context, "Error while queueing video frame", 
ret);
}

-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
}

@end
@@ -263,17 +253,13 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
+OSStatus ret = CMSimpleQueueEnqueue(_context->audio_frames_queue, 
audioFrame);

-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
+if (ret != noErr) {
+  av_log_avfoundation_error(_context, "Error while queueing audio frame", 
ret);
}

-_context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
-
-unlock_frames(_context);
-
-++_context->audio_frames_captured;
+CFRetain(audioFrame);
}

@end
@@ -288,6 +274,30 @@ static void destroy_context(AVFContext* ctx)
[ctx->avf_delegaterelease];
[ctx->avf_audio_delegate release];

+CMSampleBufferRef fra

[FFmpeg-devel] [PATCH v8 1/3] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-31 Thread Romain Beauxis

* Implement support for AudioConverter
* Switch to AudioConverter's API to convert unsupported PCM
  formats (non-interleaved, non-packed) to supported formats
* Minimize data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref:
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services

Signed-off-by: Romain Beauxis 
—
[Sorry for the noise but an issue came up with the previous set]

This is the first patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
unnoticed with video frames but this makes avfoundation essentially unusable
for audio.

The patches are now being included in our production build so they are tested
and usable in production.

Changelog for this patch:
* v2: None
* v3: None
* v4: None
* v5: Fix indentation/wrapping
* v6: None
* v7: Removed use of kAudioConverterPropertyCalculateOutputBufferSize
 to calculate output buffer size. The calculation is trivial and this call was 
 randomly failing for no reason
* v8: None


libavdevice/avfoundation.m | 255 +
1 file changed, 145 insertions(+), 110 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..738cd93375 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,11 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32input_bytes_per_sample;
+UInt32output_bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +294,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +671,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +692,97 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;

[FFmpeg-devel] [PATCH v7 3/3] libavdevice/avfoundation.m: Allow to select devices by unique ID

2021-12-31 Thread Romain Beauxis

Signed-off-by: Romain Beauxis 
—
This is the third patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
v2: None
v3:
   * Switched unique ID to use system-prodvided unique ID
   * Implemented unique IDs for screen capture
v4: Cleanup
v5: Fix indentation/wrapping
v6: None
v7: None

This patch adds a unique ID to avfoundation devices. This is needed
because device index can change while the machine is running when
devices are plugged or unplugged and device names can be tricky to use
with localization and etc.

Example of output:
./ffmpeg -f avfoundation -list_devices true -i ""
[...]
[AVFoundation indev @ 0x158705230] AVFoundation video devices:
[AVFoundation indev @ 0x158705230] [0] FaceTime HD Camera (ID: 
47B4B64B70674B9CAD2BAE273A71F4B5)
[AVFoundation indev @ 0x158705230] [1] Capture screen 0 (ID: 
AvfilterAvfoundationCaptureScreen1)
[AVFoundation indev @ 0x158705230] AVFoundation audio devices:
[AVFoundation indev @ 0x158705230] [0] Loopback Audio (ID: 
com.rogueamoeba.Loopback.A5668B36-711E-4DF5-8A8D-7148508C735B)
[AVFoundation indev @ 0x158705230] [1] MacBook Pro Microphone 
(ID:BuiltInMicrophoneDevice)

Notes:
* Unique names do not seem to follow any specific pattern. I have used
one similar to the builtin microphone for screen capture
* The : substitution is actually required. The loopback device above did
have it in its name.

doc/indevs.texi|  6 ++--
libavdevice/avfoundation.m | 72 +-
2 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 9d8020311a..858c0fa4e4 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
-i "[[VIDEO]:[AUDIO]]"
@end example
The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or ID as shown by the 
device list.
Alternatively, the video and/or audio input device can be chosen by index using 
the
@option{
-video_device_index 
@@ -127,7 +127,9 @@ and/or
device name or index given in the input filename.

All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name 
can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.

There are two device name aliases:
@table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index c740745fce..bad0cd7155 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -39,6 +39,8 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" 
withString:@"."] UTF8String]
+
#define av_log_avfoundation_error(str, err) \
   av_log(s, AV_LOG_ERROR, "Avfoundation: %s, %s\n", str, \
 [[[NSError errorWithDomain:NSOSStatusErrorDomain code:err userInfo:nil] 
localizedDescription] UTF8String] \
@@ -805,21 +807,23 @@ static int avf_read_header(AVFormatContext *s)
int index = 0;
av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [devices_muxed 
indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices count] + [devices_muxed 
indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
if (num_screens > 0) {
CGDirectDisplayID screens[num_screens];
CGGetActiveDisplayList(num_screens, screens, _sc

[FFmpeg-devel] [PATCH v7 2/3] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length

2021-12-31 Thread Romain Beauxis

* Use a CMSimpleQueueEnqueue with maximum length to queue and process incoming 
audio and video frames.
* Log avfoundation errors.
* Use AVERROR_EXTERNAL instead of AVERROR(EIO) in avfoundation errors.

Signed-off-by: Romain Beauxis 
—
This is the second patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
unnoticed with video frames but this makes avfoundation essentially unusable
for audio.

The patches are now being included in our production build so they are tested
and usable in production.

This patch fixes the concurrency model. Avfoundation runs its own
producing thread to send produced frames and ffmpeg runs its own thread
to consume them.

The existing implementation stores the last transmitted frame and uses a
mutex to avoid concurrent access. However, this leads to situations
where upcoming frames can be dropped if the ffmpeg thread is acessing
the latest frame. This happens even when the thread would otherwise
catch up and process frames fast enought.

This patches changes this implementation to use a buffer queue with a
max queue length and encapsulated thread-safety. This greatly simplifies
the logic of the calling code and gives the consuming thread a chance to
process all frames concurrently to the producing thread while avoiding
memory leaks.

Changes:
v2: None
v3: Switched queue implementation to CMSimpleQueue
v4: None
v5: Fix indentation/wrapping
v6: Fix audio/video frame queue cleanup logic
v7: Enhance avfoundation error reporting: add human-readable description,
  use AVERROR_EXTERNAL instead of AVERROR(EIO)

libavdevice/avfoundation.m | 196 ++---
1 file changed, 96 insertions(+), 100 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 738cd93375..c740745fce 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
 */

#import 
-#include 
+#import 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -39,6 +39,11 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+#define av_log_avfoundation_error(str, err) \
+   av_log(s, AV_LOG_ERROR, "Avfoundation: %s, %s\n", str, \
+ [[[NSError errorWithDomain:NSOSStatusErrorDomain code:err userInfo:nil] 
localizedDescription] UTF8String] \
+  )
+
static const int avf_time_base = 100;

static const AVRational avf_time_base_q = {
@@ -80,13 +85,12 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define MAX_QUEUED_FRAMES 10
+
typedef struct
{
AVClass*class;

-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
id  avf_delegate;
id  avf_audio_delegate;

@@ -122,8 +126,8 @@
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+CMSimpleQueueRef  audio_frames_queue;
+CMSimpleQueueRef  video_frames_queue;

AVCaptureDevice  *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -132,16 +136,6 @@
int  observed_quit;
} AVFContext;

-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
@@ -219,17 +213,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
-}
-
-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
+CMSimpleQueueEnqueue(_context->video_frames_queue, videoFrame);
}

@end
@@ -263,17 +248,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
-}
-
-_context->current_audio_frame =

[FFmpeg-devel] [PATCH v7 1/3] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-31 Thread Romain Beauxis

* Implement support for AudioConverter
* Switch to AudioConverter's API to convert unsupported PCM
  formats (non-interleaved, non-packed) to supported formats
* Minimize data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref:
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services

Signed-off-by: Romain Beauxis 
—

This is the first patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
 returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
 unnoticed with video frames but this makes avfoundation essentially unusable
 for audio.

The patches are now being included in our production build so they are tested
and usable in production.

Changelog for this patch:
* v2: None
* v3: None
* v4: None
* v5: Fix indentation/wrapping
* v6: None
* v7: Removed use of kAudioConverterPropertyCalculateOutputBufferSize
  to calculate output buffer size. The calculation is trivial and this call was 
  randomly failing for no reason

libavdevice/avfoundation.m | 255 +
1 file changed, 145 insertions(+), 110 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..738cd93375 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,11 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32input_bytes_per_sample;
+UInt32output_bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +294,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +671,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +692,97 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;
-ctx->audio_packed  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsPacke

Re: [FFmpeg-devel] [PATCH v6 01/03] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-31 Thread Romain Beauxis



> On Dec 28, 2021, at 6:54 PM, Aman Karmani  wrote:
> 
> 
> 
> On Tue, Dec 28, 2021 at 2:50 PM Romain Beauxis  wrote:
> This is the first patch of a series of 3 that fix, cleanup and enhance the
> avfoundation implementation for libavdevice.
> 
> The patches have been submitted a couple of times now and have
> received very nice feedback for the last two however but they do not seem
> to have been considered for inclusion thus far. 
> 
> These patches come from an actual user-facing application relying on
> libavdevice’s implementation of avfoundation audio input. Without them,
> Avfoundation is practically unusable as it will:
> * Refuse to process certain specific audio input format that are actually
>   returned by the OS for some users (packed PCM audio)
> * Drop audio frames, resulting in corrupted audio input. This might have been
>   unnoticed with video frames but this makes avfoundation essentially unusable
>   for audio.
> 
> The patches are now being included in our production build so they are tested
> and usable in production.
> 
> So, this bares the question: is avfoundation still supported and actively 
> maintained
> in libavdevice? It feels that such important bugs should have been noticed by 
> now
> and also generated a little more interest in fixing them.
> 
> Thanks for working on this, and addressing all the feedback so far.
> 
> The patchset LGTM, and I think it should be applied.
> 
> Looks like MAINTAINERS lists Thilo for avfoundation.m. I'm not sure if he's 
> seen this yet, so I'm cc'ing on this reply.
> 
> If we don't hear in the next couple weeks, I can apply these changes.


Thank you, this is much appreciated!

We discovered a bug in the audio converter patch, I’m posting a new updated 
series right away & will CC everyone here.

Thanks!

> 
> 
> Thanks for y’all feedback!
> — Romain
> -
> 
> Changes:
> * v2: None
> * v3: None
> * v4: None
> * v5: Fix indentation/wrapping
> * v6: None
> 
> * Implement support for AudioConverter
> * Switch to AudioConverter's API to convert unsupported PCM
>   formats (non-interleaved, non-packed) to supported formats
> * Minimize data copy.
> 
> This fixes: https://trac.ffmpeg.org/ticket/9502
> 
> API ref:
> https://developer.apple.com/documentation/audiotoolbox/audio_converter_services
> 
> Signed-off-by: Romain Beauxis 
> ---
> libavdevice/avfoundation.m | 250 +
> 1 file changed, 144 insertions(+), 106 deletions(-)
> 
> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
> index 0cd6e646d5..79c9207cfa 100644
> --- a/libavdevice/avfoundation.m
> +++ b/libavdevice/avfoundation.m
> @@ -111,16 +111,10 @@
> 
> int num_video_devices;
> 
> -int audio_channels;
> -int audio_bits_per_sample;
> -int audio_float;
> -int audio_be;
> -int audio_signed_integer;
> -int audio_packed;
> -int audio_non_interleaved;
> -
> -int32_t *audio_buffer;
> -int audio_buffer_size;
> +UInt32audio_buffers;
> +UInt32audio_channels;
> +UInt32bytes_per_sample;
> +AudioConverterRef audio_converter;
> 
> enum AVPixelFormat pixel_format;
> 
> @@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
> ctx->avf_delegate= NULL;
> ctx->avf_audio_delegate = NULL;
> 
> -av_freep(>audio_buffer);
> +if (ctx->audio_converter) {
> +  AudioConverterDispose(ctx->audio_converter);
> +  ctx->audio_converter = NULL;
> +}
> 
> pthread_mutex_destroy(>frame_lock);
> 
> @@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
> AVFContext *ctx = (AVFContext*)s->priv_data;
> CMFormatDescriptionRef format_desc;
> AVStream* stream = avformat_new_stream(s, NULL);
> +AudioStreamBasicDescription output_format = {0};
> +int audio_bits_per_sample, audio_float, audio_be;
> +int audio_signed_integer, audio_packed, audio_non_interleaved;
> +int must_convert = 0;
> 
> if (!stream) {
> return 1;
> @@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
> avpriv_set_pts_info(stream, 64, 1, avf_time_base);
> 
> format_desc = 
> CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
> -const AudioStreamBasicDescription *basic_desc = 
> CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
> +const AudioStreamBasicDescription *input_format = 
> CMAudioFormatDescriptionGetStreamBasicDescription(format

[FFmpeg-devel] [PATCH v6 03/03] libavdevice/avfoundation.m: Allow to select devices by unique ID

2021-12-28 Thread Romain Beauxis

This is the third patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
v2: None
v3:
 * Switched unique ID to use system-prodvided unique ID
 * Implemented unique IDs for screen capture
v4: Cleanup
v5: Fix indentation/wrapping
V6: None

This patch adds a unique ID to avfoundation devices. This is needed
because device index can change while the machine is running when
devices are plugged or unplugged and device names can be tricky to use
with localization and etc.

Example of output:
./ffmpeg -f avfoundation -list_devices true -i ""
[...]
[AVFoundation indev @ 0x158705230] AVFoundation video devices:
[AVFoundation indev @ 0x158705230] [0] FaceTime HD Camera (ID: 
47B4B64B70674B9CAD2BAE273A71F4B5)
[AVFoundation indev @ 0x158705230] [1] Capture screen 0 (ID: 
AvfilterAvfoundationCaptureScreen1)
[AVFoundation indev @ 0x158705230] AVFoundation audio devices:
[AVFoundation indev @ 0x158705230] [0] Loopback Audio (ID: 
com.rogueamoeba.Loopback.A5668B36-711E-4DF5-8A8D-7148508C735B)
[AVFoundation indev @ 0x158705230] [1] MacBook Pro Microphone 
(ID:BuiltInMicrophoneDevice)

Notes:
* Unique names do not seem to follow any specific pattern. I have used
one similar to the builtin microphone for screen capture
* The : substitution is actually required. The loopback device above did
have it in its name.

Signed-off-by: Romain Beauxis 
---
doc/indevs.texi|  6 ++--
libavdevice/avfoundation.m | 72 +-
2 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 9d8020311a..858c0fa4e4 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
-i "[[VIDEO]:[AUDIO]]"
@end example
The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or ID as shown by the 
device list.
Alternatively, the video and/or audio input device can be chosen by index using 
the
@option{
   -video_device_index 
@@ -127,7 +127,9 @@ and/or
device name or index given in the input filename.

All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name 
can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.

There are two device name aliases:
@table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index f7bd5be404..fda5a4d261 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -39,6 +39,8 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" 
withString:@"."] UTF8String]
+
static const int avf_time_base = 100;

static const AVRational avf_time_base_q = {
@@ -797,21 +799,23 @@ static int avf_read_header(AVFormatContext *s)
   int index = 0;
   av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
   for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
   }
   for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [devices_muxed 
indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices count] + [devices_muxed 
indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
   }
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
   if (num_screens > 0) {
   CGDirectDisplayID screens[num_screens];
   CGGetActiveDisplayList(num_screens, screens, _screens);
   for (int i = 0; i < num_screens; i++) {
-av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", 
ctx->num

[FFmpeg-devel] [PATCH v6 02/03] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length

2021-12-28 Thread Romain Beauxis

This is the second patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
v2: None
v3: Switched queue implementation to CMSimpleQueue
v4: None
v5: Fix indentation/wrapping
V6: Fix audio/video frame queue cleanup logic

This patch fixes the concurrency model. Avfoundation runs its own
producing thread to send produced frames and ffmpeg runs its own thread
to consume them.

The existing implementation stores the last transmitted frame and uses a
mutex to avoid concurrent access. However, this leads to situations
where upcoming frames can be dropped if the ffmpeg thread is acessing
the latest frame. This happens even when the thread would otherwise
catch up and process frames fast enought.

This patches changes this implementation to use a buffer queue with a
max queue length and encapsulated thread-safety. This greatly simplifies
the logic of the calling code and gives the consuming thread a chance to
process all frames concurrently to the producing thread while avoiding
memory leaks.

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 169 +
1 file changed, 76 insertions(+), 93 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 79c9207cfa..f7bd5be404 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
 */

#import 
-#include 
+#import 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -80,13 +80,12 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define MAX_QUEUED_FRAMES 10
+
typedef struct
{
AVClass*class;

-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
id  avf_delegate;
id  avf_audio_delegate;

@@ -121,8 +120,8 @@
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+CMSimpleQueueRef  audio_frames_queue;
+CMSimpleQueueRef  video_frames_queue;

AVCaptureDevice  *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -131,16 +130,6 @@
int  observed_quit;
} AVFContext;

-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
@@ -218,17 +207,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
-}
-
-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
+CMSimpleQueueEnqueue(_context->video_frames_queue, videoFrame);
}

@end
@@ -262,17 +242,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
-}
-
-_context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
-
-unlock_frames(_context);
-
-++_context->audio_frames_captured;
+CFRetain(audioFrame);
+CMSimpleQueueEnqueue(_context->audio_frames_queue, audioFrame);
}

@end
@@ -287,6 +258,30 @@ static void destroy_context(AVFContext* ctx)
[ctx->avf_delegaterelease];
[ctx->avf_audio_delegate release];

+CMSampleBufferRef frame;
+
+if (ctx->video_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);
+while (frame) {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);
+}
+
+CFRelease(ctx->video_frames_queue);
+ctx->video_frames_queue = NULL;
+}
+
+if (ctx->audio_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);
+while (frame) {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);
+}
+
+CFRelease(ctx->audio_frames_queue);
+ctx->audio_frames_queue = NULL;
+}
+
ctx->capture_session = NULL;
ctx->video_output= NULL;
ctx->audio_output= NULL;
@@ -297,12 +292,6 @@ static void destroy_context(AVFContext* ctx)
  Au

[FFmpeg-devel] [PATCH v6 01/03] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-28 Thread Romain Beauxis

This is the first patch of a series of 3 that fix, cleanup and enhance the
avfoundation implementation for libavdevice.

The patches have been submitted a couple of times now and have
received very nice feedback for the last two however but they do not seem
to have been considered for inclusion thus far. 

These patches come from an actual user-facing application relying on
libavdevice’s implementation of avfoundation audio input. Without them,
Avfoundation is practically unusable as it will:
* Refuse to process certain specific audio input format that are actually
  returned by the OS for some users (packed PCM audio)
* Drop audio frames, resulting in corrupted audio input. This might have been
  unnoticed with video frames but this makes avfoundation essentially unusable
  for audio.

The patches are now being included in our production build so they are tested
and usable in production.

So, this bares the question: is avfoundation still supported and actively 
maintained
in libavdevice? It feels that such important bugs should have been noticed by 
now
and also generated a little more interest in fixing them.

Thanks for y’all feedback!
— Romain
-

Changes:
* v2: None
* v3: None
* v4: None
* v5: Fix indentation/wrapping
* v6: None

* Implement support for AudioConverter
* Switch to AudioConverter's API to convert unsupported PCM
  formats (non-interleaved, non-packed) to supported formats
* Minimize data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref:
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 250 +
1 file changed, 144 insertions(+), 106 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..79c9207cfa 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,10 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioForm

Re: [FFmpeg-devel] [PATCH v4 3/3] libavdevice/avfoundation.m: Allow to select devices by unique ID.

2021-12-18 Thread Romain Beauxis



> On Dec 17, 2021, at 3:51 PM, Marvin Scholz  wrote:
> 
> 
> 
> On 17 Dec 2021, at 16:12, Romain Beauxis wrote:
> 
>> This is the third patch of a series of 3 that cleanup and enhance the
>> avfoundation implementation for libavdevice.
>> 
>> Changes:
>> v2: None
>> v3:
>>  * Switched unique ID to use system-prodvided unique ID
>>  * Implemented unique IDs for screen capture
>> v4: Cleanup
>> 
>> This patch adds a unique ID to avfoundation devices. This is needed
>> because device index can change while the machine is running when
>> devices are plugged or unplugged and device names can be tricky to use
>> with localization and etc.
>> 
>> Example of output:
>> ./ffmpeg -f avfoundation -list_devices true -i ""
>> [...]
>> [AVFoundation indev @ 0x158705230] AVFoundation video devices:
>> [AVFoundation indev @ 0x158705230] [0] FaceTime HD Camera (ID:
>> 47B4B64B70674B9CAD2BAE273A71F4B5)
>> [AVFoundation indev @ 0x158705230] [1] Capture screen 0 (ID:
>> AvfilterAvfoundationCaptureScreen1)
>> [AVFoundation indev @ 0x158705230] AVFoundation audio devices:
>> [AVFoundation indev @ 0x158705230] [0] Loopback Audio (ID:
>> com.rogueamoeba.Loopback.A5668B36-711E-4DF5-8A8D-7148508C735B)
>> [AVFoundation indev @ 0x158705230] [1] MacBook Pro Microphone (ID:
>> BuiltInMicrophoneDevice)
>> 
>> Notes:
>> * Unique names do not seem to follow any specific pattern. I have used
>> one similar to the builtin microphone for screen capture
>> * The : substitution is actually required. The loopback device above did
>> have it in its name.
>> 
> 
> Is there no way to escape the : in the command so that we would not
> need to mess with the ID the system gives us?
> And if we need to, it would be ideal to have a fully reversible way of
> doing so, as then you could just reverse the mangling and use
> `deviceWithUniqueID:` instead of iterating all devices.

I was just thinking about this. while that would indeed be possible and a great 
approach, I realize now that because the implementation needs to otherwise 
match the filename if it is used, we would still need to iterate through all 
the devices. The current code, while annoying, at least does one loop to catch 
both use of filename and unique ID.

> That said, if thats not easily doable I am fine with the patch as-is,
> aside from the minor comments below, thanks for your work on this.

Great thanks and thanks for the great reviews! Are you interested just in this 
patch? In which case, should I repost it rebased on the latest main instead of 
the previous patch?

> 
>> Signed-off-by: Romain Beauxis 
>> ---
>> doc/indevs.texi|  6 ++--
>> libavdevice/avfoundation.m | 72 +-
>> 2 files changed, 60 insertions(+), 18 deletions(-)
>> 
>> diff --git a/doc/indevs.texi b/doc/indevs.texi
>> index 5be647f70a..2b55399c8c 100644
>> --- a/doc/indevs.texi
>> +++ b/doc/indevs.texi
>> @@ -114,7 +114,7 @@ The input filename has to be given in the following 
>> syntax:
>> -i "[[VIDEO]:[AUDIO]]"
>> @end example
>> The first entry selects the video input while the latter selects the audio 
>> input.
>> -The stream has to be specified by the device name or the device index as 
>> shown by the device list.
>> +The stream has to be specified by the device name, index or ID as shown by 
>> the device list.
>> Alternatively, the video and/or audio input device can be chosen by index 
>> using the
>> @option{
>> -video_device_index 
>> @@ -127,7 +127,9 @@ and/or
>> device name or index given in the input filename.
>>  All available devices can be enumerated by using @option{-list_devices 
>> true}, listing
>> -all device names and corresponding indices.
>> +all device names, corresponding indices and IDs, when available. Device 
>> name can be +tricky to use when localized and device index can change when 
>> devices are plugged or unplugged. A device
>> +hash, when available, uniquely identifies a device and should not change 
>> over time.
> 
> This should say ID I think, as hash was never mentioned before.

ACK

>>  There are two device name aliases:
>> @table @code
>> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
>> index b602cfbe95..25286507d6 100644
>> --- a/libavdevice/avfoundation.m
>> +++ b/libavdevice/avfoundation.m
>> @@ -39,6 +39,8 @@
>> #include "libavutil/imgutils.h"
>> #include "avdevice.h"
>> +#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrences

[FFmpeg-devel] [PATCH v5 3/3] libavdevice/avfoundation.m: Allow to select devices by unique ID

2021-12-17 Thread Romain Beauxis

(Apologies for the previously mangled message, I’m still honing my tools here!)

This is the third patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
v2: None
v3:
  * Switched unique ID to use system-prodvided unique ID
  * Implemented unique IDs for screen capture
v4: Cleanup
v5: Fix indentation/wrapping

This patch adds a unique ID to avfoundation devices. This is needed
because device index can change while the machine is running when
devices are plugged or unplugged and device names can be tricky to use
with localization and etc.

Example of output:
./ffmpeg -f avfoundation -list_devices true -i ""
[...]
[AVFoundation indev @ 0x158705230] AVFoundation video devices:
[AVFoundation indev @ 0x158705230] [0] FaceTime HD Camera (ID: 
47B4B64B70674B9CAD2BAE273A71F4B5)
[AVFoundation indev @ 0x158705230] [1] Capture screen 0 (ID: 
AvfilterAvfoundationCaptureScreen1)
[AVFoundation indev @ 0x158705230] AVFoundation audio devices:
[AVFoundation indev @ 0x158705230] [0] Loopback Audio (ID: 
com.rogueamoeba.Loopback.A5668B36-711E-4DF5-8A8D-7148508C735B)
[AVFoundation indev @ 0x158705230] [1] MacBook Pro Microphone 
(ID:BuiltInMicrophoneDevice)

Notes:
* Unique names do not seem to follow any specific pattern. I have used
one similar to the builtin microphone for screen capture
* The : substitution is actually required. The loopback device above did
have it in its name.

Signed-off-by: Romain Beauxis 
---
doc/indevs.texi|  6 ++--
libavdevice/avfoundation.m | 72 +-
2 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 5be647f70a..2b55399c8c 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
-i "[[VIDEO]:[AUDIO]]"
@end example
The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or ID as shown by the 
device list.
Alternatively, the video and/or audio input device can be chosen by index using 
the
@option{
-video_device_index 
@@ -127,7 +127,9 @@ and/or
device name or index given in the input filename.

All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name 
can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.

There are two device name aliases:
@table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index b602cfbe95..25286507d6 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -39,6 +39,8 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" 
withString:@"."] UTF8String]
+
static const int avf_time_base = 100;

static const AVRational avf_time_base_q = {
@@ -797,21 +799,23 @@ static int avf_read_header(AVFormatContext *s)
int index = 0;
av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [devices_muxed 
indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices count] + [devices_muxed 
indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
if (num_screens > 0) {
CGDirectDisplayID screens[num_screens];
CGGetActiveDisplayList(num_screens, screens, _screens);
for (int i = 0; i < num_screens; i++) {
-

[FFmpeg-devel] [PATCH v5 2/3] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length

2021-12-17 Thread Romain Beauxis

(Apologies for the previously mangled message, I’m still honing my tools here!)

This is the second patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
v2: None
v3: Switched queue implementation to CMSimpleQueue
v4: None
v5: Fix indentation/wrapping

This patch fixes the concurrency model. Avfoundation runs its own
producing thread to send produced frames and ffmpeg runs its own thread
to consume them.

The existing implementation stores the last transmitted frame and uses a
mutex to avoid concurrent access. However, this leads to situations
where upcoming frames can be dropped if the ffmpeg thread is acessing
the latest frame. This happens even when the thread would otherwise
catch up and process frames fast enought.

This patches changes this implementation to use a buffer queue with a
max queue length and encapsulated thread-safety. This greatly simplifies
the logic of the calling code and gives the consuming thread a chance to
process all frames concurrently to the producing thread while avoiding
memory leaks.

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 169 +
1 file changed, 76 insertions(+), 93 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 79c9207cfa..b602cfbe95 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
 */

#import 
-#include 
+#import 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -80,13 +80,12 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define MAX_QUEUED_FRAMES 10
+
typedef struct
{
AVClass*class;

-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
id  avf_delegate;
id  avf_audio_delegate;

@@ -121,8 +120,8 @@
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+CMSimpleQueueRef  audio_frames_queue;
+CMSimpleQueueRef  video_frames_queue;

AVCaptureDevice  *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -131,16 +130,6 @@
int  observed_quit;
} AVFContext;

-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
@@ -218,17 +207,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
-}
-
-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
+CMSimpleQueueEnqueue(_context->video_frames_queue, videoFrame);
}

@end
@@ -262,17 +242,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
-}
-
-_context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
-
-unlock_frames(_context);
-
-++_context->audio_frames_captured;
+CFRetain(audioFrame);
+CMSimpleQueueEnqueue(_context->audio_frames_queue, audioFrame);
}

@end
@@ -287,6 +258,30 @@ static void destroy_context(AVFContext* ctx)
[ctx->avf_delegaterelease];
[ctx->avf_audio_delegate release];

+CMSampleBufferRef frame;
+
+if (ctx->video_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);
+do {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);
+} while (frame);
+
+CFRelease(ctx->video_frames_queue);
+ctx->video_frames_queue = NULL;
+}
+
+if (ctx->audio_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);
+do {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);
+} while (frame);
+
+CFRelease(ctx->audio_frames_queue);
+ctx->audio_frames_queue = NULL;
+}
+
ctx->capture_session = NULL;
ctx->video_output= NULL;
ctx->audio_output= NULL;
@@ -297,12 +292,6 @@ static voi

[FFmpeg-devel] [PATCH v5 1/3] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-17 Thread Romain Beauxis

(Apologies for the previously mangled message, I’m still honing my tools here!)

This is the first patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
* v2: None
* v3: None
* v4: None
* v5: Fix indentation/wrapping

* Implement support for AudioConverter
* Switch to AudioConverter's API to convert unsupported PCM
  formats (non-interleaved, non-packed) to supported formats
* Minimize data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref:
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 250 +
1 file changed, 144 insertions(+), 106 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..79c9207cfa 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,10 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;
-ctx->audio_packed  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsPacked;
-ctx->audio_non_interleaved = basic_desc->mFormatFlags & 
kAudioFormatFlagIsNonInterleaved;
-
-if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_float &&
-ctx->audio_bits_per_sample == 32 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : 
AV_CODEC_ID_PCM_F32LE;
-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_signed_integer &&
-ctx->audio_bits_per_sample == 16 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : 
AV_CODEC_ID_PCM_S16LE;
-} else if (basic_d

[FFmpeg-devel] [PATCH v4 3/3] libavdevice/avfoundation.m: Allow to select devices by unique ID.

2021-12-17 Thread Romain Beauxis




This is the third patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
v2: None
v3:
  * Switched unique ID to use system-prodvided unique ID
  * Implemented unique IDs for screen capture
v4: Cleanup

This patch adds a unique ID to avfoundation devices. This is needed
because device index can change while the machine is running when
devices are plugged or unplugged and device names can be tricky to use
with localization and etc.

Example of output:
./ffmpeg -f avfoundation -list_devices true -i ""
[...]
[AVFoundation indev @ 0x158705230] AVFoundation video devices:
[AVFoundation indev @ 0x158705230] [0] FaceTime HD Camera (ID:
47B4B64B70674B9CAD2BAE273A71F4B5)
[AVFoundation indev @ 0x158705230] [1] Capture screen 0 (ID:
AvfilterAvfoundationCaptureScreen1)
[AVFoundation indev @ 0x158705230] AVFoundation audio devices:
[AVFoundation indev @ 0x158705230] [0] Loopback Audio (ID:
com.rogueamoeba.Loopback.A5668B36-711E-4DF5-8A8D-7148508C735B)
[AVFoundation indev @ 0x158705230] [1] MacBook Pro Microphone (ID:
BuiltInMicrophoneDevice)

Notes:
* Unique names do not seem to follow any specific pattern. I have used
one similar to the builtin microphone for screen capture
* The : substitution is actually required. The loopback device above did
have it in its name.

Signed-off-by: Romain Beauxis 
---
 doc/indevs.texi|  6 ++--
 libavdevice/avfoundation.m | 72 +-
 2 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 5be647f70a..2b55399c8c 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following 
syntax:

 -i "[[VIDEO]:[AUDIO]]"
 @end example
 The first entry selects the video input while the latter selects the 
audio input.
-The stream has to be specified by the device name or the device index 
as shown by the device list.
+The stream has to be specified by the device name, index or ID as shown 
by the device list.
 Alternatively, the video and/or audio input device can be chosen by 
index using the

 @option{
 -video_device_index 
@@ -127,7 +127,9 @@ and/or
 device name or index given in the input filename.
  All available devices can be enumerated by using 
@option{-list_devices true}, listing

-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device 
name can be +tricky to use when localized and device index can change 
when devices are plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not 
change over time.

  There are two device name aliases:
 @table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index b602cfbe95..25286507d6 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -39,6 +39,8 @@
 #include "libavutil/imgutils.h"
 #include "avdevice.h"
 +#define CLEANUP_DEVICE_ID(s) [[s 
stringByReplacingOccurrencesOfString:@":" withString:@"."] UTF8String]

+
 static const int avf_time_base = 100;
  static const AVRational avf_time_base_q = {
@@ -797,21 +799,23 @@ static int avf_read_header(AVFormatContext *s)
 int index = 0;
 av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
 for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);

 }
 for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [devices_muxed 
indexOfObject:device];

-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices count] + [devices_muxed 
indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);

 }
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
 if (num_screens > 0) {
 CGDirectDisplayID screens[num_screens];
 CGGetActiveDisplayList(num_screens, screens, _screens);
 for (int i = 0; i < num_screens; i++) {
-av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", 
ctx-&

[FFmpeg-devel] [PATCH v4 2/3] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length.

2021-12-17 Thread Romain Beauxis


This is the second patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes:
v2: None
v3: Switched queue implementation to CMSimpleQueue
v4: None

This patch fixes the concurrency model. Avfoundation runs its own
producing thread to send produced frames and ffmpeg runs its own thread
to consume them.

The existing implementation stores the last transmitted frame and uses a
mutex to avoid concurrent access. However, this leads to situations
where upcoming frames can be dropped if the ffmpeg thread is acessing
the latest frame. This happens even when the thread would otherwise
catch up and process frames fast enought.

This patches changes this implementation to use a buffer queue with a
max queue length and encapsulated thread-safety. This greatly simplifies
the logic of the calling code and gives the consuming thread a chance to
process all frames concurrently to the producing thread while avoiding
memory leaks.

Signed-off-by: Romain Beauxis 
---
 libavdevice/avfoundation.m | 169 +
 1 file changed, 76 insertions(+), 93 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 79c9207cfa..b602cfbe95 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
  */
  #import 
-#include 
+#import 
  #include "libavutil/channel_layout.h"
 #include "libavutil/pixdesc.h"
@@ -80,13 +80,12 @@
 { AV_PIX_FMT_NONE, 0 }
 };
 +#define MAX_QUEUED_FRAMES 10
+
 typedef struct
 {
 AVClass*class;
 -int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
 id  avf_delegate;
 id  avf_audio_delegate;
 @@ -121,8 +120,8 @@
 AVCaptureSession *capture_session;
 AVCaptureVideoDataOutput *video_output;
 AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+CMSimpleQueueRef  audio_frames_queue;
+CMSimpleQueueRef  video_frames_queue;
  AVCaptureDevice  *observed_device;
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -131,16 +130,6 @@
 int  observed_quit;
 } AVFContext;
 -static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
 /** FrameReciever class - delegate for AVCaptureSession
  */
 @interface AVFFrameReceiver : NSObject
@@ -218,17 +207,8 @@ - (void)  captureOutput:(AVCaptureOutput 
*)captureOutput

   didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
  fromConnection:(AVCaptureConnection *)connection
 {
-lock_frames(_context);
-
-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
-}
-
-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
+CMSimpleQueueEnqueue(_context->video_frames_queue, videoFrame);
 }
  @end
@@ -262,17 +242,8 @@ - (void)  captureOutput:(AVCaptureOutput 
*)captureOutput

   didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
  fromConnection:(AVCaptureConnection *)connection
 {
-lock_frames(_context);
-
-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
-}
-
-_context->current_audio_frame = 
(CMSampleBufferRef)CFRetain(audioFrame);

-
-unlock_frames(_context);
-
-++_context->audio_frames_captured;
+CFRetain(audioFrame);
+CMSimpleQueueEnqueue(_context->audio_frames_queue, audioFrame);
 }
  @end
@@ -287,6 +258,30 @@ static void destroy_context(AVFContext* ctx)
 [ctx->avf_delegaterelease];
 [ctx->avf_audio_delegate release];
 +CMSampleBufferRef frame;
+
+if (ctx->video_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);

+do {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);

+} while (frame);
+
+CFRelease(ctx->video_frames_queue);
+ctx->video_frames_queue = NULL;
+}
+
+if (ctx->audio_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);

+do {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);

+} while (frame);
+
+CFRelease(ctx->audio_frames_queue);
+ctx->audio_frames_queue = NULL;
+}
+
 ctx->capture_session = NULL;
 ctx->video_output= NULL;
 ctx->audio_output= NULL;
@@ -297,12 +292,6 @@ static void destroy_context(AVFContext* ctx)
   AudioConverterDi

[FFmpeg-devel] [PATCH v4 1/3] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-17 Thread Romain Beauxis

This is the first patch of a series of 3 that cleanup and enhance the 
avfoundation implementation for libavdevice.


Changes:
* v2: None
* v3: None
* v4: None

* Implement support for AudioConverter
* Switch to AudioConverter's API to convert unsupported PCM
  formats (non-interleaved, non-packed) to supported formats
* Minimize data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref: 
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services


Signed-off-by: Romain Beauxis 
---
 libavdevice/avfoundation.m | 250 +
 1 file changed, 144 insertions(+), 106 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..79c9207cfa 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,10 @@
  int num_video_devices;
 -int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32bytes_per_sample;
+AudioConverterRef audio_converter;
  enum AVPixelFormat pixel_format;
 @@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
 ctx->avf_delegate= NULL;
 ctx->avf_audio_delegate = NULL;
 -av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}
  pthread_mutex_destroy(>frame_lock);
 @@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
 AVFContext *ctx = (AVFContext*)s->priv_data;
 CMFormatDescriptionRef format_desc;
 AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;
  if (!stream) {
 return 1;
@@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
 avpriv_set_pts_info(stream, 64, 1, avf_time_base);
  format_desc = 
CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

 -if (!basic_desc) {
+if (!input_format) {
 unlock_frames(ctx);
 av_log(s, AV_LOG_ERROR, "audio format not available\n");
 return 1;
 }
 +if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at 
the moment\n");

+return 1;
+}
+
 stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
 stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

 -ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;
-ctx->audio_packed  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsPacked;
-ctx->audio_non_interleaved = basic_desc->mFormatFlags & 
kAudioFormatFlagIsNonInterleaved;

-
-if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_float &&
-ctx->audio_bits_per_sample == 32 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? 
AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;

-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_signed_integer &&
-ctx->audio_bits_per_sample == 16 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? 
AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;

-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_sign

Re: [FFmpeg-devel] [PATCH v2 03/03] libavdevice/avfoundation.m: Allow to select devices by digest.

2021-12-14 Thread Romain Beauxis

Just sent an updated patch here: 
http://ffmpeg.org/pipermail/ffmpeg-devel/2021-December/289686.html

> On Dec 13, 2021, at 12:25 PM, Marvin Scholz  wrote:
> 
> On 13 Dec 2021, at 17:40, Romain Beauxis wrote:
> 
>> This is the third patch of a series of 3 that cleanup and enhance the
>> avfoundation implementation for libavdevice.
>> 
>> This patch adds a digest to avfoundation devices, when available. This
>> is needed because device index can change while the machine is running when
>> devices are plugged or unplugged and device names can be tricky to use with 
>> localization
>> and etc.
>> 
>> The only device type that are excluded are screen capture because the logic 
>> to select
>> them seems a little different and I wanted to minimized the changes. Also, 
>> for these
>> devices, the name is localized in english, quite straight forward and should 
>> not change.
>> 
>> Signed-off-by: Romain Beauxis 
>> ---
> 
> 
> Hi,
> thanks for the patch, however I fail to see the benefit of it.
> You mention that using the name is complicated because it is localized,
> but your patch just seems to use the device name and hashes it, which
> does not mitigate the issues with the localized name (that could change
> when the language is changed) but just hides it behind a hash, making
> this problem even more obscure and confusing.
> (Correct me if I read your patch wrong, but it seems to just do that.)
> 
> Instead I think that you should use the uniqueID property of an 
> AVCaptureDevice,
> documented here:
> https://developer.apple.com/documentation/avfoundation/avcapturedevice/1390477-uniqueid?language=objc
>  
> <https://developer.apple.com/documentation/avfoundation/avcapturedevice/1390477-uniqueid?language=objc>
> 
> According to the docs, this seems more appropriated, as it does not depend on
> the localization and does not change when the device is unplugged and 
> re-plugged.
> 
> Regards,
> Marvin Scholz
> 
>> doc/indevs.texi|  6 ++--
>> libavdevice/avfoundation.m | 60 ++
>> 2 files changed, 58 insertions(+), 8 deletions(-)
>> 
>> diff --git a/doc/indevs.texi b/doc/indevs.texi
>> index 5be647f70a..8345b64a28 100644
>> --- a/doc/indevs.texi
>> +++ b/doc/indevs.texi
>> @@ -114,7 +114,7 @@ The input filename has to be given in the following 
>> syntax:
>> -i "[[VIDEO]:[AUDIO]]"
>> @end example
>> The first entry selects the video input while the latter selects the audio 
>> input.
>> -The stream has to be specified by the device name or the device index as 
>> shown by the device list.
>> +The stream has to be specified by the device name, index or digest as shown 
>> by the device list.
>> Alternatively, the video and/or audio input device can be chosen by index 
>> using the
>> @option{
>>-video_device_index 
>> @@ -127,7 +127,9 @@ and/or
>> device name or index given in the input filename.
>> 
>> All available devices can be enumerated by using @option{-list_devices 
>> true}, listing
>> -all device names and corresponding indices.
>> +all device names, corresponding indices and digests, when available. Device 
>> name can be
>> +tricky to use when localized and device index can change when devices are 
>> plugged or unplugged. A device
>> +hash, when available, uniquely identifies a device and should not change 
>> over time.
>> 
>> There are two device name aliases:
>> @table @code
>> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
>> index 95414fd16a..bede51bda0 100644
>> --- a/libavdevice/avfoundation.m
>> +++ b/libavdevice/avfoundation.m
>> @@ -26,6 +26,7 @@
>> */
>> 
>> #import 
>> +#import 
>> 
>> #include "libavutil/channel_layout.h"
>> #include "libavutil/pixdesc.h"
>> @@ -79,6 +80,28 @@
>>{ AV_PIX_FMT_NONE, 0 }
>> };
>> 
>> +#define DEVICES_DIGEST_LENGTH 8
>> +
>> +@interface AvdeviceAvfoundationDigest : NSObject
>> ++ (NSString *)fromString:(NSString *)input;
>> +@end
>> +
>> +@implementation AvdeviceAvfoundationDigest : NSObject
>> ++ (NSString *) fromString:(NSString *)input {
>> +const char *cStr = [input UTF8String];
>> +unsigned char digest[CC_SHA256_DIGEST_LENGTH];
>> +CC_SHA256( cStr, strlen(cStr), digest );
>> +
>> +NSMutableString *output = [NSMutableString 
>> stringWithCapacity:CC_SHA256_DIGEST_LENGTH * 2];
>> +
>> +for(int i = 0; i <

Re: [FFmpeg-devel] [PATCH v2 02/03] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length.

2021-12-14 Thread Romain Beauxis

Just sent an updated patch here: 
http://ffmpeg.org/pipermail/ffmpeg-devel/2021-December/289684.html

> On Dec 13, 2021, at 3:12 PM, Marvin Scholz  wrote:
> 
> 
> 
> On 13 Dec 2021, at 21:29, Romain Beauxis wrote:
> 
>>> On Dec 13, 2021, at 12:56 PM, Marvin Scholz  wrote:
>>> 
>>> 
>>> 
>>> On 13 Dec 2021, at 17:39, Romain Beauxis wrote:
>>> 
>>>> This is the second patch of a series of 3 that cleanup and enhance the
>>>> avfoundation implementation for libavdevice.
>>>> 
>>>> This patch fixes the concurrency model. Avfoundation runs its own 
>>>> producing thread
>>>> to send produced frames and ffmpeg runs its own thread to consume them.
>>>> 
>>>> The existing implementation stores the last transmitted frame and uses a 
>>>> mutex
>>>> to avoid concurrent access. However, this leads to situations where 
>>>> upcoming frames
>>>> can be dropped if the ffmpeg thread is acessing the latest frame. This 
>>>> happens
>>>> even when the thread would otherwise catch up and process frames fast 
>>>> enought.
>>>> 
>>>> This patches changes this implementation to use a buffer queue with a max 
>>>> queue length
>>>> and encapsulated thread-safety. This greatly simplifies the logic of the 
>>>> calling code
>>>> and gives the consuming thread a chance to process all frames concurrently 
>>>> to the producing
>>>> thread while avoiding memory leaks.
>>> 
>>> Couldn't this just use CMSimpleQueue 
>>> https://developer.apple.com/documentation/coremedia/cmsimplequeue?language=objc
>>> or CMBufferQueue?
>> 
>> I’m happy to switch to this one, which seems more directly related to the 
>> task at hand if you think it is a better primitive.
>> 
> 
> I did not check in details but if either of the existing implementations 
> referred to above do the task you need here,
> I would prefer if it is used instead of writing your own implementation.
> 
>>> The implementation of the queue in this patch does not seem right, see 
>>> review below.
>>> 
>>>> 
>>>> Signed-off-by: Romain Beauxis 
>>>> ---
>>>> libavdevice/avfoundation.m | 220 +
>>>> 1 file changed, 127 insertions(+), 93 deletions(-)
>>>> 
>>>> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
>>>> index 79c9207cfa..95414fd16a 100644
>>>> --- a/libavdevice/avfoundation.m
>>>> +++ b/libavdevice/avfoundation.m
>>>> @@ -26,7 +26,6 @@
>>>> */
>>>> 
>>>> #import 
>>>> -#include 
>>>> 
>>>> #include "libavutil/channel_layout.h"
>>>> #include "libavutil/pixdesc.h"
>>>> @@ -80,13 +79,97 @@
>>>>   { AV_PIX_FMT_NONE, 0 }
>>>> };
>>>> 
>>>> +#define MAX_QUEUED_OBJECTS 10
>>>> +
>>>> +@interface AvdeviceAvfoundationBuffer : NSObject
>>>> ++ (AvdeviceAvfoundationBuffer *) 
>>>> fromCMSampleBufferRef:(CMSampleBufferRef)sampleBuffer;
>>>> +- (CMSampleBufferRef) getCMSampleBuffer;
>>>> +@end
>>>> +
>>>> +@implementation AvdeviceAvfoundationBuffer {
>>>> +CMSampleBufferRef sampleBuffer;
>>>> +}
>>>> +
>>>> ++ (AvdeviceAvfoundationBuffer *) 
>>>> fromCMSampleBufferRef:(CMSampleBufferRef)sampleBuffer {
>>>> +return [[AvdeviceAvfoundationBuffer alloc] init:sampleBuffer];
>>>> +}
>>>> +
>>>> +- (id) init:(CMSampleBufferRef)buffer {
>>>> +sampleBuffer = buffer;
>>>> +return self;
>>>> +}
>>>> +
>>>> +- (CMSampleBufferRef) getCMSampleBuffer {
>>>> +return sampleBuffer;
>>>> +}
>>>> +@end
>>>> +
>>>> +@interface AvdeviceAvfoundationBufferQueue : NSObject
>>>> +- (CMSampleBufferRef) dequeue;
>>>> +- (NSUInteger) count;
>>>> +- (void) enqueue:(CMSampleBufferRef)obj;
>>>> +@end
>>>> +
>>>> +@implementation AvdeviceAvfoundationBufferQueue {
>>>> +NSLock *mutex;
>>>> +NSMutableArray *queue;
>>>> +}
>>>> +
>>>> +- (id) init {
>>>> +mutex = [[[NSLock alloc] init] retain];
>>>> +queue = [[[NSMutableArray a

[FFmpeg-devel] [PATCH v3 03/03] libavdevice/avfoundation.m: Allow to select devices by unique ID.

2021-12-14 Thread Romain Beauxis

This is the third patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes since last version:
* Switched unique ID to use system-prodvided unique ID
* Implemented unique IDs for screen capture

This patch adds a unique ID to avfoundation devices. This
is needed because device index can change while the machine is running when
devices are plugged or unplugged and device names can be tricky to use with 
localization
and etc.

Signed-off-by: Romain Beauxis 
---
doc/indevs.texi|  6 ++--
libavdevice/avfoundation.m | 73 +-
2 files changed, 61 insertions(+), 18 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 5be647f70a..2b55399c8c 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
-i "[[VIDEO]:[AUDIO]]"
@end example
The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or ID as shown by the 
device list.
Alternatively, the video and/or audio input device can be chosen by index using 
the
@option{
-video_device_index 
@@ -127,7 +127,9 @@ and/or
device name or index given in the input filename.

All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and IDs, when available. Device name 
can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.

There are two device name aliases:
@table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index b602cfbe95..036fb57568 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -39,6 +39,8 @@
#include "libavutil/imgutils.h"
#include "avdevice.h"

+#define CLEANUP_DEVICE_ID(s) [[s stringByReplacingOccurrencesOfString:@":" 
withString:@"."] UTF8String]
+
static const int avf_time_base = 100;

static const AVRational avf_time_base_q = {
@@ -797,21 +799,23 @@ static int avf_read_header(AVFormatContext *s)
int index = 0;
av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
for (AVCaptureDevice *device in devices_muxed) {
-const char *name = [[device localizedName] UTF8String];
-index= [devices count] + [devices_muxed 
indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+index= [devices count] + [devices_muxed 
indexOfObject:device];
+av_log(ctx, AV_LOG_INFO, "[%d] %s (ID: %s)\n", index, name, 
uniqueId);
}
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
if (num_screens > 0) {
CGDirectDisplayID screens[num_screens];
CGGetActiveDisplayList(num_screens, screens, _screens);
for (int i = 0; i < num_screens; i++) {
-av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", 
ctx->num_video_devices + i, i);
+av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d (ID: 
AvfilterAvfoundationCptureScreen%d)\n", ctx->num_video_devices + i, i, 
screens[i]);
}
}
#endif
@@ -819,9 +823,10 @@ static int avf_read_header(AVFormatContext *s)
av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
for (AVCaptureDevice *device in devices) {
-const char *name = [[device localizedName] UTF8String];
-int index  = [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+const char *name = [[device localizedName] UTF8String];
+const char *uniqueId = CLEANUP_DEVICE_ID([device uniqueID]);
+int index= [devices

[FFmpeg-devel] [PATCH v3 02/03] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length.

2021-12-14 Thread Romain Beauxis

This is the second patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes since last version:
* Switched queue implementation to CMSimpleQueue

This patch fixes the concurrency model. Avfoundation runs its own producing 
thread
to send produced frames and ffmpeg runs its own thread to consume them.

The existing implementation stores the last transmitted frame and uses a mutex
to avoid concurrent access. However, this leads to situations where upcoming 
frames
can be dropped if the ffmpeg thread is acessing the latest frame. This happens
even when the thread would otherwise catch up and process frames fast enought.

This patches changes this implementation to use a buffer queue with a max queue 
length
and encapsulated thread-safety. This greatly simplifies the logic of the 
calling code
and gives the consuming thread a chance to process all frames concurrently to 
the producing
thread while avoiding memory leaks.

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 169 +
1 file changed, 76 insertions(+), 93 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 79c9207cfa..b602cfbe95 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,7 @@
 */

#import 
-#include 
+#import 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -80,13 +80,12 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define MAX_QUEUED_FRAMES 10
+
typedef struct
{
AVClass*class;

-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
id  avf_delegate;
id  avf_audio_delegate;

@@ -121,8 +120,8 @@
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+CMSimpleQueueRef  audio_frames_queue;
+CMSimpleQueueRef  video_frames_queue;

AVCaptureDevice  *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -131,16 +130,6 @@
int  observed_quit;
} AVFContext;

-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
@@ -218,17 +207,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
-}
-
-_context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
-
-unlock_frames(_context);
-
-++_context->frames_captured;
+CFRetain(videoFrame);
+CMSimpleQueueEnqueue(_context->video_frames_queue, videoFrame);
}

@end
@@ -262,17 +242,8 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_audio_frame != nil) {
-CFRelease(_context->current_audio_frame);
-}
-
-_context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
-
-unlock_frames(_context);
-
-++_context->audio_frames_captured;
+CFRetain(audioFrame);
+CMSimpleQueueEnqueue(_context->audio_frames_queue, audioFrame);
}

@end
@@ -287,6 +258,30 @@ static void destroy_context(AVFContext* ctx)
[ctx->avf_delegaterelease];
[ctx->avf_audio_delegate release];

+CMSampleBufferRef frame;
+
+if (ctx->video_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);
+do {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->video_frames_queue);
+} while (frame);
+
+CFRelease(ctx->video_frames_queue);
+ctx->video_frames_queue = NULL;
+}
+
+if (ctx->audio_frames_queue) {
+frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);
+do {
+  CFRelease(frame);
+  frame = 
(CMSampleBufferRef)CMSimpleQueueDequeue(ctx->audio_frames_queue);
+} while (frame);
+
+CFRelease(ctx->audio_frames_queue);
+ctx->audio_frames_queue = NULL;
+}
+
ctx->capture_session = NULL;
ctx->video_output= NULL;
ctx->audio_output= NULL;
@@ -297,12 +292,6 @@ static void destroy_context(AVFContext* ctx)
  AudioConverterDispose(ctx->audio_converter);
  ctx->audio_

[FFmpeg-devel] [PATCH v3 01/03] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-14 Thread Romain Beauxis

This is the first patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

Changes since previous version: none

This patch:
* Implements support for AudioConverter
* Switches to AudioConverter's API to convert unsupported PCM
 formats (non-interleaved, non-packed) to supported formats
* Minimizes data copy.

This fixes: 
https://trac.ffmpeg.org/ticket/9502


API ref: 
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services


Signed-off-by: Romain Beauxis <
toots at rastageeks.org>
---
libavdevice/avfoundation.m | 250 +
1 file changed, 144 insertions(+), 106 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..79c9207cfa 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,10 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;
-ctx->audio_packed  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsPacked;
-ctx->audio_non_interleaved = basic_desc->mFormatFlags & 
kAudioFormatFlagIsNonInterleaved;
-
-if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_float &&
-ctx->audio_bits_per_sample == 32 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : 
AV_CODEC_ID_PCM_F32LE;
-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_signed_integer &&
-ctx->audio_bits_per_sample == 16 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : 
AV_CODEC_ID_PCM_S16LE;
-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->aud

Re: [FFmpeg-devel] [PATCH v2 02/03] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length.

2021-12-13 Thread Romain Beauxis



> On Dec 13, 2021, at 12:56 PM, Marvin Scholz  wrote:
> 
> 
> 
> On 13 Dec 2021, at 17:39, Romain Beauxis wrote:
> 
>> This is the second patch of a series of 3 that cleanup and enhance the
>> avfoundation implementation for libavdevice.
>> 
>> This patch fixes the concurrency model. Avfoundation runs its own producing 
>> thread
>> to send produced frames and ffmpeg runs its own thread to consume them.
>> 
>> The existing implementation stores the last transmitted frame and uses a 
>> mutex
>> to avoid concurrent access. However, this leads to situations where upcoming 
>> frames
>> can be dropped if the ffmpeg thread is acessing the latest frame. This 
>> happens
>> even when the thread would otherwise catch up and process frames fast 
>> enought.
>> 
>> This patches changes this implementation to use a buffer queue with a max 
>> queue length
>> and encapsulated thread-safety. This greatly simplifies the logic of the 
>> calling code
>> and gives the consuming thread a chance to process all frames concurrently 
>> to the producing
>> thread while avoiding memory leaks.
> 
> Couldn't this just use CMSimpleQueue 
> https://developer.apple.com/documentation/coremedia/cmsimplequeue?language=objc
> or CMBufferQueue?

I’m happy to switch to this one, which seems more directly related to the task 
at hand if you think it is a better primitive.

> The implementation of the queue in this patch does not seem right, see review 
> below.
> 
>> 
>> Signed-off-by: Romain Beauxis 
>> ---
>> libavdevice/avfoundation.m | 220 +
>> 1 file changed, 127 insertions(+), 93 deletions(-)
>> 
>> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
>> index 79c9207cfa..95414fd16a 100644
>> --- a/libavdevice/avfoundation.m
>> +++ b/libavdevice/avfoundation.m
>> @@ -26,7 +26,6 @@
>> */
>> 
>> #import 
>> -#include 
>> 
>> #include "libavutil/channel_layout.h"
>> #include "libavutil/pixdesc.h"
>> @@ -80,13 +79,97 @@
>>{ AV_PIX_FMT_NONE, 0 }
>> };
>> 
>> +#define MAX_QUEUED_OBJECTS 10
>> +
>> +@interface AvdeviceAvfoundationBuffer : NSObject
>> ++ (AvdeviceAvfoundationBuffer *) 
>> fromCMSampleBufferRef:(CMSampleBufferRef)sampleBuffer;
>> +- (CMSampleBufferRef) getCMSampleBuffer;
>> +@end
>> +
>> +@implementation AvdeviceAvfoundationBuffer {
>> +CMSampleBufferRef sampleBuffer;
>> +}
>> +
>> ++ (AvdeviceAvfoundationBuffer *) 
>> fromCMSampleBufferRef:(CMSampleBufferRef)sampleBuffer {
>> +return [[AvdeviceAvfoundationBuffer alloc] init:sampleBuffer];
>> +}
>> +
>> +- (id) init:(CMSampleBufferRef)buffer {
>> +sampleBuffer = buffer;
>> +return self;
>> +}
>> +
>> +- (CMSampleBufferRef) getCMSampleBuffer {
>> +return sampleBuffer;
>> +}
>> +@end
>> +
>> +@interface AvdeviceAvfoundationBufferQueue : NSObject
>> +- (CMSampleBufferRef) dequeue;
>> +- (NSUInteger) count;
>> +- (void) enqueue:(CMSampleBufferRef)obj;
>> +@end
>> +
>> +@implementation AvdeviceAvfoundationBufferQueue {
>> +NSLock *mutex;
>> +NSMutableArray *queue;
>> +}
>> +
>> +- (id) init {
>> +mutex = [[[NSLock alloc] init] retain];
>> +queue = [[[NSMutableArray alloc] init] retain];
>> +return self;
>> +}
>> +
>> +- (oneway void) release {
>> +NSEnumerator *enumerator = [queue objectEnumerator];
>> +AvdeviceAvfoundationBuffer *buffer;
>> +
>> +while (buffer = [enumerator nextObject]) {
>> +CFRelease([buffer getCMSampleBuffer]);
>> +}
>> +
>> +[mutex release];
>> +[queue release];
>> +}
> 
> Shouldn't this be done in dealloc instead of release?
> Especially as retain is not subclassed, so this seems
> like it could lead to over-releasing resources.

I’m fairly new to objective-c’s memory model, I’ll double check those.

> 
>> +
>> +- (NSUInteger) count {
>> +[mutex lock];
>> +NSUInteger c = [queue count];
>> +[mutex unlock];
>> +return c;
>> +}
> 
> This does not look right, the count can change after it is returned
> and the caller does not hold a lock to prevent this.

For a generic queue it is indeed. However, here, it is used in a monotonic 
fashion only:
* One thread only increases the frame count by pushing new ones
* One thread only decreases the frame count by pulling existing ones

Frame count is only us

Re: [FFmpeg-devel] [PATCH v2 03/03] libavdevice/avfoundation.m: Allow to select devices by digest.

2021-12-13 Thread Romain Beauxis




> On Dec 13, 2021, at 12:25 PM, Marvin Scholz  wrote:
> 
> On 13 Dec 2021, at 17:40, Romain Beauxis wrote:
> 
>> This is the third patch of a series of 3 that cleanup and enhance the
>> avfoundation implementation for libavdevice.
>> 
>> This patch adds a digest to avfoundation devices, when available. This
>> is needed because device index can change while the machine is running when
>> devices are plugged or unplugged and device names can be tricky to use with 
>> localization
>> and etc.
>> 
>> The only device type that are excluded are screen capture because the logic 
>> to select
>> them seems a little different and I wanted to minimized the changes. Also, 
>> for these
>> devices, the name is localized in english, quite straight forward and should 
>> not change.
>> 
>> Signed-off-by: Romain Beauxis 
>> ---
> 
> 
> Hi,
> thanks for the patch, however I fail to see the benefit of it.
> You mention that using the name is complicated because it is localized,
> but your patch just seems to use the device name and hashes it, which
> does not mitigate the issues with the localized name (that could change
> when the language is changed) but just hides it behind a hash, making
> this problem even more obscure and confusing.
> (Correct me if I read your patch wrong, but it seems to just do that.)
> 
> Instead I think that you should use the uniqueID property of an 
> AVCaptureDevice,
> documented here:
> https://developer.apple.com/documentation/avfoundation/avcapturedevice/1390477-uniqueid?language=objc
> 
> According to the docs, this seems more appropriated, as it does not depend on
> the localization and does not change when the device is unplugged and 
> re-plugged.

Oh great, I totally missed this one. Will submit a new version of the changes 
soon then, thanks!

> 
>> doc/indevs.texi|  6 ++--
>> libavdevice/avfoundation.m | 60 ++
>> 2 files changed, 58 insertions(+), 8 deletions(-)
>> 
>> diff --git a/doc/indevs.texi b/doc/indevs.texi
>> index 5be647f70a..8345b64a28 100644
>> --- a/doc/indevs.texi
>> +++ b/doc/indevs.texi
>> @@ -114,7 +114,7 @@ The input filename has to be given in the following 
>> syntax:
>> -i "[[VIDEO]:[AUDIO]]"
>> @end example
>> The first entry selects the video input while the latter selects the audio 
>> input.
>> -The stream has to be specified by the device name or the device index as 
>> shown by the device list.
>> +The stream has to be specified by the device name, index or digest as shown 
>> by the device list.
>> Alternatively, the video and/or audio input device can be chosen by index 
>> using the
>> @option{
>>-video_device_index 
>> @@ -127,7 +127,9 @@ and/or
>> device name or index given in the input filename.
>> 
>> All available devices can be enumerated by using @option{-list_devices 
>> true}, listing
>> -all device names and corresponding indices.
>> +all device names, corresponding indices and digests, when available. Device 
>> name can be
>> +tricky to use when localized and device index can change when devices are 
>> plugged or unplugged. A device
>> +hash, when available, uniquely identifies a device and should not change 
>> over time.
>> 
>> There are two device name aliases:
>> @table @code
>> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
>> index 95414fd16a..bede51bda0 100644
>> --- a/libavdevice/avfoundation.m
>> +++ b/libavdevice/avfoundation.m
>> @@ -26,6 +26,7 @@
>> */
>> 
>> #import 
>> +#import 
>> 
>> #include "libavutil/channel_layout.h"
>> #include "libavutil/pixdesc.h"
>> @@ -79,6 +80,28 @@
>>{ AV_PIX_FMT_NONE, 0 }
>> };
>> 
>> +#define DEVICES_DIGEST_LENGTH 8
>> +
>> +@interface AvdeviceAvfoundationDigest : NSObject
>> ++ (NSString *)fromString:(NSString *)input;
>> +@end
>> +
>> +@implementation AvdeviceAvfoundationDigest : NSObject
>> ++ (NSString *) fromString:(NSString *)input {
>> +const char *cStr = [input UTF8String];
>> +unsigned char digest[CC_SHA256_DIGEST_LENGTH];
>> +CC_SHA256( cStr, strlen(cStr), digest );
>> +
>> +NSMutableString *output = [NSMutableString 
>> stringWithCapacity:CC_SHA256_DIGEST_LENGTH * 2];
>> +
>> +for(int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++)
>> +[output appendFormat:@"%02x", digest[i]];
>> +
>> +// The "d" prefix makes sure that

[FFmpeg-devel] [PATCH v2 03/03] libavdevice/avfoundation.m: Allow to select devices by digest.

2021-12-13 Thread Romain Beauxis

This is the third patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

This patch adds a digest to avfoundation devices, when available. This
is needed because device index can change while the machine is running when
devices are plugged or unplugged and device names can be tricky to use with 
localization
and etc.

The only device type that are excluded are screen capture because the logic to 
select
them seems a little different and I wanted to minimized the changes. Also, for 
these
devices, the name is localized in english, quite straight forward and should 
not change.

Signed-off-by: Romain Beauxis 
---
doc/indevs.texi|  6 ++--
libavdevice/avfoundation.m | 60 ++
2 files changed, 58 insertions(+), 8 deletions(-)

diff --git a/doc/indevs.texi b/doc/indevs.texi
index 5be647f70a..8345b64a28 100644
--- a/doc/indevs.texi
+++ b/doc/indevs.texi
@@ -114,7 +114,7 @@ The input filename has to be given in the following syntax:
-i "[[VIDEO]:[AUDIO]]"
@end example
The first entry selects the video input while the latter selects the audio 
input.
-The stream has to be specified by the device name or the device index as shown 
by the device list.
+The stream has to be specified by the device name, index or digest as shown by 
the device list.
Alternatively, the video and/or audio input device can be chosen by index using 
the
@option{
-video_device_index 
@@ -127,7 +127,9 @@ and/or
device name or index given in the input filename.

All available devices can be enumerated by using @option{-list_devices true}, 
listing
-all device names and corresponding indices.
+all device names, corresponding indices and digests, when available. Device 
name can be 
+tricky to use when localized and device index can change when devices are 
plugged or unplugged. A device
+hash, when available, uniquely identifies a device and should not change over 
time.

There are two device name aliases:
@table @code
diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 95414fd16a..bede51bda0 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,6 +26,7 @@
 */

#import 
+#import 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -79,6 +80,28 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define DEVICES_DIGEST_LENGTH 8
+
+@interface AvdeviceAvfoundationDigest : NSObject
++ (NSString *)fromString:(NSString *)input;
+@end
+
+@implementation AvdeviceAvfoundationDigest : NSObject
++ (NSString *) fromString:(NSString *)input {
+const char *cStr = [input UTF8String];
+unsigned char digest[CC_SHA256_DIGEST_LENGTH];
+CC_SHA256( cStr, strlen(cStr), digest );
+
+NSMutableString *output = [NSMutableString 
stringWithCapacity:CC_SHA256_DIGEST_LENGTH * 2];
+
+for(int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++)
+[output appendFormat:@"%02x", digest[i]];
+
+// The "d" prefix makes sure that digest strings are never mistaken for 
numbers.
+return [@"d" stringByAppendingString:[output 
substringToIndex:DEVICES_DIGEST_LENGTH]];
+}
+@end
+
#define MAX_QUEUED_OBJECTS 10

@interface AvdeviceAvfoundationBuffer : NSObject
@@ -860,13 +883,15 @@ static int avf_read_header(AVFormatContext *s)
av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
for (AVCaptureDevice *device in devices) {
const char *name = [[device localizedName] UTF8String];
+NSString *digest = [AvdeviceAvfoundationDigest 
fromString:[[NSString alloc] initWithUTF8String:name]];
index= [devices indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+av_log(ctx, AV_LOG_INFO, "[%d] %s (digest: %s)\n", index, name, 
[digest UTF8String]);
}
for (AVCaptureDevice *device in devices_muxed) {
const char *name = [[device localizedName] UTF8String];
+NSString *digest = [AvdeviceAvfoundationDigest 
fromString:[[NSString alloc] initWithUTF8String:name]];
index= [devices count] + [devices_muxed 
indexOfObject:device];
-av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+av_log(ctx, AV_LOG_INFO, "[%d] %s (digest: %s)\n", index, name, 
[digest UTF8String]);
}
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
if (num_screens > 0) {
@@ -882,8 +907,9 @@ static int avf_read_header(AVFormatContext *s)
devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
for (AVCaptureDevice *device in devices) {
const char *name = [[device localizedName] UTF8String];
+NSString *digest = [AvdeviceAvfoundationDigest 
fromString:[[NSString alloc] initWithUTF8String:name]];
int index  = [devices indexOfObject:device];
-av_log(ctx

[FFmpeg-devel] [PATCH v2 02/03] libavdevice/avfoundation.m: Replace mutex-based concurrency handling in avfoundation.m by a thread-safe fifo queue with maximum length.

2021-12-13 Thread Romain Beauxis

This is the second patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

This patch fixes the concurrency model. Avfoundation runs its own producing 
thread
to send produced frames and ffmpeg runs its own thread to consume them.

The existing implementation stores the last transmitted frame and uses a mutex
to avoid concurrent access. However, this leads to situations where upcoming 
frames
can be dropped if the ffmpeg thread is acessing the latest frame. This happens
even when the thread would otherwise catch up and process frames fast enought.

This patches changes this implementation to use a buffer queue with a max queue 
length
and encapsulated thread-safety. This greatly simplifies the logic of the 
calling code
and gives the consuming thread a chance to process all frames concurrently to 
the producing
thread while avoiding memory leaks.

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 220 +
1 file changed, 127 insertions(+), 93 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 79c9207cfa..95414fd16a 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -26,7 +26,6 @@
 */

#import 
-#include 

#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
@@ -80,13 +79,97 @@
{ AV_PIX_FMT_NONE, 0 }
};

+#define MAX_QUEUED_OBJECTS 10
+
+@interface AvdeviceAvfoundationBuffer : NSObject
++ (AvdeviceAvfoundationBuffer *) 
fromCMSampleBufferRef:(CMSampleBufferRef)sampleBuffer;
+- (CMSampleBufferRef) getCMSampleBuffer;
+@end
+
+@implementation AvdeviceAvfoundationBuffer {
+CMSampleBufferRef sampleBuffer;
+}
+
++ (AvdeviceAvfoundationBuffer *) 
fromCMSampleBufferRef:(CMSampleBufferRef)sampleBuffer {
+return [[AvdeviceAvfoundationBuffer alloc] init:sampleBuffer];
+}
+
+- (id) init:(CMSampleBufferRef)buffer {
+sampleBuffer = buffer;
+return self;
+}
+
+- (CMSampleBufferRef) getCMSampleBuffer {
+return sampleBuffer;
+}
+@end
+
+@interface AvdeviceAvfoundationBufferQueue : NSObject
+- (CMSampleBufferRef) dequeue;
+- (NSUInteger) count;
+- (void) enqueue:(CMSampleBufferRef)obj;
+@end
+
+@implementation AvdeviceAvfoundationBufferQueue {
+NSLock *mutex;
+NSMutableArray *queue;
+}
+
+- (id) init {
+mutex = [[[NSLock alloc] init] retain];
+queue = [[[NSMutableArray alloc] init] retain];
+return self;
+}
+
+- (oneway void) release {
+NSEnumerator *enumerator = [queue objectEnumerator];
+AvdeviceAvfoundationBuffer *buffer;
+
+while (buffer = [enumerator nextObject]) {
+CFRelease([buffer getCMSampleBuffer]);
+}
+
+[mutex release];
+[queue release];
+}
+
+- (NSUInteger) count {
+[mutex lock];
+NSUInteger c = [queue count];
+[mutex unlock];
+return c;
+}
+
+- (CMSampleBufferRef) dequeue {
+[mutex lock];
+
+if ([queue count] < 1) {
+  [mutex unlock];
+  return nil;
+}
+
+AvdeviceAvfoundationBuffer *buffer = [queue objectAtIndex:0];
+CMSampleBufferRef sampleBuffer = [buffer getCMSampleBuffer];
+[queue removeObjectAtIndex:0];
+[mutex unlock];
+
+return sampleBuffer;
+}
+
+- (void) enqueue:(CMSampleBufferRef)buffer {
+[mutex lock];
+while (MAX_QUEUED_OBJECTS < [queue count]) {
+  [queue removeObjectAtIndex:0];
+}
+[queue addObject:[AvdeviceAvfoundationBuffer 
fromCMSampleBufferRef:(CMSampleBufferRef)CFRetain(buffer)]];
+[mutex unlock];
+}
+@end
+
typedef struct
{
AVClass*class;

-int frames_captured;
-int audio_frames_captured;
-pthread_mutex_t frame_lock;
id  avf_delegate;
id  avf_audio_delegate;

@@ -121,8 +204,8 @@
AVCaptureSession *capture_session;
AVCaptureVideoDataOutput *video_output;
AVCaptureAudioDataOutput *audio_output;
-CMSampleBufferRef current_frame;
-CMSampleBufferRef current_audio_frame;
+AvdeviceAvfoundationBufferQueue *audio_frames;
+AvdeviceAvfoundationBufferQueue *video_frames;

AVCaptureDevice  *observed_device;
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
@@ -131,16 +214,6 @@
int  observed_quit;
} AVFContext;

-static void lock_frames(AVFContext* ctx)
-{
-pthread_mutex_lock(>frame_lock);
-}
-
-static void unlock_frames(AVFContext* ctx)
-{
-pthread_mutex_unlock(>frame_lock);
-}
-
/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
@@ -218,17 +291,7 @@ - (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
 fromConnection:(AVCaptureConnection *)connection
{
-lock_frames(_context);
-
-if (_context->current_frame != nil) {
-CFRelease(_context->current_frame);
-}
-
-_context->current_frame = (CMSampleBufferRef

[FFmpeg-devel] [PATCH v2 01/03] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-13 Thread Romain Beauxis

This is the first patch of a series of 3 that cleanup and enhance the
avfoundation implementation for libavdevice.

This patch:
* Implements support for AudioConverter
* Switches to AudioConverter's API to convert unsupported PCM
 formats (non-interleaved, non-packed) to supported formats
* Minimizes data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref: 
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 250 +
1 file changed, 144 insertions(+), 106 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..79c9207cfa 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,10 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;
-ctx->audio_packed  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsPacked;
-ctx->audio_non_interleaved = basic_desc->mFormatFlags & 
kAudioFormatFlagIsNonInterleaved;
-
-if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_float &&
-ctx->audio_bits_per_sample == 32 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : 
AV_CODEC_ID_PCM_F32LE;
-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_signed_integer &&
-ctx->audio_bits_per_sample == 16 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : 
AV_CODEC_ID_PCM_S16LE;
-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_signed_integer &&
-ctx->audio_bits_per_sample == 24

Re: [FFmpeg-devel] [PATCH] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-08 Thread Romain Beauxis

> On Dec 7, 2021, at 7:21 AM, Thilo Borgmann  wrote:
> 
> Hi,
> 
> will look at this soon (tm), ping me if I don‘t. 

Great, thank you!

Would it be a good use of your time to send the two other patches that I have 
pending as well?

— Romain
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".

Re: [FFmpeg-devel] [PATCH] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-12-01 Thread Romain Beauxis

Hi there!

Anyone interested in the patch below? It fixes an issue with some
macos avfoundation input devices that return formats currently not
supported by the implementation. I also have another important bugfix
in the concurrency model of the implementation waiting for this to be
merged first.

Otherwise, is there any appropriate step to help getting this merged?

Thanks for any insight!
-- Romain

> On Nov 30, 2021, at 12:02 AM, Romain Beauxis  wrote:
> 
> * Implement support for AudioConverter
> * Switch to AudioConverter's API to convert unsupported PCM
> formats (non-interleaved, non-packed) to supported formats
> * Minimize data copy.
> 
> This fixes: https://trac.ffmpeg.org/ticket/9502
> 
> API ref: 
> https://developer.apple.com/documentation/audiotoolbox/audio_converter_services
> 
> Signed-off-by: Romain Beauxis 
> ---
> libavdevice/avfoundation.m | 250 +
> 1 file changed, 144 insertions(+), 106 deletions(-)
> 
> diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
> index 0cd6e646d5..79c9207cfa 100644
> --- a/libavdevice/avfoundation.m
> +++ b/libavdevice/avfoundation.m
> @@ -111,16 +111,10 @@
> 
>int num_video_devices;
> 
> -int audio_channels;
> -int audio_bits_per_sample;
> -int audio_float;
> -int audio_be;
> -int audio_signed_integer;
> -int audio_packed;
> -int audio_non_interleaved;
> -
> -int32_t *audio_buffer;
> -int audio_buffer_size;
> +UInt32audio_buffers;
> +UInt32audio_channels;
> +UInt32bytes_per_sample;
> +AudioConverterRef audio_converter;
> 
>enum AVPixelFormat pixel_format;
> 
> @@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
>ctx->avf_delegate= NULL;
>ctx->avf_audio_delegate = NULL;
> 
> -av_freep(>audio_buffer);
> +if (ctx->audio_converter) {
> +  AudioConverterDispose(ctx->audio_converter);
> +  ctx->audio_converter = NULL;
> +}
> 
>pthread_mutex_destroy(>frame_lock);
> 
> @@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
>AVFContext *ctx = (AVFContext*)s->priv_data;
>CMFormatDescriptionRef format_desc;
>AVStream* stream = avformat_new_stream(s, NULL);
> +AudioStreamBasicDescription output_format = {0};
> +int audio_bits_per_sample, audio_float, audio_be;
> +int audio_signed_integer, audio_packed, audio_non_interleaved;
> +int must_convert = 0;
> 
>if (!stream) {
>return 1;
> @@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
>avpriv_set_pts_info(stream, 64, 1, avf_time_base);
> 
>format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
> -const AudioStreamBasicDescription *basic_desc = 
> CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
> +const AudioStreamBasicDescription *input_format = 
> CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
> 
> -if (!basic_desc) {
> +if (!input_format) {
>unlock_frames(ctx);
>av_log(s, AV_LOG_ERROR, "audio format not available\n");
>return 1;
>}
> 
> +if (input_format->mFormatID != kAudioFormatLinearPCM) {
> +unlock_frames(ctx);
> +av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
> moment\n");
> +return 1;
> +}
> +
>stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
> -stream->codecpar->sample_rate= basic_desc->mSampleRate;
> -stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
> +stream->codecpar->sample_rate= input_format->mSampleRate;
> +stream->codecpar->channels   = input_format->mChannelsPerFrame;
>stream->codecpar->channel_layout = 
> av_get_default_channel_layout(stream->codecpar->channels);
> 
> -ctx->audio_channels= basic_desc->mChannelsPerFrame;
> -ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
> -ctx->audio_float   = basic_desc->mFormatFlags & 
> kAudioFormatFlagIsFloat;
> -ctx->audio_be  = basic_desc->mFormatFlags & 
> kAudioFormatFlagIsBigEndian;
> -ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
> kAudioFormatFlagIsSignedInteger;
> -ctx->audio_packed  = basic_desc->mFormatFlags & 
> kAudioFormatFlagIsPacked;
> -ctx->audio_non_interleaved = basic_desc->mFormatFlags &

[FFmpeg-devel] [PATCH] libavdevice/avfoundation.m: use AudioConvert, extend supported formats

2021-11-30 Thread Romain Beauxis

* Implement support for AudioConverter
* Switch to AudioConverter's API to convert unsupported PCM
 formats (non-interleaved, non-packed) to supported formats
* Minimize data copy.

This fixes: https://trac.ffmpeg.org/ticket/9502

API ref: 
https://developer.apple.com/documentation/audiotoolbox/audio_converter_services

Signed-off-by: Romain Beauxis 
---
libavdevice/avfoundation.m | 250 +
1 file changed, 144 insertions(+), 106 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index 0cd6e646d5..79c9207cfa 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -111,16 +111,10 @@

int num_video_devices;

-int audio_channels;
-int audio_bits_per_sample;
-int audio_float;
-int audio_be;
-int audio_signed_integer;
-int audio_packed;
-int audio_non_interleaved;
-
-int32_t *audio_buffer;
-int audio_buffer_size;
+UInt32audio_buffers;
+UInt32audio_channels;
+UInt32bytes_per_sample;
+AudioConverterRef audio_converter;

enum AVPixelFormat pixel_format;

@@ -299,7 +293,10 @@ static void destroy_context(AVFContext* ctx)
ctx->avf_delegate= NULL;
ctx->avf_audio_delegate = NULL;

-av_freep(>audio_buffer);
+if (ctx->audio_converter) {
+  AudioConverterDispose(ctx->audio_converter);
+  ctx->audio_converter = NULL;
+}

pthread_mutex_destroy(>frame_lock);

@@ -673,6 +670,10 @@ static int get_audio_config(AVFormatContext *s)
AVFContext *ctx = (AVFContext*)s->priv_data;
CMFormatDescriptionRef format_desc;
AVStream* stream = avformat_new_stream(s, NULL);
+AudioStreamBasicDescription output_format = {0};
+int audio_bits_per_sample, audio_float, audio_be;
+int audio_signed_integer, audio_packed, audio_non_interleaved;
+int must_convert = 0;

if (!stream) {
return 1;
@@ -690,60 +691,95 @@ static int get_audio_config(AVFormatContext *s)
avpriv_set_pts_info(stream, 64, 1, avf_time_base);

format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
-const AudioStreamBasicDescription *basic_desc = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
+const AudioStreamBasicDescription *input_format = 
CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

-if (!basic_desc) {
+if (!input_format) {
unlock_frames(ctx);
av_log(s, AV_LOG_ERROR, "audio format not available\n");
return 1;
}

+if (input_format->mFormatID != kAudioFormatLinearPCM) {
+unlock_frames(ctx);
+av_log(s, AV_LOG_ERROR, "only PCM audio format are supported at the 
moment\n");
+return 1;
+}
+
stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
-stream->codecpar->sample_rate= basic_desc->mSampleRate;
-stream->codecpar->channels   = basic_desc->mChannelsPerFrame;
+stream->codecpar->sample_rate= input_format->mSampleRate;
+stream->codecpar->channels   = input_format->mChannelsPerFrame;
stream->codecpar->channel_layout = 
av_get_default_channel_layout(stream->codecpar->channels);

-ctx->audio_channels= basic_desc->mChannelsPerFrame;
-ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
-ctx->audio_float   = basic_desc->mFormatFlags & 
kAudioFormatFlagIsFloat;
-ctx->audio_be  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsBigEndian;
-ctx->audio_signed_integer  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsSignedInteger;
-ctx->audio_packed  = basic_desc->mFormatFlags & 
kAudioFormatFlagIsPacked;
-ctx->audio_non_interleaved = basic_desc->mFormatFlags & 
kAudioFormatFlagIsNonInterleaved;
-
-if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_float &&
-ctx->audio_bits_per_sample == 32 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : 
AV_CODEC_ID_PCM_F32LE;
-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_signed_integer &&
-ctx->audio_bits_per_sample == 16 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : 
AV_CODEC_ID_PCM_S16LE;
-} else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
-ctx->audio_signed_integer &&
-ctx->audio_bits_per_sample == 24 &&
-ctx->audio_packed) {
-stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : 
A

82 matches

Mail list logo