player: add complex filter graph support

See --lavfi-complex option.

This is still quite rough. There's no support for dynamic configuration
of any kind. There are probably corner cases where playback might freeze
or burn 100% CPU (due to dataflow problems when interaction with
libavfilter).

Future possible plans might include:
- freely switch tracks by providing some sort of default track graph
  label
- automatically enabling audio visualization
- automatically mix audio or stack video when multiple tracks are
  selected at once (similar to how multiple sub tracks can be selected)

Author: wm4

DOCS/man/options.rst

@@ -1396,7 +1396,7 @@ Subtitles
     .. admonition:: Warning
 
         Enabling hinting can lead to mispositioned text (in situations it's
-        supposed to match up with video background), or reduce the smoothness
+        supposed to match up video background), or reduce the smoothness
         of animations with some badly authored ASS scripts. It is recommended
         to not use this option, unless really needed.
 
@@ -3553,3 +3553,45 @@ Miscellaneous
     Force the contents of the ``media-title`` property to this value. Useful
     for scripts which want to set a title, without overriding the user's
     setting in ``--title``.
+
+``--lavfi-complex=<string>``
+    Set a "complex" libavfilter filter, which means a single filter graph can
+    take input from multiple source audio and video tracks. The graph can result
+    in a single audio or video output (or both).
+
+    Currently, the filter graph labels are used to select the participating
+    input tracks and audio/video output. The following rules apply:
+
+    - A label of the form ``aidN`` selects audio track N as input (e.g.
+      ``aid1``).
+    - A label of the form ``vidN`` selects video track N as input.
+    - A label named ``ao`` will be connected to the audio input.
+    - A label named ``vo`` will be connected to the video output.
+
+    Each label can be used only once. If you want to use e.g. an audio stream
+    for multiple filters, you need to use the ``asplit`` filter. Multiple
+    video or audio outputs are not possible, but you can use filters to merge
+    them into one.
+
+    The complex filter can not be changed yet during playback. It's also not
+    possible to change the tracks connected to the filter at runtime. Other
+    tracks, as long as they're not connected to the filter, and the
+    corresponding output is not connected to the filter, can still be freely
+    changed.
+
+    .. admonition:: Examples
+
+        - ``--lavfi-complex='[aid1] asplit [ao] [t] ; [t] aphasemeter [vo]'``
+          Play audio track 1, and visualize it as video using the ``aphasemeter``
+          filter.
+        - ``--lavfi-complex='[vid1] [vid2] vstack [vo]'``
+          Stack video track 1 and 2 and play them at the same time. Note that
+          both tracks need to have the same width, or filter initialization
+          will fail (you can add ``scale`` filters before the ``vstack`` filter
+          to fix the size).
+        - ``--lavfi-complex='[aid1] asplit [ao] [t] ; [t] aphasemeter [t2] ; [vid1] [t2] overlay [vo]'``
+          Play audio track 1, and overlay its visualization over video track 1.
+
+    See the Ffmpeg libavfilter documentation for details on the filter.
+
+

options/options.c

@@ -226,6 +226,8 @@ const m_option_t mp_opts[] = {
     OPT_STRINGLIST("alang", stream_lang[STREAM_AUDIO], 0),
     OPT_STRINGLIST("slang", stream_lang[STREAM_SUB], 0),
 
+    OPT_STRING("lavfi-complex", lavfi_complex, 0),
+
     OPT_CHOICE("audio-display", audio_display, 0,
                ({"no", 0}, {"attachment", 1})),
 

options/options.h

@@ -186,6 +186,7 @@ typedef struct MPOpts {
     int ignore_path_in_watch_later_config;
     int pause;
     int keep_open;
+    char *lavfi_complex;
     int stream_id[2][STREAM_TYPE_COUNT];
     int stream_id_ff[STREAM_TYPE_COUNT];
     char **stream_lang[STREAM_TYPE_COUNT];

player/audio.c

@@ -49,6 +49,7 @@ enum {
     AD_EOF = -2,
     AD_NEW_FMT = -3,
     AD_WAIT = -4,
+    AD_NO_PROGRESS = -5,
 };
 
 // Use pitch correction only for speed adjustments by the user, not minor sync
@@ -204,6 +205,18 @@ void uninit_audio_out(struct MPContext *mpctx)
 
 static void ao_chain_uninit(struct ao_chain *ao_c)
 {
+    struct track *track = ao_c->track;
+    if (track) {
+        assert(track->ao_c == ao_c);
+        track->ao_c = NULL;
+        assert(track->d_audio == ao_c->audio_src);
+        track->d_audio = NULL;
+        audio_uninit(ao_c->audio_src);
+    }
+
+    if (ao_c->filter_src)
+        lavfi_set_connected(ao_c->filter_src, false);
+
     af_destroy(ao_c->af);
     talloc_free(ao_c->input_frame);
     talloc_free(ao_c->ao_buffer);
@@ -213,18 +226,10 @@ static void ao_chain_uninit(struct ao_chain *ao_c)
 void uninit_audio_chain(struct MPContext *mpctx)
 {
     if (mpctx->ao_chain) {
-        struct track *track = mpctx->current_track[0][STREAM_AUDIO];
-        assert(track);
-        assert(track->d_audio == mpctx->ao_chain->audio_src);
-
         mixer_uninit_audio(mpctx->mixer);
-
-        audio_uninit(track->d_audio);
-        track->d_audio = NULL;
-        mpctx->ao_chain->audio_src = NULL;
-
         ao_chain_uninit(mpctx->ao_chain);
         mpctx->ao_chain = NULL;
+
         mpctx->audio_status = STATUS_EOF;
         reselect_demux_streams(mpctx);
 
@@ -379,6 +384,9 @@ int init_audio_decoder(struct MPContext *mpctx, struct track *track)
     return 1;
 
 init_error:
+    if (track->sink)
+        lavfi_set_connected(track->sink, false);
+    track->sink = NULL;
     audio_uninit(track->d_audio);
     track->d_audio = NULL;
     error_on_track(mpctx, track);
@@ -387,31 +395,46 @@ int init_audio_decoder(struct MPContext *mpctx, struct track *track)
 
 void reinit_audio_chain(struct MPContext *mpctx)
 {
-    assert(!mpctx->ao_chain);
+    reinit_audio_chain_src(mpctx, NULL);
+}
 
-    struct track *track = mpctx->current_track[0][STREAM_AUDIO];
-    struct sh_stream *sh = track ? track->stream : NULL;
-    if (!sh) {
-        uninit_audio_out(mpctx);
-        goto no_audio;
+void reinit_audio_chain_src(struct MPContext *mpctx, struct lavfi_pad *src)
+{
+    struct track *track = NULL;
+    struct sh_stream *sh = NULL;
+    if (!src) {
+        track = mpctx->current_track[0][STREAM_AUDIO];
+        if (!track)
+            return;
+        sh = track ? track->stream : NULL;
+        if (!sh) {
+            uninit_audio_out(mpctx);
+            goto no_audio;
+        }
     }
+    assert(!mpctx->ao_chain);
 
     mp_notify(mpctx, MPV_EVENT_AUDIO_RECONFIG, NULL);
 
     struct ao_chain *ao_c = ao_c = talloc_zero(NULL, struct ao_chain);
     mpctx->ao_chain = ao_c;
     ao_c->log = mpctx->log;
     ao_c->af = af_new(mpctx->global);
-    ao_c->af->replaygain_data = sh->codec->replaygain_data;
+    if (sh)
+        ao_c->af->replaygain_data = sh->codec->replaygain_data;
     ao_c->spdif_passthrough = true;
     ao_c->pts = MP_NOPTS_VALUE;
     ao_c->ao_buffer = mp_audio_buffer_create(NULL);
     ao_c->ao = mpctx->ao;
 
-    if (!init_audio_decoder(mpctx, track))
-        goto init_error;
-
-    ao_c->audio_src = track->d_audio;
+    ao_c->filter_src = src;
+    if (!ao_c->filter_src) {
+        ao_c->track = track;
+        track->ao_c = ao_c;
+        if (!init_audio_decoder(mpctx, track))
+            goto init_error;
+        ao_c->audio_src = track->d_audio;
+    }
 
     reset_audio_state(mpctx);
 
@@ -597,15 +620,18 @@ static int decode_new_frame(struct ao_chain *ao_c)
     if (ao_c->input_frame)
         return AD_OK;
 
-    int res = DATA_AGAIN;
-    while (res == DATA_AGAIN) {
+    int res = DATA_EOF;
+    if (ao_c->filter_src) {
+        res = lavfi_request_frame_a(ao_c->filter_src, &ao_c->input_frame);
+    } else if (ao_c->audio_src) {
         audio_work(ao_c->audio_src);
         res = audio_get_frame(ao_c->audio_src, &ao_c->input_frame);
     }
 
     switch (res) {
     case DATA_OK:       return AD_OK;
     case DATA_WAIT:     return AD_WAIT;
+    case DATA_AGAIN:    return AD_NO_PROGRESS;
     case DATA_EOF:      return AD_EOF;
     default:            abort();
     }
@@ -633,6 +659,8 @@ static int filter_audio(struct ao_chain *ao_c, struct mp_audio_buffer *outbuf,
             break;
 
         res = decode_new_frame(ao_c);
+        if (res == AD_NO_PROGRESS)
+            break;
         if (res < 0) {
             // drain filters first (especially for true EOF case)
             copy_output(afs, outbuf, minsamples, true);
@@ -764,6 +792,10 @@ void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
         status = filter_audio(mpctx->ao_chain, ao_c->ao_buffer, playsize);
         if (status == AD_WAIT)
             return;
+        if (status == AD_NO_PROGRESS) {
+            mpctx->sleeptime = 0;
+            return;
+        }
         if (status == AD_NEW_FMT) {
             /* The format change isn't handled too gracefully. A more precise
              * implementation would require draining buffered old-format audio

player/core.h

@@ -31,6 +31,8 @@
 #include "video/mp_image.h"
 #include "video/out/vo.h"
 
+#include "lavfi.h"
+
 // definitions used internally by the core player code
 
 enum stop_play_reason {
@@ -149,6 +151,11 @@ struct track {
     struct dec_video *d_video;
     struct dec_audio *d_audio;
 
+    // Where the decoded result goes to (one of them is not NULL if active)
+    struct vo_chain *vo_c;
+    struct ao_chain *ao_c;
+    struct lavfi_pad *sink;
+
     // For external subtitles, which are read fully on init. Do not attempt
     // to read packets from them.
     bool preloaded;
@@ -170,6 +177,8 @@ struct vo_chain {
     // Last known input_mpi format (so vf can be reinitialized any time).
     struct mp_image_params input_format;
 
+    struct track *track;
+    struct lavfi_pad *filter_src;
     struct dec_video *video_src;
 
     // - video consists of a single picture, which should be shown only once
@@ -195,6 +204,8 @@ struct ao_chain {
     // Last known input_mpi format (so vf can be reinitialized any time).
     struct mp_audio input_format;
 
+    struct track *track;
+    struct lavfi_pad *filter_src;
     struct dec_audio *audio_src;
 };
 
@@ -294,6 +305,8 @@ typedef struct MPContext {
     // Currently, this is used for the secondary subtitle track only.
     struct track *current_track[NUM_PTRACKS][STREAM_TYPE_COUNT];
 
+    struct lavfi *lavfi;
+
     // Uses: accessing metadata (consider ordered chapters case, where the main
     // demuxer defines metadata), or special purpose demuxers like TV.
     struct demuxer *master_demuxer;
@@ -438,6 +451,8 @@ void clear_audio_output_buffers(struct MPContext *mpctx);
 void update_playback_speed(struct MPContext *mpctx);
 void uninit_audio_out(struct MPContext *mpctx);
 void uninit_audio_chain(struct MPContext *mpctx);
+int init_audio_decoder(struct MPContext *mpctx, struct track *track);
+void reinit_audio_chain_src(struct MPContext *mpctx, struct lavfi_pad *src);
 
 // configfiles.c
 void mp_parse_cfgfiles(struct MPContext *mpctx);
@@ -558,11 +573,13 @@ int video_vf_vo_control(struct vo_chain *vo_c, int vf_cmd, void *data);
 void reset_video_state(struct MPContext *mpctx);
 int init_video_decoder(struct MPContext *mpctx, struct track *track);
 int reinit_video_chain(struct MPContext *mpctx);
+int reinit_video_chain_src(struct MPContext *mpctx, struct lavfi_pad *src);
 int reinit_video_filters(struct MPContext *mpctx);
 void write_video(struct MPContext *mpctx, double endpts);
 void mp_force_video_refresh(struct MPContext *mpctx);
 void uninit_video_out(struct MPContext *mpctx);
 void uninit_video_chain(struct MPContext *mpctx);
 double calc_average_frame_duration(struct MPContext *mpctx);
+int init_video_decoder(struct MPContext *mpctx, struct track *track);
 
 #endif /* MPLAYER_MP_CORE_H */