video: rewrite filtering glue code

Get rid of the old vf.c code. Replace it with a generic filtering
framework, which can potentially handle more than just --vf. At least
reimplementing --af with this code is planned.

This changes some --vf semantics (including runtime behavior and the
"vf" command). The most important ones are listed in interface-changes.

vf_convert.c is renamed to f_swscale.c. It is now an internal filter
that can not be inserted by the user manually.

f_lavfi.c is a refactor of player/lavfi.c. The latter will be removed
once --lavfi-complex is reimplemented on top of f_lavfi.c. (which is
conceptually easy, but a big mess due to the data flow changes).

The existing filters are all changed heavily. The data flow of the new
filter framework is different. Especially EOF handling changes - EOF is
now a "frame" rather than a state, and must be passed through exactly
once.

Another major thing is that all filters must support dynamic format
changes. The filter reconfig() function goes away. (This sounds complex,
but since all filters need to handle EOF draining anyway, they can use
the same code, and it removes the mess with reconfig() having to predict
the output format, which completely breaks with libavfilter anyway.)

In addition, there is no automatic format negotiation or conversion.
libavfilter's primitive and insufficient API simply doesn't allow us to
do this in a reasonable way. Instead, filters can use f_autoconvert as
sub-filter, and tell it which formats they support. This filter will in
turn add actual conversion filters, such as f_swscale, to perform
necessary format changes.

vf_vapoursynth.c uses the same basic principle of operation as before,
but with worryingly different details in data flow. Still appears to
work.

The hardware deint filters (vf_vavpp.c, vf_d3d11vpp.c, vf_vdpaupp.c) are
heavily changed. Fortunately, they all used refqueue.c, which is for
sharing the data flow logic (especially for managing future/past
surfaces and such). It turns out it can be used to factor out most of
the data flow. Some of these filters accepted software input. Instead of
having ad-hoc upload code in each filter, surface upload is now
delegated to f_autoconvert, which can use f_hwupload to perform this.

Exporting VO capabilities is still a big mess (mp_stream_info stuff).

The D3D11 code drops the redundant image formats, and all code uses the
hw_subfmt (sw_format in FFmpeg) instead. Although that too seems to be a
big mess for now.

f_async_queue is unused.

21 files changed, 4632 insertions, 0 deletions

diff --git a/filters/f_auto_filters.c b/filters/f_auto_filters.c
new file mode 100644
index 0000000000..eac6f745ca
--- /dev/null
+++ b/filters/f_auto_filters.c
@@ -0,0 +1,244 @@
+#include <math.h>
+
+#include "common/common.h"
+#include "common/msg.h"
+#include "options/m_config.h"
+#include "options/options.h"
+#include "video/mp_image.h"
+
+#include "f_auto_filters.h"
+#include "f_swscale.h"
+#include "f_utils.h"
+#include "filter.h"
+#include "filter_internal.h"
+#include "user_filters.h"
+
+struct deint_priv {
+    struct mp_subfilter sub;
+    int prev_imgfmt;
+    int prev_setting;
+    struct m_config_cache *opts;
+};
+
+static void deint_process(struct mp_filter *f)
+{
+    struct deint_priv *p = f->priv;
+
+    if (!mp_subfilter_read(&p->sub))
+        return;
+
+    struct mp_frame frame = p->sub.frame;
+
+    if (mp_frame_is_signaling(frame)) {
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (frame.type != MP_FRAME_VIDEO) {
+        MP_ERR(f, "video input required!\n");
+        mp_filter_internal_mark_failed(f);
+        return;
+    }
+
+    m_config_cache_update(p->opts);
+    struct filter_opts *opts = p->opts->opts;
+
+    if (!opts->deinterlace)
+        mp_subfilter_destroy(&p->sub);
+
+    struct mp_image *img = frame.data;
+
+    if (img->imgfmt == p->prev_imgfmt && p->prev_setting == opts->deinterlace) {
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (!mp_subfilter_drain_destroy(&p->sub))
+        return;
+
+    assert(!p->sub.filter);
+
+    p->prev_imgfmt = img->imgfmt;
+    p->prev_setting = opts->deinterlace;
+    if (!p->prev_setting) {
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (img->imgfmt == IMGFMT_VDPAU) {
+        char *args[] = {"deint", "yes", NULL};
+        p->sub.filter =
+            mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "vdpaupp", args);
+    } else if (img->imgfmt == IMGFMT_VAAPI) {
+        p->sub.filter =
+            mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "vavpp", NULL);
+    } else if (img->imgfmt == IMGFMT_D3D11) {
+        p->sub.filter =
+            mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "d3d11vpp", NULL);
+    } else if (mp_sws_supports_input(img->imgfmt)) {
+        char *args[] = {"mode", "send_field", "deint", "interlaced", NULL};
+        p->sub.filter =
+            mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "yadif", args);
+    } else {
+        MP_ERR(f, "no deinterlace filter available for this format\n");
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (!p->sub.filter)
+        MP_ERR(f, "creating deinterlacer failed\n");
+
+    mp_subfilter_continue(&p->sub);
+}
+
+static void deint_reset(struct mp_filter *f)
+{
+    struct deint_priv *p = f->priv;
+
+    mp_subfilter_reset(&p->sub);
+}
+
+static void deint_destroy(struct mp_filter *f)
+{
+    struct deint_priv *p = f->priv;
+
+    mp_subfilter_reset(&p->sub);
+    TA_FREEP(&p->sub.filter);
+}
+
+static const struct mp_filter_info deint_filter = {
+    .name = "deint",
+    .priv_size = sizeof(struct deint_priv),
+    .process = deint_process,
+    .reset = deint_reset,
+    .destroy = deint_destroy,
+};
+
+struct mp_filter *mp_deint_create(struct mp_filter *parent)
+{
+    struct mp_filter *f = mp_filter_create(parent, &deint_filter);
+    if (!f)
+        return NULL;
+
+    struct deint_priv *p = f->priv;
+
+    p->sub.in = mp_filter_add_pin(f, MP_PIN_IN, "in");
+    p->sub.out = mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    p->opts = m_config_cache_alloc(f, f->global, &filter_conf);
+
+    return f;
+}
+
+struct rotate_priv {
+    struct mp_subfilter sub;
+    int prev_rotate;
+    int prev_imgfmt;
+    int target_rotate;
+};
+
+static void rotate_process(struct mp_filter *f)
+{
+    struct rotate_priv *p = f->priv;
+
+    if (!mp_subfilter_read(&p->sub))
+        return;
+
+    struct mp_frame frame = p->sub.frame;
+
+    if (mp_frame_is_signaling(frame)) {
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (frame.type != MP_FRAME_VIDEO) {
+        MP_ERR(f, "video input required!\n");
+        return;
+    }
+
+    struct mp_image *img = frame.data;
+
+    if (img->params.rotate == p->prev_rotate &&
+        img->imgfmt == p->prev_imgfmt)
+    {
+        img->params.rotate = p->target_rotate;
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (!mp_subfilter_drain_destroy(&p->sub))
+        return;
+
+    assert(!p->sub.filter);
+
+    int rotate = p->prev_rotate = img->params.rotate;
+    p->target_rotate = rotate;
+    p->prev_imgfmt = img->imgfmt;
+
+    struct mp_stream_info *info = mp_filter_find_stream_info(f);
+    if (rotate == 0 || (info && info->rotate90 && !(rotate % 90))) {
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (mp_sws_supports_input(img->imgfmt)) {
+        MP_ERR(f, "Video rotation with this format not supported\n");
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    double angle = rotate / 360.0 * M_PI * 2;
+    char *args[] = {"angle", mp_tprintf(30, "%f", angle),
+                    "ow", mp_tprintf(30, "rotw(%f)", angle),
+                    "oh", mp_tprintf(30, "roth(%f)", angle),
+                    NULL};
+    p->sub.filter =
+        mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "rotate", args);
+
+    if (p->sub.filter) {
+        MP_INFO(f, "Inserting rotation filter.\n");
+        p->target_rotate = 0;
+    } else {
+        MP_ERR(f, "could not create rotation filter\n");
+    }
+
+    mp_subfilter_continue(&p->sub);
+}
+
+static void rotate_reset(struct mp_filter *f)
+{
+    struct rotate_priv *p = f->priv;
+
+    mp_subfilter_reset(&p->sub);
+}
+
+static void rotate_destroy(struct mp_filter *f)
+{
+    struct rotate_priv *p = f->priv;
+
+    mp_subfilter_reset(&p->sub);
+    TA_FREEP(&p->sub.filter);
+}
+
+static const struct mp_filter_info rotate_filter = {
+    .name = "autorotate",
+    .priv_size = sizeof(struct rotate_priv),
+    .process = rotate_process,
+    .reset = rotate_reset,
+    .destroy = rotate_destroy,
+};
+
+struct mp_filter *mp_autorotate_create(struct mp_filter *parent)
+{
+    struct mp_filter *f = mp_filter_create(parent, &rotate_filter);
+    if (!f)
+        return NULL;
+
+    struct rotate_priv *p = f->priv;
+    p->prev_rotate = -1;
+
+    p->sub.in = mp_filter_add_pin(f, MP_PIN_IN, "in");
+    p->sub.out = mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    return f;
+}
diff --git a/filters/f_auto_filters.h b/filters/f_auto_filters.h
new file mode 100644
index 0000000000..5f1a99f636
--- /dev/null
+++ b/filters/f_auto_filters.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include "filter.h"
+
+// A filter which inserts the required deinterlacing filter based on the
+// hardware decode mode and the deinterlace user option.
+struct mp_filter *mp_deint_create(struct mp_filter *parent);
+
+// Rotate according to mp_image.rotate and VO capabilities.
+struct mp_filter *mp_autorotate_create(struct mp_filter *parent);
diff --git a/filters/f_autoconvert.c b/filters/f_autoconvert.c
new file mode 100644
index 0000000000..687a846ae5
--- /dev/null
+++ b/filters/f_autoconvert.c
@@ -0,0 +1,288 @@
+#include "config.h"
+
+#include "common/common.h"
+#include "common/msg.h"
+#include "video/hwdec.h"
+#include "video/mp_image.h"
+
+#include "f_autoconvert.h"
+#include "f_hwtransfer.h"
+#include "f_swscale.h"
+#include "f_utils.h"
+#include "filter.h"
+#include "filter_internal.h"
+
+struct priv {
+    struct mp_log *log;
+
+    struct mp_subfilter sub;
+
+    bool force_update;
+
+    int *imgfmts;
+    int *subfmts;
+    int num_imgfmts;
+
+    // Enable special conversion for the final stage before the VO.
+    bool vo_convert;
+
+    // sws state
+    int in_imgfmt, in_subfmt;
+
+    struct mp_autoconvert public;
+};
+
+// Dummy filter for bundling sub-conversion filters.
+static const struct mp_filter_info convert_filter = {
+    .name = "convert",
+};
+
+// For hw decoding: thing which can convert between underlying surface formats.
+// The filter detects the needed target format from struct mp_hwdec_ctx.
+struct subfmt_conv {
+    int hw_imgfmt;
+    struct mp_filter *(*create)(struct mp_filter *parent);
+};
+
+static const struct subfmt_conv subfmt_converters[] = {
+#if HAVE_D3D_HWACCEL
+    {IMGFMT_D3D11, vf_d3d11_create_outconv},
+#endif
+    {0}
+};
+
+void mp_autoconvert_clear(struct mp_autoconvert *c)
+{
+    struct priv *p = c->f->priv;
+
+    p->num_imgfmts = 0;
+}
+
+void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt, int subfmt)
+{
+    struct priv *p = c->f->priv;
+
+    MP_TARRAY_GROW(p, p->imgfmts, p->num_imgfmts);
+    MP_TARRAY_GROW(p, p->subfmts, p->num_imgfmts);
+
+    p->imgfmts[p->num_imgfmts] = imgfmt;
+    p->subfmts[p->num_imgfmts] = subfmt;
+
+    p->num_imgfmts += 1;
+    p->force_update = true;
+}
+
+void mp_autoconvert_add_vo_hwdec_subfmts(struct mp_autoconvert *c,
+                                         struct mp_hwdec_devices *devs)
+{
+    struct priv *p = c->f->priv;
+    assert(devs);
+
+    int prev_format = 0;
+
+    for (int n = 0; ; n++) {
+        struct mp_hwdec_ctx *ctx = hwdec_devices_get_n(devs, n);
+        if (!ctx)
+            break;
+        if (!ctx->hw_imgfmt || !ctx->supported_formats)
+            continue;
+        // Very hacky: don't let d3d11-egl-rgb overwrite d3d11-egl
+        if (ctx->hw_imgfmt == prev_format)
+            continue;
+        prev_format = ctx->hw_imgfmt;
+        // Stupidity: VOs export imgfmt only, so subfmt is always 0. Remove it
+        // to fix it up.
+        for (int i = 0; i < p->num_imgfmts; i++) {
+            if (p->imgfmts[i] != ctx->hw_imgfmt)
+                continue;
+
+            int count = p->num_imgfmts;
+            MP_TARRAY_REMOVE_AT(p->imgfmts, count, i);
+            count = p->num_imgfmts;
+            MP_TARRAY_REMOVE_AT(p->subfmts, count, i);
+            p->num_imgfmts -= 1;
+            break;
+        }
+        for (int i = 0; ctx->supported_formats[i]; i++)
+            mp_autoconvert_add_imgfmt(c, ctx->hw_imgfmt, ctx->supported_formats[i]);
+    }
+
+    p->vo_convert = true;
+}
+
+static void handle_video_frame(struct mp_filter *f)
+{
+    struct priv *p = f->priv;
+
+    struct mp_frame frame = p->sub.frame;
+    if (frame.type != MP_FRAME_VIDEO) {
+        MP_ERR(p, "video input required!\n");
+        mp_filter_internal_mark_failed(f);
+        return;
+    }
+
+    struct mp_image *img = frame.data;
+
+    if (p->force_update)
+        p->in_imgfmt = p->in_subfmt = 0;
+
+    if (img->imgfmt == p->in_imgfmt && img->params.hw_subfmt == p->in_subfmt) {
+        mp_subfilter_continue(&p->sub);
+        return;
+    }
+
+    if (!mp_subfilter_drain_destroy(&p->sub)) {
+        p->in_imgfmt = p->in_subfmt = 0;
+        return;
+    }
+
+    p->in_imgfmt = img->params.imgfmt;
+    p->in_subfmt = img->params.hw_subfmt;
+    p->force_update = false;
+
+    bool different_subfmt = false;
+
+    for (int n = 0; n < p->num_imgfmts; n++) {
+        bool samefmt = img->params.imgfmt == p->imgfmts[n];
+        bool samesubffmt = img->params.hw_subfmt == p->subfmts[n];
+        if (samefmt && !samesubffmt)
+            different_subfmt = true;
+        if (samefmt && (samesubffmt || !p->subfmts[n])) {
+            mp_subfilter_continue(&p->sub);
+            return;
+        }
+    }
+
+    struct mp_stream_info *info = mp_filter_find_stream_info(f);
+
+    struct mp_filter *conv = mp_filter_create(f, &convert_filter);
+    mp_filter_add_pin(conv, MP_PIN_IN, "in");
+    mp_filter_add_pin(conv, MP_PIN_OUT, "out");
+
+    struct mp_filter *filters[2] = {0};
+    bool need_sws = true;
+
+    int *fmts = p->imgfmts;
+    int num_fmts = p->num_imgfmts;
+
+    // Source is sw, all targets are hw -> try to upload.
+    bool sw_to_hw = !IMGFMT_IS_HWACCEL(img->imgfmt);
+    for (int n = 0; n < num_fmts; n++)
+        sw_to_hw &= IMGFMT_IS_HWACCEL(fmts[n]);
+
+    if (sw_to_hw && num_fmts > 0) {
+        // We can probably use this! Very lazy and very approximate.
+        struct mp_hwupload *upload = mp_hwupload_create(conv, fmts[0]);
+        if (upload) {
+            MP_INFO(p, "HW-uploading to %s\n", mp_imgfmt_to_name(fmts[0]));
+            filters[1] = upload->f;
+            fmts = upload->upload_fmts;
+            num_fmts = upload->num_upload_fmts;
+        }
+    } else if (p->vo_convert && different_subfmt && info && info->hwdec_devs) {
+        for (int n = 0; subfmt_converters[n].hw_imgfmt; n++) {
+            if (subfmt_converters[n].hw_imgfmt == img->imgfmt) {
+                MP_INFO(p, "Using HW sub-conversion.\n");
+                filters[1] = subfmt_converters[n].create(conv);
+                if (filters[1]) {
+                    need_sws = false;
+                    break;
+                }
+            }
+        }
+    }
+
+    if (need_sws) {
+        // Create a new conversion filter.
+        struct mp_sws_filter *sws = mp_sws_filter_create(conv);
+        if (!sws) {
+            MP_ERR(p, "error creating conversion filter\n");
+            return;
+        }
+
+        int out = mp_sws_find_best_out_format(img->imgfmt,  fmts, num_fmts);
+        if (!out) {
+            MP_ERR(p, "can't find video conversion for %s/%s\n",
+                   mp_imgfmt_to_name(img->imgfmt),
+                   mp_imgfmt_to_name(img->params.hw_subfmt));
+            talloc_free(conv);
+            mp_filter_internal_mark_failed(f);
+            return;
+        }
+
+        if (out == img->imgfmt) {
+            // Can happen if hwupload goes to same format.
+            talloc_free(sws->f);
+        } else {
+            sws->out_format = out;
+            MP_INFO(p, "Converting %s -> %s\n", mp_imgfmt_to_name(img->imgfmt),
+                    mp_imgfmt_to_name(sws->out_format));
+            filters[0] = sws->f;
+        }
+    }
+
+    mp_chain_filters(conv->ppins[0], conv->ppins[1], filters, 2);
+
+    p->sub.filter = conv;
+    mp_subfilter_continue(&p->sub);
+}
+
+static void process(struct mp_filter *f)
+{
+    struct priv *p = f->priv;
+
+    if (!mp_subfilter_read(&p->sub))
+        return;
+
+    struct mp_frame frame = p->sub.frame;
+
+    if (!mp_frame_is_signaling(frame)) {
+        if (p->num_imgfmts) {
+            handle_video_frame(f);
+            return;
+        }
+    }
+
+    mp_subfilter_continue(&p->sub);
+}
+
+static void reset(struct mp_filter *f)
+{
+    struct priv *p = f->priv;
+
+    mp_subfilter_reset(&p->sub);
+}
+
+static void destroy(struct mp_filter *f)
+{
+    struct priv *p = f->priv;
+
+    mp_subfilter_reset(&p->sub);
+    TA_FREEP(&p->sub.filter);
+}
+
+static const struct mp_filter_info autoconvert_filter = {
+    .name = "autoconvert",
+    .priv_size = sizeof(struct priv),
+    .process = process,
+    .reset = reset,
+    .destroy = destroy,
+};
+
+struct mp_autoconvert *mp_autoconvert_create(struct mp_filter *parent)
+{
+    struct mp_filter *f = mp_filter_create(parent, &autoconvert_filter);
+    if (!f)
+        return NULL;
+
+    mp_filter_add_pin(f, MP_PIN_IN, "in");
+    mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    struct priv *p = f->priv;
+    p->public.f = f;
+    p->log = f->log;
+    p->sub.in = f->ppins[0];
+    p->sub.out = f->ppins[1];
+
+    return &p->public;
+}
diff --git a/filters/f_autoconvert.h b/filters/f_autoconvert.h
new file mode 100644
index 0000000000..72af21a0df
--- /dev/null
+++ b/filters/f_autoconvert.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "filter.h"
+
+// A filter which automatically creates and uses a conversion filter based on
+// the filter settings, or passes through data unchanged if no conversion is
+// required.
+struct mp_autoconvert {
+    // f->pins[0] is input, f->pins[1] is output
+    struct mp_filter *f;
+};
+
+// (to free this, free the filter itself, mp_autoconvert.f)
+struct mp_autoconvert *mp_autoconvert_create(struct mp_filter *parent);
+
+// Add the imgfmt as allowed video image format, and error on non-video frames.
+// Each call adds to the list of allowed formats. Before the first call, all
+// formats are allowed (even non-video).
+// subfmt can be used to specify underlying surface formats for hardware formats,
+// otherwise must be 0.
+void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt, int subfmt);
+
+// Add the formats supported by the hwdec interops (or essentially refine them),
+// and trigger conversion if hw_subfmts mismatch. This is mostly a hack for
+// D3D11/ANGLE (which supports NV12 only).
+// Must be called mp_autoconvert_add_imgfmt(), and overrides them where formats
+// collide.
+struct mp_hwdec_devices;
+void mp_autoconvert_add_vo_hwdec_subfmts(struct mp_autoconvert *c,
+                                         struct mp_hwdec_devices *devs);
+
+// Reset set of allowed formats back to initial state. (This does not flush
+// any frames or remove currently active filters, although to get reasonable
+// behavior, you need to readd all previously allowed formats, or reset the
+// filter.)
+void mp_autoconvert_clear(struct mp_autoconvert *c);
+
+// vf_d3d11vpp.c
+struct mp_filter *vf_d3d11_create_outconv(struct mp_filter *parent);
diff --git a/filters/f_hwtransfer.c b/filters/f_hwtransfer.c
new file mode 100644
index 0000000000..6ffda567ae
--- /dev/null
+++ b/filters/f_hwtransfer.c
@@ -0,0 +1,299 @@
+#include <libavutil/buffer.h>
+#include <libavutil/hwcontext.h>
+#include <libavutil/mem.h>
+
+#include "video/fmt-conversion.h"
+#include "video/hwdec.h"
+#include "video/mp_image.h"
+#include "video/mp_image_pool.h"
+
+#include "f_hwtransfer.h"
+#include "filter_internal.h"
+
+struct priv {
+    AVBufferRef *av_device_ctx;
+
+    AVBufferRef *hw_pool;
+
+    int last_input_fmt;
+    int last_upload_fmt;
+    int last_sw_fmt;
+
+    struct mp_hwupload public;
+};
+
+static bool update_format_decision(struct priv *p, int input_fmt)
+{
+    struct mp_hwupload *u = &p->public;
+
+    if (!input_fmt)
+        return false;
+
+    if (input_fmt == p->last_input_fmt)
+        return true;
+
+    p->last_input_fmt = 0;
+
+    int res = mp_imgfmt_select_best_list(u->upload_fmts, u->num_upload_fmts,
+                                         input_fmt);
+
+    if (!res)
+        return false;
+
+    // Find which sw format we should use.
+    // NOTE: if there are ever any hw APIs that actually do expensive
+    // conversions on mismatching format uploads, we should probably first look
+    // which sw format is preferred?
+    int index = -1;
+    for (int n = 0; n < u->num_upload_fmts; n++) {
+        if (u->upload_fmts[n] == res)
+            index = n;
+    }
+
+    if (index < 0)
+        return false;
+
+    for (int n = 0; n < u->num_fmts; n++) {
+        if (u->fmt_upload_index[n] >= index &&
+            index  < u->fmt_upload_index[n] + u->fmt_upload_num[n])
+        {
+            p->last_input_fmt = input_fmt;
+            p->last_upload_fmt = u->upload_fmts[index];
+            p->last_sw_fmt = u->fmts[n];
+            MP_INFO(u->f, "upload %s -> %s\n",
+                    mp_imgfmt_to_name(p->last_sw_fmt),
+                    mp_imgfmt_to_name(p->last_input_fmt));
+            return true;
+        }
+    }
+
+    return false;
+}
+
+int mp_hwupload_find_upload_format(struct mp_hwupload *u, int imgfmt)
+{
+    struct priv *p = u->f->priv;
+
+    if (!update_format_decision(p, imgfmt))
+        return 0;
+    return p->last_upload_fmt;
+}
+
+static void process(struct mp_filter *f)
+{
+    struct priv *p = f->priv;
+
+    if (!mp_pin_can_transfer_data(f->ppins[1], f->ppins[0]))
+        return;
+
+    struct mp_frame frame = mp_pin_out_read(f->ppins[0]);
+    if (mp_frame_is_signaling(frame)) {
+        mp_pin_in_write(f->ppins[1], frame);
+        return;
+    }
+    if (frame.type != MP_FRAME_VIDEO) {
+        MP_ERR(f, "unsupported frame type\n");
+        goto error;
+    }
+    struct mp_image *src = frame.data;
+
+    // As documented, just pass though HW frames.
+    if (IMGFMT_IS_HWACCEL(src->imgfmt)) {
+        mp_pin_in_write(f->ppins[1], frame);
+        return;
+    }
+
+    if (src->w % 2 || src->h % 2) {
+        MP_ERR(f, "non-mod 2 input frames unsupported\n");
+        goto error;
+    }
+
+    if (!update_format_decision(p, src->imgfmt)) {
+        MP_ERR(f, "no hw upload format found\n");
+        goto error;
+    }
+
+    if (!mp_update_av_hw_frames_pool(&p->hw_pool, p->av_device_ctx,
+                                     p->public.hw_imgfmt, p->last_sw_fmt,
+                                     src->w, src->h))
+    {
+        MP_ERR(f, "failed to create frame pool\n");
+        goto error;
+    }
+
+    struct mp_image *dst = mp_av_pool_image_hw_upload(p->hw_pool, src);
+    if (!dst)
+        goto error;
+
+    mp_frame_unref(&frame);
+    mp_pin_in_write(f->ppins[1], MAKE_FRAME(MP_FRAME_VIDEO, dst));
+
+    return;
+
+error:
+    mp_frame_unref(&frame);
+    MP_ERR(f, "failed to upload frame\n");
+    mp_filter_internal_mark_failed(f);
+}
+
+static void destroy(struct mp_filter *f)
+{
+    struct priv *p = f->priv;
+
+    av_buffer_unref(&p->hw_pool);
+    av_buffer_unref(&p->av_device_ctx);
+}
+
+static const struct mp_filter_info hwupload_filter = {
+    .name = "hwupload",
+    .priv_size = sizeof(struct priv),
+    .process = process,
+    .destroy = destroy,
+};
+
+// The VO layer might have restricted format support. It might actually
+// work if this is input to a conversion filter anyway, but our format
+// negotiation is too stupid and non-existent to detect this.
+// So filter out all not explicitly supported formats.
+static bool vo_supports(struct mp_hwdec_ctx *ctx, int hw_fmt, int sw_fmt)
+{
+    if (!ctx->hw_imgfmt)
+        return true; // if unset, all formats are allowed
+    if (ctx->hw_imgfmt != hw_fmt)
+        return false;
+
+    for (int i = 0; ctx->supported_formats &&  ctx->supported_formats[i]; i++) {
+        if (ctx->supported_formats[i] == sw_fmt)
+            return true;
+    }
+
+    return false;
+}
+
+static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
+{
+    struct priv *p = u->f->priv;
+
+    u->hw_imgfmt = hw_imgfmt;
+    u->num_fmts = 0;
+    u->num_upload_fmts = 0;
+
+    struct mp_stream_info *info = mp_filter_find_stream_info(u->f);
+    if (!info || !info->hwdec_devs) {
+        MP_ERR(u->f, "no hw context\n");
+        return false;
+    }
+
+    struct mp_hwdec_ctx *ctx = NULL;
+    AVHWFramesConstraints *cstr = NULL;
+
+    for (int n = 0; ; n++) {
+        struct mp_hwdec_ctx *cur = hwdec_devices_get_n(info->hwdec_devs, n);
+        if (!cur)
+            break;
+        if (!cur->av_device_ref)
+            continue;
+        cstr = av_hwdevice_get_hwframe_constraints(cur->av_device_ref, NULL);
+        if (!cstr)
+            continue;
+        bool found = false;
+        for (int i = 0; cstr->valid_hw_formats &&
+                        cstr->valid_hw_formats[i] != AV_PIX_FMT_NONE; i++)
+        {
+            found |= cstr->valid_hw_formats[i] == imgfmt2pixfmt(hw_imgfmt);
+        }
+        if (found && (!cur->hw_imgfmt || cur->hw_imgfmt == hw_imgfmt)) {
+            ctx = cur;
+            break;
+        }
+        av_hwframe_constraints_free(&cstr);
+    }
+
+    if (!ctx) {
+        MP_ERR(u->f, "no support for this hw format\n");
+        return false;
+    }
+
+    // Probe for supported formats. This is very roundabout, because the
+    // hwcontext API does not give us this information directly. We resort to
+    // creating temporary AVHWFramesContexts in order to retrieve the list of
+    // supported formats. This should be relatively cheap as we don't create
+    // any real frames (although some backends do for probing info).
+
+    for (int n = 0; cstr->valid_sw_formats &&
+                    cstr->valid_sw_formats[n] != AV_PIX_FMT_NONE; n++)
+    {
+        int imgfmt = pixfmt2imgfmt(cstr->valid_sw_formats[n]);
+        if (!imgfmt)
+            continue;
+
+        MP_VERBOSE(u->f, "looking at format %s\n", mp_imgfmt_to_name(imgfmt));
+
+        // Creates an AVHWFramesContexts with the given parameters.
+        AVBufferRef *frames = NULL;
+        if (!mp_update_av_hw_frames_pool(&frames, ctx->av_device_ref,
+                                        hw_imgfmt, imgfmt, 128, 128))
+        {
+            MP_WARN(u->f, "failed to allocate pool\n");
+            continue;
+        }
+
+        enum AVPixelFormat *fmts;
+        if (av_hwframe_transfer_get_formats(frames,
+                            AV_HWFRAME_TRANSFER_DIRECTION_TO, &fmts, 0) >= 0)
+        {
+            int index = u->num_fmts;
+            MP_TARRAY_APPEND(p, u->fmts, u->num_fmts, imgfmt);
+            MP_TARRAY_GROW(p, u->fmt_upload_index, index);
+            MP_TARRAY_GROW(p, u->fmt_upload_num, index);
+
+            u->fmt_upload_index[index] = u->num_upload_fmts;
+
+            for (int i = 0; fmts[i] != AV_PIX_FMT_NONE; i++) {
+                int fmt = pixfmt2imgfmt(fmts[i]);
+                if (!fmt)
+                    continue;
+                MP_VERBOSE(u->f, "supports %s\n", mp_imgfmt_to_name(fmt));
+                if (vo_supports(ctx, hw_imgfmt, fmt))
+                    MP_TARRAY_APPEND(p, u->upload_fmts, u->num_upload_fmts, fmt);
+            }
+
+            u->fmt_upload_num[index] =
+                u->num_upload_fmts - u->fmt_upload_index[index];
+
+            av_free(fmts);
+        }
+
+        av_buffer_unref(&frames);
+    }
+
+    p->av_device_ctx = av_buffer_ref(ctx->av_device_ref);
+    if (!p->av_device_ctx)
+        return false;
+
+    return u->num_upload_fmts > 0;
+}
+
+struct mp_hwupload *mp_hwupload_create(struct mp_filter *parent, int hw_imgfmt)
+{
+    struct mp_filter *f = mp_filter_create(parent, &hwupload_filter);
+    if (!f)
+        return NULL;
+
+    struct priv *p = f->priv;
+    struct mp_hwupload *u = &p->public;
+    u->f = f;
+
+    mp_filter_add_pin(f, MP_PIN_IN, "in");
+    mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    if (!probe_formats(u, hw_imgfmt)) {
+        MP_ERR(f, "hardware format not supported\n");
+        goto error;
+    }
+
+    return u;
+error:
+    talloc_free(f);
+    return NULL;
+}
diff --git a/filters/f_hwtransfer.h b/filters/f_hwtransfer.h
new file mode 100644
index 0000000000..4595cb393d
--- /dev/null
+++ b/filters/f_hwtransfer.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include "filter.h"
+
+// A filter which uploads sw frames to hw. Ignores hw frames.
+struct mp_hwupload {
+    struct mp_filter *f;
+
+    // Hardware wrapper format, e.g. IMGFMT_VAAPI.
+    int hw_imgfmt;
+
+    // List of supported underlying surface formats.
+    int *fmts;
+    int num_fmts;
+    // List of supported upload image formats. May contain duplicate entries
+    // (which should be ignored).
+    int *upload_fmts;
+    int num_upload_fmts;
+    // For fmts[n], fmt_upload_index[n] gives the index of the first supported
+    // upload format in upload_fmts[], and fmt_upload_num[n] gives the number
+    // of formats at this position.
+    int *fmt_upload_index;
+    int *fmt_upload_num;
+};
+
+struct mp_hwupload *mp_hwupload_create(struct mp_filter *parent, int hw_imgfmt);
+
+// Return the best format suited for upload that is supported for a given input
+// imgfmt. This returns the same as imgfmt if the format is natively supported,
+// and otherwise a format that likely results in the least loss.
+// Returns 0 if completely unsupported.
+int mp_hwupload_find_upload_format(struct mp_hwupload *u, int imgfmt);
diff --git a/filters/f_lavfi.c b/filters/f_lavfi.c
new file mode 100644
index 0000000000..a97f126efb
--- /dev/null
+++ b/filters/f_lavfi.c
@@ -0,0 +1,952 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <inttypes.h>
+#include <stdarg.h>
+#include <assert.h>
+
+#include <libavutil/avstring.h>
+#include <libavutil/mem.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/rational.h>
+#include <libavutil/error.h>
+#include <libavutil/opt.h>
+#include <libavfilter/avfilter.h>
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+
+#include "common/common.h"
+#include "common/av_common.h"
+#include "common/tags.h"
+#include "common/msg.h"
+
+#include "audio/format.h"
+#include "audio/aframe.h"
+#include "video/mp_image.h"
+#include "audio/fmt-conversion.h"
+#include "video/fmt-conversion.h"
+#include "video/hwdec.h"
+
+#include "f_lavfi.h"
+#include "filter.h"
+#include "filter_internal.h"
+#include "user_filters.h"
+
+#if LIBAVFILTER_VERSION_MICRO < 100
+#define av_buffersink_get_frame_flags(a, b, c) av_buffersink_get_frame(a, b)
+#define AV_BUFFERSINK_FLAG_NO_REQUEST 0
+#endif
+
+struct lavfi {
+    struct mp_log *log;
+    struct mp_filter *f;
+
+    char *graph_string;
+    char **graph_opts;
+    bool force_bidir;
+    enum mp_frame_type force_type;
+    bool direct_filter;
+    char **direct_filter_opts;
+
+    AVFilterGraph *graph;
+    // Set to true once all inputs have been initialized, and the graph is
+    // linked.
+    bool initialized;
+
+    // Graph is draining to either handle format changes (if input format
+    // changes for one pad, recreate the graph after draining all buffered
+    // frames), or undo previously sent EOF (libavfilter does not accept
+    // input anymore after sending EOF, so recreate the graph to "unstuck" it).
+    bool draining_recover;
+
+    // Filter can't be put into a working state.
+    bool failed;
+
+    struct lavfi_pad **in_pads;
+    int num_in_pads;
+
+    struct lavfi_pad **out_pads;
+    int num_out_pads;
+
+    struct lavfi_pad **all_pads;
+    int num_all_pads;
+
+    AVFrame *tmp_frame;
+
+    struct mp_lavfi public;
+};
+
+struct lavfi_pad {
+    struct lavfi *main;
+    enum mp_frame_type type;
+    enum mp_pin_dir dir;
+    char *name; // user-given pad name
+
+    struct mp_pin *pin; // internal pin for this (never NULL once initialized)
+    int pin_index;
+
+    AVFilterContext *filter;
+    int filter_pad;
+    // buffersrc or buffersink connected to filter/filter_pad
+    AVFilterContext *buffer;
+    AVRational timebase;
+    bool buffer_is_eof; // received/sent EOF to the buffer
+
+    struct mp_tags *metadata;
+
+    // 1-frame queue for input.
+    struct mp_frame pending;
+
+    // Used to check input format changes.
+    struct mp_frame in_fmt;
+};
+
+// Free the libavfilter graph (not c), reset all state.
+// Does not free pending data intentionally.
+static void free_graph(struct lavfi *c)
+{
+    avfilter_graph_free(&c->graph);
+    for (int n = 0; n < c->num_all_pads; n++) {
+        struct lavfi_pad *pad = c->all_pads[n];
+
+        pad->filter = NULL;
+        pad->filter_pad = -1;
+        pad->buffer = NULL;
+        mp_frame_unref(&pad->in_fmt);
+        pad->buffer_is_eof = false;
+    }
+    c->initialized = false;
+    c->draining_recover = false;
+}
+
+static void add_pad(struct lavfi *c, int dir, int index, AVFilterContext *filter,
+                    int filter_pad, const char *name, bool first_init)
+{
+    if (c->failed)
+        return;
+
+    enum AVMediaType avmt;
+    if (dir == MP_PIN_IN) {
+        avmt = avfilter_pad_get_type(filter->input_pads, filter_pad);
+    } else {
+        avmt = avfilter_pad_get_type(filter->output_pads, filter_pad);
+    }
+    int type;
+    switch (avmt) {
+    case AVMEDIA_TYPE_VIDEO: type = MP_FRAME_VIDEO; break;
+    case AVMEDIA_TYPE_AUDIO: type = MP_FRAME_AUDIO; break;
+    default:
+        MP_FATAL(c, "unknown media type\n");
+        c->failed = true;
+        return;
+    }
+
+    // For anonymous pads, just make something up. libavfilter allows duplicate
+    // pad names (while we don't), so we check for collisions along with normal
+    // duplicate pads below.
+    char tmp[80];
+    const char *dir_string = dir == MP_PIN_IN ? "in" : "out";
+    if (name) {
+        if (c->direct_filter) {
+            // libavfilter has this very unpleasant thing that filter labels
+            // don't have to be unique - in particular, both input and output
+            // are usually named "default". With direct filters, the user has
+            // no chance to provide better names, so do something to resolve it.
+            snprintf(tmp, sizeof(tmp), "%s_%s", name, dir_string);
+            name = tmp;
+        }
+    } else {
+        snprintf(tmp, sizeof(tmp), "%s%d", dir_string, index);
+        name = tmp;
+    }
+
+    struct lavfi_pad *p = NULL;
+    for (int n = 0; n < c->num_all_pads; n++) {
+        if (strcmp(c->all_pads[n]->name, name) == 0) {
+            p = c->all_pads[n];
+            break;
+        }
+    }
+
+    if (p) {
+        // Graph recreation case: reassociate an existing pad.
+        if (p->filter) {
+            // Collision due to duplicate names.
+            MP_FATAL(c, "more than one pad with label '%s'\n", name);
+            c->failed = true;
+            return;
+        }
+        if (p->dir != dir || p->type != type) {
+            // libavfilter graph parser behavior not deterministic.
+            MP_FATAL(c, "pad '%s' changed type or direction\n", name);
+            c->failed = true;
+            return;
+        }
+    } else {
+        if (!first_init) {
+            MP_FATAL(c, "filter pad '%s' got added later?\n", name);
+            c->failed = true;
+            return;
+        }
+        p = talloc_zero(c, struct lavfi_pad);
+        p->main = c;
+        p->dir = dir;
+        p->name = talloc_strdup(p, name);
+        p->type = type;
+        p->pin_index = -1;
+        p->metadata = talloc_zero(p, struct mp_tags);
+        if (p->dir == MP_PIN_IN)
+            MP_TARRAY_APPEND(c, c->in_pads, c->num_in_pads, p);
+        if (p->dir == MP_PIN_OUT)
+            MP_TARRAY_APPEND(c, c->out_pads, c->num_out_pads, p);
+        MP_TARRAY_APPEND(c, c->all_pads, c->num_all_pads, p);
+    }
+    p->filter = filter;
+    p->filter_pad = filter_pad;
+}
+
+static void add_pads(struct lavfi *c, int dir, AVFilterInOut *l, bool first_init)
+{
+    int index = 0;
+    for (; l; l = l->next)
+        add_pad(c, dir, index++, l->filter_ctx, l->pad_idx, l->name, first_init);
+}
+
+static void add_pads_direct(struct lavfi *c, int dir, AVFilterContext *f,
+                            AVFilterPad *pads, int num_pads, bool first_init)
+{
+    for (int n = 0; n < num_pads; n++)
+        add_pad(c, dir, n, f, n, avfilter_pad_get_name(pads, n), first_init);
+}
+
+// Parse the user-provided filter graph, and populate the unlinked filter pads.
+static void precreate_graph(struct lavfi *c, bool first_init)
+{
+    assert(!c->graph);
+
+    c->failed = false;
+
+    c->graph = avfilter_graph_alloc();
+    if (!c->graph)
+        abort();
+
+    if (mp_set_avopts(c->log, c->graph, c->graph_opts) < 0)
+        goto error;
+
+    if (c->direct_filter) {
+        AVFilterContext *filter = avfilter_graph_alloc_filter(c->graph,
+                            avfilter_get_by_name(c->graph_string), "filter");
+        if (!filter) {
+            MP_FATAL(c, "filter '%s' not found or failed to allocate\n",
+                     c->graph_string);
+            goto error;
+        }
+
+        if (mp_set_avopts(c->log, filter->priv, c->direct_filter_opts) < 0)
+            goto error;
+
+        if (avfilter_init_str(filter, NULL) < 0) {
+            MP_FATAL(c, "filter failed to initialize\n");
+            goto error;
+        }
+
+        add_pads_direct(c, MP_PIN_IN, filter, filter->input_pads,
+                        filter->nb_inputs, first_init);
+        add_pads_direct(c, MP_PIN_OUT, filter, filter->output_pads,
+                        filter->nb_outputs, first_init);
+    } else {
+        AVFilterInOut *in = NULL, *out = NULL;
+        if (avfilter_graph_parse2(c->graph, c->graph_string, &in, &out) < 0) {
+            MP_FATAL(c, "parsing the filter graph failed\n");
+            goto error;
+        }
+        add_pads(c, MP_PIN_IN, in, first_init);
+        add_pads(c, MP_PIN_OUT, out, first_init);
+        avfilter_inout_free(&in);
+        avfilter_inout_free(&out);
+    }
+
+    for (int n = 0; n < c->num_all_pads; n++)
+        c->failed |= !c->all_pads[n]->filter;
+
+    if (c->failed)
+        goto error;
+
+    return;
+
+error:
+    free_graph(c);
+    c->failed = true;
+    mp_filter_internal_mark_failed(c->f);
+    return;
+}
+
+// Ensure to send EOF to each input pad, so the graph can be drained properly.
+static void send_global_eof(struct lavfi *c)
+{
+    for (int n = 0; n < c->num_in_pads; n++) {
+        struct lavfi_pad *pad = c->in_pads[n];
+        if (!pad->buffer || pad->buffer_is_eof)
+            continue;
+
+        if (av_buffersrc_add_frame(pad->buffer, NULL) < 0)
+            MP_FATAL(c, "could not send EOF to filter\n");
+
+        pad->buffer_is_eof = true;
+    }
+}
+
+// libavfilter allows changing some parameters on the fly, but not
+// others.
+static bool is_aformat_ok(struct mp_aframe *a, struct mp_aframe *b)
+{
+    struct mp_chmap ca = {0}, cb = {0};
+    mp_aframe_get_chmap(a, &ca);
+    mp_aframe_get_chmap(b, &cb);
+    return mp_chmap_equals(&ca, &cb) &&
+           mp_aframe_get_rate(a) == mp_aframe_get_rate(b) &&
+           mp_aframe_get_format(a) == mp_aframe_get_format(b);
+}
+static bool is_vformat_ok(struct mp_image *a, struct mp_image *b)
+{
+    return a->imgfmt == b->imgfmt &&
+           a->w == b->w && a->h && b->h &&
+           a->params.p_w == b->params.p_w && a->params.p_h == b->params.p_h;
+}
+static bool is_format_ok(struct mp_frame a, struct mp_frame b)
+{
+    if (a.type == b.type && a.type == MP_FRAME_VIDEO)
+        return is_vformat_ok(a.data, b.data);
+    if (a.type == b.type && a.type == MP_FRAME_AUDIO)
+        return is_aformat_ok(a.data, b.data);
+    return false;
+}
+
+static void read_pad_input(struct lavfi *c, struct lavfi_pad *pad)
+{
+    assert(pad->dir == MP_PIN_IN);
+
+    if (pad->pending.type || c->draining_recover)
+        return;
+
+    pad->pending = mp_pin_out_read(pad->pin);
+
+    if (pad->pending.type && pad->pending.type != MP_FRAME_EOF &&
+        pad->pending.type != pad->type)
+    {
+        MP_FATAL(c, "unknown frame %s\n", mp_frame_type_str(pad->pending.type));
+        mp_frame_unref(&pad->pending);
+    }
+
+    if (mp_frame_is_data(pad->pending) && pad->in_fmt.type &&
+        !is_format_ok(pad->pending, pad->in_fmt))
+    {
+        if (!c->draining_recover)
+            MP_VERBOSE(c, "format change on %s\n", pad->name);
+        c->draining_recover = true;
+        if (c->initialized)
+            send_global_eof(c);
+    }
+}
+
+// Attempt to initialize all pads. Return true if all are initialized, or
+// false if more data is needed (or on error).
+static bool init_pads(struct lavfi *c)
+{
+    if (!c->graph)
+        goto error;
+
+    for (int n = 0; n < c->num_out_pads; n++) {
+        struct lavfi_pad *pad = c->out_pads[n];
+        if (pad->buffer)
+            continue;
+
+        const AVFilter *dst_filter = NULL;
+        if (pad->type == MP_FRAME_AUDIO) {
+            dst_filter = avfilter_get_by_name("abuffersink");
+        } else if (pad->type == MP_FRAME_VIDEO) {
+            dst_filter = avfilter_get_by_name("buffersink");
+        } else {
+            assert(0);
+        }
+
+        if (!dst_filter)
+            goto error;
+
+        char name[256];
+        snprintf(name, sizeof(name), "mpv_sink_%s", pad->name);
+
+        if (avfilter_graph_create_filter(&pad->buffer, dst_filter,
+                                         name, NULL, NULL, c->graph) < 0)
+            goto error;
+
+        if (avfilter_link(pad->filter, pad->filter_pad, pad->buffer, 0) < 0)
+            goto error;
+    }
+
+    for (int n = 0; n < c->num_in_pads; n++) {
+        struct lavfi_pad *pad = c->in_pads[n];
+        if (pad->buffer)
+            continue;
+
+        mp_frame_unref(&pad->in_fmt);
+
+        read_pad_input(c, pad);
+        // no input data, format unknown, can't init, wait longer.
+        if (!pad->pending.type)
+            return false;
+
+        if (mp_frame_is_data(pad->pending)) {
+            assert(pad->pending.type == pad->type);
+
+            pad->in_fmt = mp_frame_ref(pad->pending);
+            if (!pad->in_fmt.type)
+                goto error;
+
+            if (pad->in_fmt.type == MP_FRAME_VIDEO)
+                mp_image_unref_data(pad->in_fmt.data);
+            if (pad->in_fmt.type == MP_FRAME_AUDIO)
+                mp_aframe_unref_data(pad->in_fmt.data);
+        }
+
+        if (pad->pending.type == MP_FRAME_EOF && !pad->in_fmt.type) {
+            // libavfilter makes this painful. Init it with a dummy config,
+            // just so we can tell it the stream is EOF.
+            if (pad->type == MP_FRAME_AUDIO) {
+                struct mp_aframe *fmt = mp_aframe_create();
+                mp_aframe_set_format(fmt, AF_FORMAT_FLOAT);
+                mp_aframe_set_chmap(fmt, &(struct mp_chmap)MP_CHMAP_INIT_STEREO);
+                mp_aframe_set_rate(fmt, 48000);
+                pad->in_fmt = (struct mp_frame){MP_FRAME_AUDIO, fmt};
+            }
+            if (pad->type == MP_FRAME_VIDEO) {
+                struct mp_image *fmt = talloc_zero(NULL, struct mp_image);
+                mp_image_setfmt(fmt, IMGFMT_420P);
+                mp_image_set_size(fmt, 64, 64);
+                pad->in_fmt = (struct mp_frame){MP_FRAME_VIDEO, fmt};
+            }
+        }
+
+        if (pad->in_fmt.type != pad->type)
+            goto error;
+
+        AVBufferSrcParameters *params = av_buffersrc_parameters_alloc();
+        if (!params)
+            goto error;
+
+        pad->timebase = AV_TIME_BASE_Q;
+
+        char *filter_name = NULL;
+        if (pad->type == MP_FRAME_AUDIO) {
+            struct mp_aframe *fmt = pad->in_fmt.data;
+            params->format = af_to_avformat(mp_aframe_get_format(fmt));
+            params->sample_rate = mp_aframe_get_rate(fmt);
+            struct mp_chmap chmap = {0};
+            mp_aframe_get_chmap(fmt, &chmap);
+            params->channel_layout = mp_chmap_to_lavc(&chmap);
+            pad->timebase = (AVRational){1, mp_aframe_get_rate(fmt)};
+            filter_name = "abuffer";
+        } else if (pad->type == MP_FRAME_VIDEO) {
+            struct mp_image *fmt = pad->in_fmt.data;
+            params->format = imgfmt2pixfmt(fmt->imgfmt);
+            params->width = fmt->w;
+            params->height = fmt->h;
+            params->sample_aspect_ratio.num = fmt->params.p_w;
+            params->sample_aspect_ratio.den = fmt->params.p_h;
+            params->hw_frames_ctx = fmt->hwctx;
+            filter_name = "buffer";
+        } else {
+            assert(0);
+        }
+
+        params->time_base = pad->timebase;
+
+        const AVFilter *filter = avfilter_get_by_name(filter_name);
+        if (filter) {
+            char name[256];
+            snprintf(name, sizeof(name), "mpv_src_%s", pad->name);
+
+            pad->buffer = avfilter_graph_alloc_filter(c->graph, filter, name);
+        }
+        if (!pad->buffer) {
+            av_free(params);
+            goto error;
+        }
+
+        int ret = av_buffersrc_parameters_set(pad->buffer, params);
+        av_free(params);
+        if (ret < 0)
+            goto error;
+
+        if (avfilter_init_str(pad->buffer, NULL) < 0)
+            goto error;
+
+        if (avfilter_link(pad->buffer, 0, pad->filter, pad->filter_pad) < 0)
+            goto error;
+    }
+
+    return true;
+error:
+    MP_FATAL(c, "could not initialize filter pads\n");
+    c->failed = true;
+    mp_filter_internal_mark_failed(c->f);
+    return false;
+}
+
+static void dump_graph(struct lavfi *c)
+{
+#if LIBAVFILTER_VERSION_MICRO >= 100
+    MP_DBG(c, "Filter graph:\n");
+    char *s = avfilter_graph_dump(c->graph, NULL);
+    if (s)
+        MP_DBG(c, "%s\n", s);
+    av_free(s);
+#endif
+}
+
+// Initialize the graph if all inputs have formats set. If it's already
+// initialized, or can't be initialized yet, do nothing.
+static void init_graph(struct lavfi *c)
+{
+    assert(!c->initialized);
+
+    if (!c->graph)
+        precreate_graph(c, false);
+
+    if (init_pads(c)) {
+        struct mp_stream_info *info = mp_filter_find_stream_info(c->f);
+        if (info && info->hwdec_devs) {
+            struct mp_hwdec_ctx *hwdec = hwdec_devices_get_first(info->hwdec_devs);
+            for (int n = 0; n < c->graph->nb_filters; n++) {
+                AVFilterContext *filter = c->graph->filters[n];
+                if (hwdec && hwdec->av_device_ref)
+                    filter->hw_device_ctx = av_buffer_ref(hwdec->av_device_ref);
+            }
+        }
+
+        // And here the actual libavfilter initialization happens.
+        if (avfilter_graph_config(c->graph, NULL) < 0) {
+            MP_FATAL(c, "failed to configure the filter graph\n");
+            free_graph(c);
+            c->failed = true;
+            mp_filter_internal_mark_failed(c->f);
+            return;
+        }
+
+        // The timebase is available after configuring.
+        for (int n = 0; n < c->num_out_pads; n++) {
+            struct lavfi_pad *pad = c->out_pads[n];
+
+            pad->timebase = pad->buffer->inputs[0]->time_base;
+        }
+
+        c->initialized = true;
+
+        if (!c->direct_filter) // (output uninteresting for direct filters)
+            dump_graph(c);
+    }
+}
+
+static bool feed_input_pads(struct lavfi *c)
+{
+    bool progress = false;
+    bool was_draining = c->draining_recover;
+
+    assert(c->initialized);
+
+    for (int n = 0; n < c->num_in_pads; n++) {
+        struct lavfi_pad *pad = c->in_pads[n];
+
+        bool requested = true;
+#if LIBAVFILTER_VERSION_MICRO >= 100
+        requested = av_buffersrc_get_nb_failed_requests(pad->buffer) > 0;
+#endif
+
+        // Always request a frame after EOF so that we can know if the EOF state
+        // changes (e.g. for sparse streams with midstream EOF).
+        requested |= pad->buffer_is_eof;
+
+        if (requested)
+            read_pad_input(c, pad);
+
+        if (!pad->pending.type || c->draining_recover)
+            continue;
+
+        if (pad->buffer_is_eof) {
+            MP_WARN(c, "eof state changed on %s\n", pad->name);
+            c->draining_recover = true;
+            send_global_eof(c);
+            continue;
+        }
+
+        AVFrame *frame = mp_frame_to_av(pad->pending, &pad->timebase);
+        bool eof = pad->pending.type == MP_FRAME_EOF;
+
+        mp_frame_unref(&pad->pending);
+
+        if (!frame && !eof) {
+            MP_FATAL(c, "out of memory or unsupported format\n");
+            continue;
+        }
+
+        pad->buffer_is_eof = !frame;
+
+        if (av_buffersrc_add_frame(pad->buffer, frame) < 0)
+            MP_FATAL(c, "could not pass frame to filter\n");
+        av_frame_free(&frame);
+
+        progress = true;
+    }
+
+    if (!was_draining && c->draining_recover)
+        progress = true;
+
+    return progress;
+}
+
+static bool read_output_pads(struct lavfi *c)
+{
+    bool progress = false;
+
+    assert(c->initialized);
+
+    for (int n = 0; n < c->num_out_pads; n++) {
+        struct lavfi_pad *pad = c->out_pads[n];
+
+        if (!mp_pin_in_needs_data(pad->pin))
+            continue;
+
+        assert(pad->buffer);
+
+        int r = AVERROR_EOF;
+        if (!pad->buffer_is_eof)
+            r = av_buffersink_get_frame_flags(pad->buffer, c->tmp_frame, 0);
+        if (r >= 0) {
+#if LIBAVUTIL_VERSION_MICRO >= 100
+            mp_tags_copy_from_av_dictionary(pad->metadata, c->tmp_frame->metadata);
+#endif
+            struct mp_frame frame =
+                mp_frame_from_av(pad->type, c->tmp_frame, &pad->timebase);
+            av_frame_unref(c->tmp_frame);
+            if (frame.type) {
+                mp_pin_in_write(pad->pin, frame);
+            } else {
+                MP_ERR(c, "could not use filter output\n");
+                mp_frame_unref(&frame);
+            }
+            progress = true;
+        } else if (r == AVERROR(EAGAIN)) {
+            // We expect that libavfilter will request input on one of the
+            // input pads (via av_buffersrc_get_nb_failed_requests()).
+        } else if (r == AVERROR_EOF) {
+            if (!c->draining_recover && !pad->buffer_is_eof)
+                mp_pin_in_write(pad->pin, MP_EOF_FRAME);
+            if (!pad->buffer_is_eof)
+                progress = true;
+            pad->buffer_is_eof = true;
+        } else {
+            // Real error - ignore it.
+            MP_ERR(c, "error on filtering (%d)\n", r);
+        }
+    }
+
+    return progress;
+}
+
+static void lavfi_process(struct mp_filter *f)
+{
+    struct lavfi *c = f->priv;
+
+    if (!c->initialized)
+        init_graph(c);
+
+    while (c->initialized) {
+        bool a = read_output_pads(c);
+        bool b = feed_input_pads(c);
+        if (!a && !b)
+            break;
+    }
+
+    // Start over on format changes or EOF draining.
+    if (c->draining_recover) {
+        // Wait until all outputs got EOF.
+        bool all_eof = true;
+        for (int n = 0; n < c->num_out_pads; n++)
+            all_eof &= c->out_pads[n]->buffer_is_eof;
+
+        if (all_eof) {
+            MP_VERBOSE(c, "recovering all eof\n");
+            free_graph(c);
+            mp_filter_internal_mark_progress(c->f);
+        }
+    }
+
+    if (c->failed)
+        mp_filter_internal_mark_failed(c->f);
+}
+
+static void lavfi_reset(struct mp_filter *f)
+{
+    struct lavfi *c = f->priv;
+
+    free_graph(c);
+
+    for (int n = 0; n < c->num_in_pads; n++)
+        mp_frame_unref(&c->in_pads[n]->pending);
+}
+
+static void lavfi_destroy(struct mp_filter *f)
+{
+    struct lavfi *c = f->priv;
+
+    lavfi_reset(f);
+    av_frame_free(&c->tmp_frame);
+}
+
+static bool lavfi_command(struct mp_filter *f, struct mp_filter_command *cmd)
+{
+    struct lavfi *c = f->priv;
+
+    if (!c->initialized)
+        return false;
+
+    switch (cmd->type) {
+#if LIBAVFILTER_VERSION_MICRO >= 100
+    case MP_FILTER_COMMAND_TEXT: {
+        return avfilter_graph_send_command(c->graph, "all", cmd->cmd, cmd->arg,
+                                           &(char){0}, 0, 0) >= 0;
+    }
+#endif
+    case MP_FILTER_COMMAND_GET_META: {
+        // We can worry later about what it should do to multi output filters.
+        if (c->num_out_pads < 1)
+            return false;
+        struct mp_tags **ptags = cmd->res;
+        *ptags = mp_tags_dup(NULL, c->out_pads[0]->metadata);
+        return true;
+    }
+    default:
+        return false;
+    }
+}
+
+static const struct mp_filter_info lavfi_filter = {
+    .name = "lavfi",
+    .priv_size = sizeof(struct lavfi),
+    .process = lavfi_process,
+    .reset = lavfi_reset,
+    .destroy = lavfi_destroy,
+    .command = lavfi_command,
+};
+
+static struct lavfi *lavfi_alloc(struct mp_filter *parent)
+{
+    struct mp_filter *f = mp_filter_create(parent, &lavfi_filter);
+    if (!f)
+        return NULL;
+
+    struct lavfi *c = f->priv;
+
+    c->f = f;
+    c->log = f->log;
+    c->public.f = f;
+    c->tmp_frame = av_frame_alloc();
+    if (!c->tmp_frame)
+        abort();
+
+    return c;
+}
+
+static struct mp_lavfi *do_init(struct lavfi *c)
+{
+    precreate_graph(c, true);
+
+    if (c->failed)
+        goto error;
+
+    for (int n = 0; n < c->num_in_pads + c->num_out_pads; n++) {
+        // First add input pins to satisfy order for bidir graph types.
+        struct lavfi_pad *pad =
+            n < c->num_in_pads ? c->in_pads[n] : c->out_pads[n - c->num_in_pads];
+
+        pad->pin_index = c->f->num_pins;
+        pad->pin = mp_filter_add_pin(c->f, pad->dir, pad->name);
+
+        if (c->force_type && c->force_type != pad->type) {
+            MP_FATAL(c, "mismatching media type\n");
+            goto error;
+        }
+    }
+
+    if (c->force_bidir) {
+        if (c->f->num_pins != 2) {
+            MP_FATAL(c, "exactly 2 pads required\n");
+            goto error;
+        }
+        if (mp_pin_get_dir(c->f->ppins[0]) != MP_PIN_OUT ||
+            mp_pin_get_dir(c->f->ppins[1]) != MP_PIN_IN)
+        {
+            MP_FATAL(c, "1 input and 1 output pad required\n");
+            goto error;
+        }
+    }
+
+    return &c->public;
+
+error:
+    talloc_free(c->f);
+    return NULL;
+}
+
+struct mp_lavfi *mp_lavfi_create_graph(struct mp_filter *parent,
+                                       enum mp_frame_type type, bool bidir,
+                                       char **graph_opts, const char *graph)
+{
+    struct lavfi *c = lavfi_alloc(parent);
+    if (!c)
+        return NULL;
+
+    c->force_type = type;
+    c->force_bidir = bidir;
+    c->graph_opts = mp_dup_str_array(c, graph_opts);
+    c->graph_string = talloc_strdup(c, graph);
+
+    return do_init(c);
+}
+
+struct mp_lavfi *mp_lavfi_create_filter(struct mp_filter *parent,
+                                        enum mp_frame_type type, bool bidir,
+                                        char **graph_opts,
+                                        const char *filter, char **filter_opts)
+{
+    struct lavfi *c = lavfi_alloc(parent);
+    if (!c)
+        return NULL;
+
+    c->force_type = type;
+    c->force_bidir = bidir;
+    c->graph_opts = mp_dup_str_array(c, graph_opts);
+    c->graph_string = talloc_strdup(c, filter);
+    c->direct_filter_opts = mp_dup_str_array(c, filter_opts);
+    c->direct_filter = true;
+
+    return do_init(c);
+}
+
+struct lavfi_user_opts {
+    bool is_bridge;
+
+    char *graph;
+    char **avopts;
+
+    char *filter_name;
+    char **filter_opts;
+};
+
+static struct mp_filter *vf_lavfi_create(struct mp_filter *parent, void *options)
+{
+    struct lavfi_user_opts *opts = options;
+    struct mp_lavfi *l;
+    if (opts->is_bridge) {
+        l = mp_lavfi_create_filter(parent, MP_FRAME_VIDEO, true,
+                                   opts->avopts, opts->filter_name,
+                                   opts->filter_opts);
+    } else {
+        l = mp_lavfi_create_graph(parent, MP_FRAME_VIDEO, true,
+                                  opts->avopts, opts->graph);
+    }
+    talloc_free(opts);
+    return l ? l->f : NULL;
+}
+
+static bool is_single_video_only(const AVFilterPad *pads)
+{
+    int count = avfilter_pad_count(pads);
+    if (count != 1)
+        return false;
+    return avfilter_pad_get_type(pads, 0) == AVMEDIA_TYPE_VIDEO;
+}
+
+// Does it have exactly one video input and one video output?
+static bool is_usable(const AVFilter *filter)
+{
+    return is_single_video_only(filter->inputs) &&
+           is_single_video_only(filter->outputs);
+}
+
+static void print_help(struct mp_log *log)
+{
+    mp_info(log, "List of libavfilter filters:\n");
+    for (const AVFilter *filter = avfilter_next(NULL); filter;
+         filter = avfilter_next(filter))
+    {
+        if (is_usable(filter))
+            mp_info(log, " %-16s %s\n", filter->name, filter->description);
+    }
+    mp_info(log, "\n"
+        "This lists video->video filters only. Refer to\n"
+        "\n"
+        " https://ffmpeg.org/ffmpeg-filters.html\n"
+        "\n"
+        "to see how to use each filter and what arguments each filter takes.\n"
+        "Also, be sure to quote the FFmpeg filter string properly, e.g.:\n"
+        "\n"
+        " \"--vf=lavfi=[gradfun=20:30]\"\n"
+        "\n"
+        "Otherwise, mpv and libavfilter syntax will conflict.\n"
+        "\n");
+}
+
+#define OPT_BASE_STRUCT struct lavfi_user_opts
+
+const struct mp_user_filter_entry vf_lavfi = {
+    .desc = {
+        .description = "libavfilter bridge",
+        .name = "lavfi",
+        .priv_size = sizeof(OPT_BASE_STRUCT),
+        .options = (const m_option_t[]){
+            OPT_STRING("graph", graph, M_OPT_MIN, .min = 1),
+            OPT_KEYVALUELIST("o", avopts, 0),
+            {0}
+        },
+        .print_help = print_help,
+    },
+    .create = vf_lavfi_create,
+};
+
+const struct mp_user_filter_entry vf_lavfi_bridge = {
+    .desc = {
+        .description = "libavfilter bridge (explicit options)",
+        .name = "lavfi-bridge",
+        .priv_size = sizeof(OPT_BASE_STRUCT),
+        .options = (const m_option_t[]){
+            OPT_STRING("name", filter_name, M_OPT_MIN, .min = 1),
+            OPT_KEYVALUELIST("opts", filter_opts, 0),
+            OPT_KEYVALUELIST("o", avopts, 0),
+            {0}
+        },
+        .priv_defaults = &(const OPT_BASE_STRUCT){
+            .is_bridge = true,
+        },
+        .print_help = print_help,
+    },
+    .create = vf_lavfi_create,
+};
diff --git a/filters/f_lavfi.h b/filters/f_lavfi.h
new file mode 100644
index 0000000000..7efc96c3de
--- /dev/null
+++ b/filters/f_lavfi.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include "frame.h"
+
+// A wrapped libavfilter filter or filter graph.
+// (to free this, free the filtert itself, mp_lavfi.f)
+struct mp_lavfi {
+    // This mirrors the libavfilter pads according to the user specification.
+    struct mp_filter *f;
+};
+
+// Create a filter with the given libavfilter graph string. The graph must
+// have labels on all unconnected pads; these are exposed as pins.
+//  type: if not 0, require all pads to have a compatible media type (or error)
+//  bidir: if true, require exactly 2 pads, 1 input, 1 output (mp_lavfi.f will
+//         have the input as pin 0, and the output as pin 1)
+//  graph_opts: options for the filter graph, see mp_set_avopts() (NULL is OK)
+//  graph: a libavfilter graph specification
+struct mp_lavfi *mp_lavfi_create_graph(struct mp_filter *parent,
+                                       enum mp_frame_type type, bool bidir,
+                                       char **graph_opts, const char *graph);
+
+// Unlike mp_lavfi_create_graph(), this creates a single filter, using explicit
+// options, and without involving the libavfilter graph parser. Instead of
+// a graph, it takes a filter name, and a key-value list of filter options
+// (which are applied with mp_set_avopts()).
+struct mp_lavfi *mp_lavfi_create_filter(struct mp_filter *parent,
+                                        enum mp_frame_type type, bool bidir,
+                                        char **graph_opts,
+                                        const char *filter, char **filter_opts);
diff --git a/filters/f_output_chain.c b/filters/f_output_chain.c
new file mode 100644
index 0000000000..d98c7ca4b3
--- /dev/null
+++ b/filters/f_output_chain.c
@@ -0,0 +1,564 @@
+#include "common/global.h"
+#include "options/m_config.h"
+#include "options/m_option.h"
+#include "video/out/vo.h"
+
+#include "filter_internal.h"
+
+#include "f_autoconvert.h"
+#include "f_auto_filters.h"
+#include "f_lavfi.h"
+#include "f_output_chain.h"
+#include "f_utils.h"
+#include "user_filters.h"
+
+struct chain {
+    struct mp_filter *f;
+    struct mp_log *log;
+
+    enum mp_output_chain_type type;
+
+    // Expected media type.
+    enum mp_frame_type frame_type;
+
+    struct mp_stream_info stream_info;
+
+    struct mp_user_filter **pre_filters;
+    int num_pre_filters;
+    struct mp_user_filter **post_filters;
+    int num_post_filters;
+
+    struct mp_user_filter **user_filters;
+    int num_user_filters;
+
+    // Concatentated list of pre+user+post filters.
+    struct mp_user_filter **all_filters;
+    int num_all_filters;
+    // First input/last output of all_filters[].
+    struct mp_pin *filters_in, *filters_out;
+
+    struct mp_user_filter *input, *output;
+    struct mp_autoconvert *convert;
+
+    struct vo *vo;
+
+    struct mp_output_chain public;
+};
+
+// This wraps each individual "actual" filter for:
+//  - isolating against its failure (logging it and disabling the filter)
+//  - tracking its output format (mostly for logging)
+//  - store extra per-filter information like the filter label
+struct mp_user_filter {
+    struct chain *p;
+
+    struct mp_filter *wrapper; // parent filter for f
+    struct mp_filter *f; // the actual user filter
+    struct m_obj_settings *args; // NULL, or list of 1 item with creation args
+    char *label;
+    bool generated_label;
+    char *name;
+    bool is_output_converter;
+    bool is_input;
+
+    struct mp_image_params last_out_params;
+
+    bool failed;
+    bool error_eof_sent;
+};
+
+static void update_output_caps(struct chain *p)
+{
+    if (p->type != MP_OUTPUT_CHAIN_VIDEO)
+        return;
+
+    mp_autoconvert_clear(p->convert);
+
+    if (p->vo) {
+        uint8_t allowed_output_formats[IMGFMT_END - IMGFMT_START] = {0};
+        vo_query_formats(p->vo, allowed_output_formats);
+
+        for (int n = 0; n < MP_ARRAY_SIZE(allowed_output_formats); n++) {
+            if (allowed_output_formats[n])
+                mp_autoconvert_add_imgfmt(p->convert, IMGFMT_START + n, 0);
+        }
+
+        if (p->vo->hwdec_devs)
+            mp_autoconvert_add_vo_hwdec_subfmts(p->convert, p->vo->hwdec_devs);
+    }
+}
+
+static bool check_out_format_change(struct mp_user_filter *u,
+                                    struct mp_frame frame)
+{
+    struct chain *p = u->p;
+    bool changed = false;
+
+    if (frame.type == MP_FRAME_VIDEO) {
+        struct mp_image *img = frame.data;
+
+        if (!mp_image_params_equal(&img->params, &u->last_out_params)) {
+            MP_VERBOSE(p, "[%s] %s\n", u->name,
+                       mp_image_params_to_str(&img->params));
+            u->last_out_params = img->params;
+
+            if (u->is_input) {
+                p->public.input_params = img->params;
+
+                // Unfortunately there's no good place to update these.
+                // But a common case is enabling HW decoding, which
+                // might init some support of them in the VO, and update
+                // the VO's format list.
+                update_output_caps(p);
+            } else if (u->is_output_converter) {
+                p->public.output_params = img->params;
+            }
+
+            p->public.reconfig_happened = true;
+            changed = true;
+        }
+    }
+
+    return changed;
+}
+
+static void process_user(struct mp_filter *f)
+{
+    struct mp_user_filter *u = f->priv;
+    struct chain *p = u->p;
+
+    mp_filter_set_error_handler(u->f, f);
+    const char *name = u->label ? u->label : u->name;
+    assert(u->name);
+
+    if (!u->failed && mp_filter_has_failed(u->f)) {
+        if (u->is_output_converter) {
+            // This is a fuckup we can't ignore.
+            MP_FATAL(p, "Cannot convert decoder/filter output to any format "
+                     "supported by the output.\n");
+            p->public.failed_output_conversion = true;
+            mp_filter_wakeup(p->f);
+        } else {
+            MP_ERR(p, "Disabling filter %s because it has failed.\n", name);
+            mp_filter_reset(u->f); // clear out staled buffered data
+        }
+        u->failed = true;
+    }
+
+    if (u->failed) {
+        if (u->is_output_converter) {
+            if (mp_pin_in_needs_data(f->ppins[1])) {
+                if (!u->error_eof_sent)
+                    mp_pin_in_write(f->ppins[1], MP_EOF_FRAME);
+                u->error_eof_sent = true;
+            }
+            return;
+        }
+
+        mp_pin_transfer_data(f->ppins[1], f->ppins[0]);
+        return;
+    }
+
+    mp_pin_transfer_data(u->f->pins[0], f->ppins[0]);
+
+    if (mp_pin_can_transfer_data(f->ppins[1], u->f->pins[1])) {
+        struct mp_frame frame = mp_pin_out_read(u->f->pins[1]);
+
+        check_out_format_change(u, frame);
+
+        mp_pin_in_write(f->ppins[1], frame);
+    }
+}
+
+static void reset_user(struct mp_filter *f)
+{
+    struct mp_user_filter *u = f->priv;
+
+    u->error_eof_sent = false;
+}
+
+static void destroy_user(struct mp_filter *f)
+{
+    struct mp_user_filter *u = f->priv;
+
+    struct m_option dummy = {.type = &m_option_type_obj_settings_list};
+    m_option_free(&dummy, &u->args);
+
+    mp_filter_free_children(f);
+}
+
+static const struct mp_filter_info user_wrapper_filter = {
+    .name = "user_filter_wrapper",
+    .priv_size = sizeof(struct mp_user_filter),
+    .process = process_user,
+    .reset = reset_user,
+    .destroy = destroy_user,
+};
+
+static struct mp_user_filter *create_wrapper_filter(struct chain *p)
+{
+    struct mp_filter *f = mp_filter_create(p->f, &user_wrapper_filter);
+    if (!f)
+        abort();
+    struct mp_user_filter *wrapper = f->priv;
+    wrapper->wrapper = f;
+    wrapper->p = p;
+    mp_filter_add_pin(f, MP_PIN_IN, "in");
+    mp_filter_add_pin(f, MP_PIN_OUT, "out");
+    return wrapper;
+}
+
+// Rebuild p->all_filters and relink the filters. Non-destructive if no change.
+static void relink_filter_list(struct chain *p)
+{
+    struct mp_user_filter **all_filters[3] =
+        {p->pre_filters, p->user_filters, p->post_filters};
+    int all_filters_num[3] =
+        {p->num_pre_filters, p->num_user_filters, p->num_post_filters};
+    p->num_all_filters = 0;
+    for (int n = 0; n < 3; n++) {
+        struct mp_user_filter **filters = all_filters[n];
+        int filters_num = all_filters_num[n];
+        for (int i = 0; i < filters_num; i++)
+            MP_TARRAY_APPEND(p, p->all_filters, p->num_all_filters, filters[i]);
+    }
+
+    assert(p->num_all_filters > 0);
+
+    p->filters_in = NULL;
+    p->filters_out = NULL;
+    for (int n = 0; n < p->num_all_filters; n++) {
+        struct mp_filter *f = p->all_filters[n]->wrapper;
+        if (n == 0)
+            p->filters_in = f->pins[0];
+        if (p->filters_out)
+            mp_pin_connect(f->pins[0], p->filters_out);
+        p->filters_out = f->pins[1];
+    }
+}
+
+static void process(struct mp_filter *f)
+{
+    struct chain *p = f->priv;
+
+    if (mp_pin_can_transfer_data(p->filters_in, f->ppins[0])) {
+        struct mp_frame frame = mp_pin_out_read(f->ppins[0]);
+
+        if (frame.type == MP_FRAME_EOF) {
+            MP_VERBOSE(p, "filter input EOF\n");
+            p->public.got_input_eof = true;
+        }
+
+        mp_pin_in_write(p->filters_in, frame);
+    }
+
+    if (mp_pin_can_transfer_data(f->ppins[1], p->filters_out)) {
+        struct mp_frame frame = mp_pin_out_read(p->filters_out);
+
+        if (frame.type == MP_FRAME_EOF) {
+            MP_VERBOSE(p, "filter output EOF\n");
+            p->public.got_output_eof = true;
+        }
+
+        mp_pin_in_write(f->ppins[1], frame);
+    }
+}
+
+static void reset(struct mp_filter *f)
+{
+    struct chain *p = f->priv;
+
+    p->public.got_input_eof = false;
+    p->public.got_output_eof = false;
+}
+
+void mp_output_chain_reset_harder(struct mp_output_chain *c)
+{
+    struct chain *p = c->f->priv;
+
+    mp_filter_reset(p->f);
+
+    p->public.failed_output_conversion = false;
+    for (int n = 0; n < p->num_all_filters; n++)
+        p->all_filters[n]->failed = false;
+}
+
+static void destroy(struct mp_filter *f)
+{
+    reset(f);
+}
+
+static const struct mp_filter_info output_chain_filter = {
+    .name = "output_chain",
+    .priv_size = sizeof(struct chain),
+    .process = process,
+    .reset = reset,
+    .destroy = destroy,
+};
+
+static double get_display_fps(struct mp_stream_info *i)
+{
+    struct chain *p = i->priv;
+    double res = 0;
+    if (p->vo)
+        vo_control(p->vo, VOCTRL_GET_DISPLAY_FPS, &res);
+    return res;
+}
+
+static double get_container_fps(struct mp_stream_info *i)
+{
+    struct chain *p = i->priv;
+    return p->public.container_fps;
+}
+
+void mp_output_chain_set_vo(struct mp_output_chain *c, struct vo *vo)
+{
+    struct chain *p = c->f->priv;
+
+    p->stream_info.hwdec_devs = vo ? vo->hwdec_devs : NULL;
+    p->stream_info.osd = vo ? vo->osd : NULL;
+    p->stream_info.rotate90 = vo ? vo->driver->caps & VO_CAP_ROTATE90 : false;
+    p->vo = vo;
+    update_output_caps(p);
+}
+
+static struct mp_user_filter *find_by_label(struct chain *p, const char *label)
+{
+    for (int n = 0; n < p->num_user_filters; n++) {
+        struct mp_user_filter *u = p->user_filters[n];
+        if (label && u->label && strcmp(label, u->label) == 0)
+            return u;
+    }
+    return NULL;
+}
+
+bool mp_output_chain_command(struct mp_output_chain *c, const char *target,
+                             struct mp_filter_command *cmd)
+{
+    struct chain *p = c->f->priv;
+
+    if (!target || !target[0])
+        return false;
+
+    if (strcmp(target, "all") == 0 && cmd->type == MP_FILTER_COMMAND_TEXT) {
+        // (Following old semantics.)
+        for (int n = 0; n < p->num_user_filters; n++)
+            mp_filter_command(p->user_filters[n]->f, cmd);
+        return true;
+    }
+
+    struct mp_user_filter *f = find_by_label(p, target);
+    if (!f)
+        return false;
+
+    return mp_filter_command(f->f, cmd);
+}
+
+static bool compare_filter(struct m_obj_settings *a, struct m_obj_settings *b)
+{
+    if (a == b || !a || !b)
+        return a == b;
+
+    if (!a->name || !b->name)
+        return a->name == b->name;
+
+    if (!!a->label != !!b->label || (a->label && strcmp(a->label, b->label) != 0))
+        return false;
+
+    if (a->enabled != b->enabled)
+        return false;
+
+    if (!a->attribs || !a->attribs[0])
+        return !b->attribs || !b->attribs[0];
+
+    for (int n = 0; a->attribs[n] || b->attribs[n]; n++) {
+        if (!a->attribs[n] || !b->attribs[n])
+            return false;
+        if (strcmp(a->attribs[n], b->attribs[n]) != 0)
+            return false;
+    }
+
+    return true;
+}
+
+bool mp_output_chain_update_filters(struct mp_output_chain *c,
+                                    struct m_obj_settings *list)
+{
+    struct chain *p = c->f->priv;
+
+    struct mp_user_filter **add = NULL;      // new filters
+    int num_add = 0;
+    struct mp_user_filter **res = NULL;      // new final list
+    int num_res = 0;
+    bool *used = talloc_zero_array(NULL, bool, p->num_user_filters);
+
+    for (int n = 0; list && list[n].name; n++) {
+        struct m_obj_settings *entry = &list[n];
+
+        if (!entry->enabled)
+            continue;
+
+        struct mp_user_filter *u = NULL;
+
+        for (int i = 0; i < p->num_user_filters; i++) {
+            if (!used[i] && compare_filter(entry, p->user_filters[i]->args)) {
+                u = p->user_filters[i];
+                used[i] = true;
+                break;
+            }
+        }
+
+        if (!u) {
+            u = create_wrapper_filter(p);
+            u->name = talloc_strdup(u, entry->name);
+            u->label = talloc_strdup(u, entry->label);
+            u->f = mp_create_user_filter(u->wrapper, p->type, entry->name,
+                                         entry->attribs);
+            if (!u->f) {
+                talloc_free(u->wrapper);
+                goto error;
+            }
+
+            struct m_obj_settings *args = (struct m_obj_settings[2]){*entry, {0}};
+
+            struct m_option dummy = {.type = &m_option_type_obj_settings_list};
+            m_option_copy(&dummy, &u->args, &args);
+
+            MP_TARRAY_APPEND(NULL, add, num_add, u);
+        }
+
+        MP_TARRAY_APPEND(p, res, num_res, u);
+    }
+
+    // At this point we definitely know we'll use the new list, so clean up.
+
+    for (int n = 0; n < p->num_user_filters; n++) {
+        if (!used[n])
+            talloc_free(p->user_filters[n]->wrapper);
+    }
+
+    talloc_free(p->user_filters);
+    p->user_filters = res;
+    p->num_user_filters = num_res;
+
+    relink_filter_list(p);
+
+    for (int n = 0; n < p->num_user_filters; n++) {
+        struct mp_user_filter *u = p->user_filters[n];
+        if (u->generated_label)
+            TA_FREEP(&u->label);
+        if (!u->label) {
+            for (int i = 0; i < 100; i++) {
+                char *label = mp_tprintf(80, "%s.%02d", u->name, i);
+                if (!find_by_label(p, label)) {
+                    u->label = talloc_strdup(u, label);
+                    u->generated_label = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    MP_VERBOSE(p, "User filter list:\n");
+    for (int n = 0; n < p->num_user_filters; n++) {
+        struct mp_user_filter *u = p->user_filters[n];
+        MP_VERBOSE(p, "  %s (%s)\n", u->name, u->label ? u->label : "-");
+    }
+    if (!p->num_user_filters)
+        MP_VERBOSE(p, "  (empty)\n");
+
+    // Filters can load hwdec interops, which might add new formats.
+    update_output_caps(p);
+
+    mp_filter_wakeup(p->f);
+
+    talloc_free(add);
+    talloc_free(used);
+    return true;
+
+error:
+    for (int n = 0; n < num_add; n++)
+        talloc_free(add[n]);
+    talloc_free(add);
+    talloc_free(used);
+    return false;
+}
+
+static void create_video_things(struct chain *p)
+{
+    p->frame_type = MP_FRAME_VIDEO;
+
+    p->stream_info.priv = p;
+    p->stream_info.get_display_fps = get_display_fps;
+    p->stream_info.get_container_fps = get_container_fps;
+
+    p->f->stream_info = &p->stream_info;
+
+    struct mp_user_filter *f = create_wrapper_filter(p);
+    f->name = "userdeint";
+    f->f = mp_deint_create(f->wrapper);
+    if (!f->f)
+        abort();
+    MP_TARRAY_APPEND(p, p->pre_filters, p->num_pre_filters, f);
+
+    f = create_wrapper_filter(p);
+    f->name = "autorotate";
+    f->f = mp_autorotate_create(f->wrapper);
+    if (!f->f)
+        abort();
+    MP_TARRAY_APPEND(p, p->post_filters, p->num_post_filters, f);
+}
+
+struct mp_output_chain *mp_output_chain_create(struct mp_filter *parent,
+                                               enum mp_output_chain_type type)
+{
+    struct mp_filter *f = mp_filter_create(parent, &output_chain_filter);
+    if (!f)
+        return NULL;
+
+    mp_filter_add_pin(f, MP_PIN_IN, "in");
+    mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    const char *log_name = NULL;
+    switch (type) {
+    case MP_OUTPUT_CHAIN_VIDEO: log_name = "!vf"; break;
+    }
+    if (log_name)
+        f->log = mp_log_new(f, parent->global->log, log_name);
+
+    struct chain *p = f->priv;
+    p->f = f;
+    p->log = f->log;
+    p->type = type;
+
+    struct mp_output_chain *c = &p->public;
+    c->f = f;
+
+    // Dummy filter for reporting and logging the input format.
+    p->input = create_wrapper_filter(p);
+    p->input->f = mp_bidir_nop_filter_create(p->input->wrapper);
+    if (!p->input->f)
+        abort();
+    p->input->name = "in";
+    p->input->is_input = true;
+    MP_TARRAY_APPEND(p, p->pre_filters, p->num_pre_filters, p->input);
+
+    switch (type) {
+    case MP_OUTPUT_CHAIN_VIDEO: create_video_things(p); break;
+    }
+
+    p->output = create_wrapper_filter(p);
+    p->convert = mp_autoconvert_create(p->output->wrapper);
+    if (!p->convert)
+        abort();
+    p->output->name = "convert";
+    p->output->is_output_converter = true;
+    p->output->f = p->convert->f;
+    MP_TARRAY_APPEND(p, p->post_filters, p->num_post_filters, p->output);
+
+    relink_filter_list(p);
+
+    reset(f);
+
+    return c;
+}
diff --git a/filters/f_output_chain.h b/filters/f_output_chain.h
new file mode 100644
index 0000000000..64667ed1bd
--- /dev/null
+++ b/filters/f_output_chain.h
@@ -0,0 +1,59 @@
+#pragma once
+
+#include "options/m_option.h"
+#include "video/mp_image.h"
+
+#include "filter.h"
+
+enum mp_output_chain_type {
+    MP_OUTPUT_CHAIN_VIDEO = 1,      // --vf
+};
+
+// A classic single-media filter chain, which reflects --vf and --af.
+// It manages the user-specified filter chain, and VO/AO output conversions.
+// Also handles some automatic filtering (auto rotation and such).
+struct mp_output_chain {
+    // This filter will have 1 input (from decoder) and 1 output (to VO/AO).
+    struct mp_filter *f;
+
+    bool got_input_eof;
+    bool got_output_eof;
+
+    // The filter chain output could not be converted to any format the output
+    // supports.
+    bool failed_output_conversion;
+
+    // Set if any formats in the chain changed. The user can reset the flag.
+    // For implementing change notifications out input/output_params.
+    bool reconfig_happened;
+
+    // --- for type==MP_OUTPUT_CHAIN_VIDEO
+    struct mp_image_params input_params;
+    struct mp_image_params output_params;
+    double container_fps;
+};
+
+// (free by freeing mp_output_chain.f)
+struct mp_output_chain *mp_output_chain_create(struct mp_filter *parent,
+                                               enum mp_output_chain_type type);
+
+// Set the VO, which will be used to determine basic capabilities like format
+// and rotation support, and to init hardware filtering things.
+// For type==MP_OUTPUT_CHAIN_VIDEO only.
+struct vo;
+void mp_output_chain_set_vo(struct mp_output_chain *p, struct vo *vo);
+
+// Send a command to the filter with the target label.
+bool mp_output_chain_command(struct mp_output_chain *p, const char *target,
+                             struct mp_filter_command *cmd);
+
+// Perform a seek reset _and_ reset all filter failure states, so that future
+// filtering continues normally.
+void mp_output_chain_reset_harder(struct mp_output_chain *p);
+
+// Try to exchange the filter list. If creation of any filter fails, roll
+// back the changes, and return false.
+struct m_obj_settings;
+bool mp_output_chain_update_filters(struct mp_output_chain *p,
+                                    struct m_obj_settings *list);
+
diff --git a/filters/f_swscale.c b/filters/f_swscale.c
new file mode 100644
index 0000000000..953b5ec77e
--- /dev/null
+++ b/filters/f_swscale.c
@@ -0,0 +1,148 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <inttypes.h>
+#include <stdarg.h>
+#include <assert.h>
+
+#include <libswscale/swscale.h>
+
+#include "common/av_common.h"
+#include "common/msg.h"
+
+#include "options/options.h"
+
+#include "video/img_format.h"
+#include "video/mp_image.h"
+#include "video/mp_image_pool.h"
+#include "video/sws_utils.h"
+#include "video/fmt-conversion.h"
+
+#include "f_swscale.h"
+#include "filter.h"
+#include "filter_internal.h"
+
+int mp_sws_find_best_out_format(int in_format, int *out_formats,
+                                int num_out_formats)
+{
+    if (sws_isSupportedInput(imgfmt2pixfmt(in_format)) < 1)
+        return 0;
+
+    int best = 0;
+    for (int n = 0; n < num_out_formats; n++) {
+        int out_format = out_formats[n];
+
+        if (sws_isSupportedOutput(imgfmt2pixfmt(out_format)) < 1)
+            continue;
+
+        if (best) {
+            int candidate = mp_imgfmt_select_best(best, out_format, in_format);
+            if (candidate)
+                best = candidate;
+        } else {
+            best = out_format;
+        }
+    }
+    return best;
+}
+
+bool mp_sws_supports_input(int imgfmt)
+{
+    return sws_isSupportedInput(imgfmt2pixfmt(imgfmt));
+}
+
+static void process(struct mp_filter *f)
+{
+    struct mp_sws_filter *s = f->priv;
+
+    if (!mp_pin_can_transfer_data(f->ppins[1], f->ppins[0]))
+        return;
+
+    struct mp_frame frame = mp_pin_out_read(f->ppins[0]);
+    if (mp_frame_is_signaling(frame)) {
+        mp_pin_in_write(f->ppins[1], frame);
+        return;
+    }
+
+    if (frame.type != MP_FRAME_VIDEO) {
+        MP_ERR(f, "video frame expected\n");
+        goto error;
+    }
+
+    struct mp_image *src = frame.data;
+    int dstfmt = s->out_format ? s->out_format : src->imgfmt;
+
+    struct mp_image *dst = mp_image_pool_get(s->pool, dstfmt, src->w, src->h);
+    if (!dst)
+        goto error;
+
+    mp_image_copy_attributes(dst, src);
+
+    // If we convert from RGB to YUV, default to limited range.
+    if (mp_imgfmt_get_forced_csp(src->imgfmt) == MP_CSP_RGB &&
+        mp_imgfmt_get_forced_csp(dst->imgfmt) == MP_CSP_AUTO)
+    {
+        dst->params.color.levels = MP_CSP_LEVELS_TV;
+    }
+    mp_image_params_guess_csp(&dst->params);
+
+    bool ok = mp_sws_scale(s->sws, dst, src) >= 0;
+
+    mp_frame_unref(&frame);
+    frame = (struct mp_frame){MP_FRAME_VIDEO, dst};
+
+    if (!ok)
+        goto error;
+
+    mp_pin_in_write(f->ppins[1], frame);
+    return;
+
+error:
+    mp_frame_unref(&frame);
+    mp_filter_internal_mark_failed(f);
+    return;
+}
+
+static const struct mp_filter_info sws_filter = {
+    .name = "swscale",
+    .priv_size = sizeof(struct mp_sws_filter),
+    .process = process,
+};
+
+struct mp_sws_filter *mp_sws_filter_create(struct mp_filter *parent)
+{
+    struct mp_filter *f = mp_filter_create(parent, &sws_filter);
+    if (!f)
+        return NULL;
+
+    mp_filter_add_pin(f, MP_PIN_IN, "in");
+    mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    struct mp_sws_filter *s = f->priv;
+    s->f = f;
+    s->sws = mp_sws_alloc(s);
+    s->sws->log = f->log;
+    s->pool = mp_image_pool_new(s);
+
+    mp_sws_set_from_cmdline(s->sws, f->global);
+
+    return s;
+}
diff --git a/filters/f_swscale.h b/filters/f_swscale.h
new file mode 100644
index 0000000000..0c434c1fd3
--- /dev/null
+++ b/filters/f_swscale.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include <stdbool.h>
+
+struct mp_sws_filter {
+    struct mp_filter *f;
+    // Desired output imgfmt. If 0, uses the input format.
+    int out_format;
+    // private state
+    struct mp_sws_context *sws;
+    struct mp_image_pool *pool;
+};
+
+// Create the filter. Free it with talloc_free(mp_sws_filter.f).
+struct mp_sws_filter *mp_sws_filter_create(struct mp_filter *parent);
+
+// Return the best format based on the input format and a list of allowed output
+// formats. This tries to set the output format to the one that will result in
+// the least loss. Returns a format from out_formats[], or 0 if no format could
+// be chosen (or it's not supported by libswscale).
+int mp_sws_find_best_out_format(int in_format, int *out_formats,
+                                int num_out_formats);
+
+// Whether ther given format is supported as input format.
+bool mp_sws_supports_input(int imgfmt);
diff --git a/filters/f_utils.c b/filters/f_utils.c
new file mode 100644
index 0000000000..f984a3b33a
--- /dev/null
+++ b/filters/f_utils.c
@@ -0,0 +1,175 @@
+#include "video/mp_image.h"
+
+#include "f_utils.h"
+#include "filter_internal.h"
+
+struct frame_duration_priv {
+    struct mp_image *buffered;
+};
+
+static void frame_duration_process(struct mp_filter *f)
+{
+    struct frame_duration_priv *p = f->priv;
+
+    if (!mp_pin_can_transfer_data(f->ppins[1], f->ppins[0]))
+        return;
+
+    struct mp_frame frame = mp_pin_out_read(f->ppins[0]);
+
+    if (frame.type == MP_FRAME_EOF && p->buffered) {
+        mp_pin_in_write(f->ppins[1], MAKE_FRAME(MP_FRAME_VIDEO, p->buffered));
+        p->buffered = NULL;
+        // Pass through the actual EOF in the next iteration.
+        mp_pin_out_repeat_eof(f->ppins[0]);
+    } else if (frame.type == MP_FRAME_VIDEO) {
+        struct mp_image *next = frame.data;
+        if (p->buffered) {
+            if (p->buffered->pts != MP_NOPTS_VALUE &&
+                next->pts != MP_NOPTS_VALUE &&
+                next->pts >= p->buffered->pts)
+                p->buffered->pkt_duration = next->pts - p->buffered->pts;
+            mp_pin_in_write(f->ppins[1], MAKE_FRAME(MP_FRAME_VIDEO, p->buffered));
+        } else {
+            mp_pin_out_request_data(f->ppins[0]);
+        }
+        p->buffered = next;
+    } else {
+        mp_pin_in_write(f->ppins[1], frame);
+    }
+}
+
+static void frame_duration_reset(struct mp_filter *f)
+{
+    struct frame_duration_priv *p = f->priv;
+
+    mp_image_unrefp(&p->buffered);
+}
+
+static const struct mp_filter_info frame_duration_filter = {
+    .name = "frame_duration",
+    .priv_size = sizeof(struct frame_duration_priv),
+    .process = frame_duration_process,
+    .reset = frame_duration_reset,
+};
+
+struct mp_filter *mp_compute_frame_duration_create(struct mp_filter *parent)
+{
+    struct mp_filter *f = mp_filter_create(parent, &frame_duration_filter);
+    if (!f)
+        return NULL;
+
+    mp_filter_add_pin(f, MP_PIN_IN, "in");
+    mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    return f;
+}
+
+void mp_chain_filters(struct mp_pin *in, struct mp_pin *out,
+                      struct mp_filter **filters, int num_filters)
+{
+    for (int n = 0; n < num_filters; n++) {
+        if (!filters[n])
+            continue;
+        assert(filters[n]->num_pins == 2);
+        mp_pin_connect(filters[n]->pins[0], in);
+        in = filters[n]->pins[1];
+    }
+    mp_pin_connect(out, in);
+}
+
+// Make it repeat process().
+static void mark_progress(struct mp_subfilter *sub)
+{
+    // f == NULL is not really allowed, but at least don't crash.
+    struct mp_filter *f = mp_pin_get_manual_connection(sub->in);
+    if (f)
+        mp_filter_internal_mark_progress(f);
+}
+
+bool mp_subfilter_read(struct mp_subfilter *sub)
+{
+    if (sub->filter) {
+        if (mp_pin_can_transfer_data(sub->out, sub->filter->pins[1])) {
+            struct mp_frame frame = mp_pin_out_read(sub->filter->pins[1]);
+            if (sub->draining && frame.type == MP_FRAME_EOF) {
+                sub->draining = false;
+                TA_FREEP(&sub->filter);
+                mark_progress(sub);
+                return false;
+            }
+            mp_pin_in_write(sub->out, frame);
+            return false;
+        }
+        if (sub->draining)
+            return false;
+    }
+
+    struct mp_pin *out = sub->filter ? sub->filter->pins[0] : sub->out;
+
+    if (sub->frame.type)
+        return mp_pin_in_needs_data(out);
+
+    if (!mp_pin_can_transfer_data(out, sub->in))
+        return false;
+
+    sub->frame = mp_pin_out_read(sub->in);
+    return true;
+}
+
+void mp_subfilter_reset(struct mp_subfilter *sub)
+{
+    if (sub->filter && sub->draining)
+        TA_FREEP(&sub->filter);
+    sub->draining = false;
+    mp_frame_unref(&sub->frame);
+}
+
+void mp_subfilter_continue(struct mp_subfilter *sub)
+{
+    struct mp_pin *out = sub->filter ? sub->filter->pins[0] : sub->out;
+    // It was made sure earlier that the pin is writable, unless the filter
+    // was newly created, or a previously existing filter (which was going to
+    // accept input) was destroyed. In those cases, essentially restart
+    // data flow.
+    if (!mp_pin_in_needs_data(out)) {
+        mark_progress(sub);
+        return;
+    }
+    mp_pin_in_write(out, sub->frame);
+    sub->frame = MP_NO_FRAME;
+}
+
+void mp_subfilter_destroy(struct mp_subfilter *sub)
+{
+    TA_FREEP(&sub->filter);
+    sub->draining = false;
+}
+
+bool mp_subfilter_drain_destroy(struct mp_subfilter *sub)
+{
+    if (!sub->draining && sub->filter) {
+        // We know the filter is writable (unless the user created a new filter
+        // and immediately called this function, which is invalid).
+        mp_pin_in_write(sub->filter->pins[0], MP_EOF_FRAME);
+        sub->draining = true;
+    }
+    return !sub->filter;
+}
+
+static const struct mp_filter_info bidir_nop_filter = {
+    .name = "nop",
+};
+
+struct mp_filter *mp_bidir_nop_filter_create(struct mp_filter *parent)
+{
+    struct mp_filter *f = mp_filter_create(parent, &bidir_nop_filter);
+    if (!f)
+        return NULL;
+
+    mp_filter_add_pin(f, MP_PIN_IN, "in");
+    mp_filter_add_pin(f, MP_PIN_OUT, "out");
+
+    mp_pin_connect(f->ppins[1], f->ppins[0]);
+
+    return f;
+}
diff --git a/filters/f_utils.h b/filters/f_utils.h
new file mode 100644
index 0000000000..a59ac1601d
--- /dev/null
+++ b/filters/f_utils.h
@@ -0,0 +1,72 @@
+#pragma once
+
+#include "filter.h"
+
+// Filter that computes the exact duration of video frames by buffering 1 frame,
+// and taking the PTS difference. This supports video frames only, and stores
+// the duration in mp_image.pkt_duration. All other frame types are passed
+// through.
+struct mp_filter *mp_compute_frame_duration_create(struct mp_filter *parent);
+
+// Given the filters[0..num_filters] array, connect in with the input of the
+// first filter, connect the output of the first filter to the input to the
+// second filter, etc., until out. All filters are assumed to be bidrectional,
+// with input on pin 0 and output on pin 1. NULL entries are skipped.
+void mp_chain_filters(struct mp_pin *in, struct mp_pin *out,
+                      struct mp_filter **filters, int num_filters);
+
+// Helper for maintaining a sub-filter that is created or destroyed on demand,
+// because it might depend on frame input formats or is otherwise dynamically
+// changing. (This is overkill for more static sub filters, or entirely manual
+// filtering.)
+// To initialize this, zero-init all fields, and set the in/out fields.
+struct mp_subfilter {
+    // These two fields must be set on init. The pins must have a manual
+    // connection to the filter whose process() function calls the
+    // mp_subfilter_*() functions.
+    struct mp_pin *in, *out;
+    // Temporary buffered frame, as triggered by mp_subfilter_read(). You can
+    // not mutate this (unless you didn't create or destroy sub->filter).
+    struct mp_frame frame;
+    // The sub-filter, set by the user. Can be NULL if disabled. If set, this
+    // must be a bidirectional filter, with manual connections same as
+    // mp_sub_filter.in/out (to get the correct process() function called).
+    // Set this only if it's NULL. You should not overwrite this if it's set.
+    // Use either mp_subfilter_drain_destroy(), mp_subfilter_destroy(), or
+    // mp_subfilter_reset() to unset and destroy the filter gracefully.
+    struct mp_filter *filter;
+    // Internal state.
+    bool draining;
+};
+
+// Make requests for a new frame.
+// Returns whether sub->frame is set to anything. If true is returned, you
+// must either call mp_subfilter_continue() or mp_subfilter_drain_destroy()
+// once to continue data flow normally (otherwise it will stall). If you call
+// mp_subfilter_drain_destroy(), and it returns true, or you call
+// mp_subfilter_destroy(), you can call mp_subfilter_continue() once after it.
+// If this returns true, sub->frame is never unset (MP_FRAME_NONE).
+bool mp_subfilter_read(struct mp_subfilter *sub);
+
+// Clear internal state (usually to be called by parent filter's reset(), or
+// destroy()). This usually does not free sub->filter.
+void mp_subfilter_reset(struct mp_subfilter *sub);
+
+// Continue filtering sub->frame. This can happen after setting a new filter
+// too.
+void mp_subfilter_continue(struct mp_subfilter *sub);
+
+// Destroy the filter immediately (if it's set). You must call
+// mp_subfilter_continue() after this to propagate sub->frame.
+void mp_subfilter_destroy(struct mp_subfilter *sub);
+
+// Make sure the filter is destroyed. Returns true if the filter was destroyed.
+// If this returns false, exit your process() function, so dataflow can
+// continue normally. (process() is repeated until this function returns true,
+// which can take a while if sub->filter has many frames buffered).
+// If this returns true, call mp_subfilter_continue() to propagate sub->frame.
+// The filter is destroyed with talloc_free(sub->filter).
+bool mp_subfilter_drain_destroy(struct mp_subfilter *sub);
+
+// A bidrectional filter which passes through all data.
+struct mp_filter *mp_bidir_nop_filter_create(struct mp_filter *parent);
diff --git a/filters/filter.c b/filters/filter.c
new file mode 100644
index 0000000000..bbd8a4ff5b
--- /dev/null
+++ b/filters/filter.c
@@ -0,0 +1,790 @@
+#include <pthread.h>
+
+#include "common/common.h"
+#include "common/global.h"
+#include "common/msg.h"
+#include "video/hwdec.h"
+
+#include "filter.h"
+#include "filter_internal.h"
+
+// Note about connections:
+// They can be confusing, because pins come in pairs, and multiple pins can be
+// transitively connected via mp_pin_connect(). To avoid dealing with this,
+// mp_pin.conn is used to skip redundant connected pins.
+// Consider <1a|1b> a symbol for mp_pin pair #1 and f1 as filter #1. Then:
+//      f1 <-> <1a|1b> <-> <2a|2b> <-> <3a|3b> <-> f2
+// would be a connection from 1a to 3b. 1a could be a private pin of f1 (e.g.
+// mp_filter.ppin[0]), and 1b would be the public pin (e.g. mp_filter.pin[0]).
+// A user could have called mp_pin_connect(2a, 1b) mp_pin_connect(3a, 2b)
+// (assuming 1b has dir==MP_PIN_OUT). The end result are the following values:
+//  pin  user_conn  conn   manual_connection within_conn (uses mp_pin.data)
+//   1a   NULL       3b     f1                false        no
+//   1b   2a         NULL   NULL              true         no
+//   2a   1b         NULL   NULL              true         no
+//   2b   3a         NULL   NULL              true         no
+//   3a   2b         NULL   NULL              true         no
+//   3b   NULL       1a     f2                false        yes
+// The minimal case of f1 <-> <1a|1b> <-> f2 (1b dir=out) would be:
+//   1a   NULL       1b     f1                false        no
+//   1b   NULL       1a     f2                false        yes
+// In both cases, only the final output pin uses mp_pin.data/data_requested.
+struct mp_pin {
+    const char *name;
+    enum mp_pin_dir dir;
+    struct mp_pin *other;           // paired mp_pin representing other end
+    struct mp_filter *owner;
+
+    struct mp_pin *user_conn;       // as set by mp_pin_connect()
+    struct mp_pin *conn;            // transitive, actual end of the connection
+
+    // Set if the pin is considered connected, but has no user_conn. pin
+    // state changes are handled by the given filter. (Defaults to the root
+    // filter if the pin is for the user of a filter graph.)
+    // As an invariant, conn and manual_connection are both either set or unset.
+    struct mp_filter *manual_connection;
+
+    // Set if the pin is indirect part of a connection chain, but not one of
+    // the end pins. Basically it's a redundant in-between pin. You never access
+    // these with the pin data flow functions, because only the end pins matter.
+    // This flag is for checking and enforcing this.
+    bool within_conn;
+
+    // This is used for the final output mp_pin in connections only.
+    bool data_requested;            // true if out wants new data
+    struct mp_frame data;           // possibly buffered frame (MP_FRAME_NONE if
+                                    // empty, usually only temporary)
+};
+
+// Root filters create this, all other filters reference it.
+struct filter_runner {
+    struct mpv_global *global;
+
+    void (*wakeup_cb)(void *ctx);
+    void *wakeup_ctx;
+
+    struct mp_filter *root_filter;
+
+    // If we're currently running the filter graph (for avoiding recursion).
+    bool filtering;
+
+    // Set of filters which need process() to be called. A filter is in this
+    // array iff mp_filter_internal.pending==true.
+    struct mp_filter **pending;
+    int num_pending;
+
+    // Any outside pins have changed state.
+    bool external_pending;
+
+    // For async notifications only. We don't bother making this fine grained
+    // across filters.
+    pthread_mutex_t async_lock;
+
+    // Wakeup is pending. Protected by async_lock.
+    bool async_wakeup_sent;
+
+    // Similar to pending[]. Uses mp_filter_internal.async_pending. Protected
+    // by async_lock.
+    struct mp_filter **async_pending;
+    int num_async_pending;
+};
+
+struct mp_filter_internal {
+    const struct mp_filter_info *info;
+
+    struct mp_filter *parent;
+    struct filter_runner *runner;
+
+    struct mp_filter **children;
+    int num_children;
+
+    struct mp_filter *error_handler;
+
+    char *name;
+
+    bool pending;
+    bool async_pending;
+    bool failed;
+};
+
+static void add_pending(struct mp_filter *f)
+{
+    struct filter_runner *r = f->in->runner;
+
+    if (f->in->pending)
+        return;
+
+    // This should probably really be some sort of priority queue, but for now
+    // something naive and dumb does the job too.
+    f->in->pending = true;
+    MP_TARRAY_APPEND(r, r->pending, r->num_pending, f);
+}
+
+// Called when new work needs to be done on a pin belonging to the filter:
+//  - new data was requested
+//  - new data has been queued
+//  - or just an connect/disconnect/async notification happened
+// This means the process function for this filter has to be called next.
+static void update_filter(struct mp_filter *src, struct mp_filter *f)
+{
+    assert(f);
+    struct filter_runner *r = f->in->runner;
+
+    // Make sure the filter knows it has to make progress.
+    if (src->in->runner != r) {
+        // Connected to a different graph. The user has to drive those manually,
+        // and we simplify tell the user via the mp_filter_run() return value.
+        r->external_pending = true;
+    } else if (!f->in->pending) {
+        add_pending(f);
+
+        if (!r->filtering) {
+            // Likely the "outer" API user used an external manually connected
+            // pin, so do recursive filtering (as a not strictly necessary
+            // feature which makes outside I/O with filters easier).
+            // Also don't lose the pending state, which the user may or may not
+            // care about.
+            // Note that we must avoid calling this from within filtering,
+            // because that would make the process() functions recursive and
+            // reentrant (and hard to reason about).
+            r->external_pending |= mp_filter_run(r->root_filter);
+        }
+
+        // Need to tell user that something changed.
+        if (f == r->root_filter)
+            r->external_pending = true;
+    }
+}
+
+void mp_filter_internal_mark_progress(struct mp_filter *f)
+{
+    struct filter_runner *r = f->in->runner;
+    assert(r->filtering); // only call from f's process()
+    add_pending(f);
+}
+
+// Basically copy the async notifications to the sync ones. Done so that the
+// sync notifications don't need any locking.
+static void flush_async_notifications(struct filter_runner *r, bool queue)
+{
+    pthread_mutex_lock(&r->async_lock);
+    for (int n = 0; n < r->num_async_pending; n++) {
+        struct mp_filter *f = r->async_pending[n];
+        if (queue)
+            add_pending(f);
+        f->in->async_pending = false;
+    }
+    r->num_async_pending = 0;
+    r->async_wakeup_sent = false;
+    pthread_mutex_unlock(&r->async_lock);
+}
+
+bool mp_filter_run(struct mp_filter *filter)
+{
+    struct filter_runner *r = filter->in->runner;
+
+    r->filtering = true;
+
+    flush_async_notifications(r, true);
+
+    while (r->num_pending) {
+        struct mp_filter *next = r->pending[r->num_pending - 1];
+        r->num_pending -= 1;
+        next->in->pending = false;
+
+        if (next->in->info->process)
+            next->in->info->process(next);
+    }
+
+    r->filtering = false;
+
+    bool externals = r->external_pending;
+    r->external_pending = false;
+    return externals;
+}
+
+bool mp_pin_can_transfer_data(struct mp_pin *dst, struct mp_pin *src)
+{
+    return mp_pin_in_needs_data(dst) && mp_pin_out_request_data(src);
+}
+
+bool mp_pin_transfer_data(struct mp_pin *dst, struct mp_pin *src)
+{
+    if (!mp_pin_can_transfer_data(dst, src))
+        return false;
+    mp_pin_in_write(dst, mp_pin_out_read(src));
+    return true;
+}
+
+bool mp_pin_in_needs_data(struct mp_pin *p)
+{
+    assert(p->dir == MP_PIN_IN);
+    assert(!p->within_conn);
+    return p->conn && p->conn->manual_connection && p->conn->data_requested;
+}
+
+bool mp_pin_in_write(struct mp_pin *p, struct mp_frame frame)
+{
+    if (!mp_pin_in_needs_data(p) || frame.type == MP_FRAME_NONE) {
+        if (frame.type)
+            MP_ERR(p->owner, "losing frame on %s\n", p->name);
+        mp_frame_unref(&frame);
+        return false;
+    }
+    assert(p->conn->data.type == MP_FRAME_NONE);
+    p->conn->data = frame;
+    p->conn->data_requested = false;
+    update_filter(p->owner, p->conn->manual_connection);
+    return true;
+}
+
+bool mp_pin_out_has_data(struct mp_pin *p)
+{
+    assert(p->dir == MP_PIN_OUT);
+    assert(!p->within_conn);
+    return p->conn && p->conn->manual_connection && p->data.type != MP_FRAME_NONE;
+}
+
+bool mp_pin_out_request_data(struct mp_pin *p)
+{
+    if (mp_pin_out_has_data(p))
+        return true;
+    if (p->conn && p->conn->manual_connection && !p->data_requested) {
+        p->data_requested = true;
+        update_filter(p->owner, p->conn->manual_connection);
+    }
+    return mp_pin_out_has_data(p);
+}
+
+struct mp_frame mp_pin_out_read(struct mp_pin *p)
+{
+    if (!mp_pin_out_request_data(p))
+        return MP_NO_FRAME;
+    struct mp_frame res = p->data;
+    p->data = MP_NO_FRAME;
+    return res;
+}
+
+void mp_pin_out_unread(struct mp_pin *p, struct mp_frame frame)
+{
+    assert(p->dir == MP_PIN_OUT);
+    assert(!p->within_conn);
+    assert(p->conn && p->conn->manual_connection);
+    // Unread is allowed strictly only if you didn't do anything else with
+    // the pin since the time you read it.
+    assert(!mp_pin_out_has_data(p));
+    assert(!p->data_requested);
+    p->data = frame;
+}
+
+void mp_pin_out_repeat_eof(struct mp_pin *p)
+{
+    mp_pin_out_unread(p, MP_EOF_FRAME);
+}
+
+// Follow mp_pin pairs/connection into the "other" direction of the pin, until
+// the last pin is found. (In the simplest case, this is just p->other.) E.g.:
+//      <1a|1b> <-> <2a|2b> <-> <3a|3b>
+//          find_connected_end(2b)==1a
+//          find_connected_end(1b)==1a
+//          find_connected_end(1a)==3b
+static struct mp_pin *find_connected_end(struct mp_pin *p)
+{
+    while (1) {
+        struct mp_pin *other = p->other;
+        if (!other->user_conn)
+            return other;
+        p = other->user_conn;
+    }
+    assert(0);
+}
+
+// With p being part of a connection, create the pin_connection and set all
+// state flags.
+static void init_connection(struct mp_pin *p)
+{
+    if (p->dir == MP_PIN_IN)
+        p = p->other;
+
+    struct mp_pin *in = find_connected_end(p);
+    struct mp_pin *out = find_connected_end(p->other);
+
+    // These are the "outer" pins by definition, they have no user connections.
+    assert(!in->user_conn);
+    assert(!out->user_conn);
+
+    // Logicaly, the ends are always manual connections. A pin chain without
+    // manual connections at the ends is still disconnected (or if this
+    // attempted to extend an existing connection, becomes dangling and gets
+    // disconnected).
+    if (!in->manual_connection && !out->manual_connection)
+        return;
+
+    assert(in->dir == MP_PIN_IN);
+    assert(out->dir == MP_PIN_OUT);
+
+    struct mp_pin *cur = in;
+    while (cur) {
+        assert(!cur->within_conn && !cur->other->within_conn);
+        assert(!cur->conn && !cur->other->conn);
+        assert(!cur->data_requested); // unused for in pins
+        assert(!cur->data.type); // unused for in pins
+        assert(!cur->other->data_requested); // unset for unconnected out pins
+        assert(!cur->other->data.type); // unset for unconnected out pins
+        cur->within_conn = cur->other->within_conn = true;
+        cur = cur->other->user_conn;
+    }
+
+    in->conn = out;
+    in->within_conn = false;
+    out->conn = in;
+    out->within_conn = false;
+
+    // Scheduling so far will be messed up.
+    add_pending(in->manual_connection);
+    add_pending(out->manual_connection);
+}
+
+void mp_pin_connect(struct mp_pin *dst, struct mp_pin *src)
+{
+    assert(src->dir == MP_PIN_OUT);
+    assert(dst->dir == MP_PIN_IN);
+
+    if (dst->user_conn == src) {
+        assert(src->user_conn == dst);
+        return;
+    }
+
+    mp_pin_disconnect(src);
+    mp_pin_disconnect(dst);
+
+    src->user_conn = dst;
+    dst->user_conn = src;
+
+    init_connection(src);
+}
+
+void mp_pin_set_manual_connection(struct mp_pin *p, bool connected)
+{
+    mp_pin_set_manual_connection_for(p, connected ? p->owner->in->parent : NULL);
+}
+
+void mp_pin_set_manual_connection_for(struct mp_pin *p, struct mp_filter *f)
+{
+    if (p->manual_connection == f)
+        return;
+    if (p->within_conn)
+        mp_pin_disconnect(p);
+    p->manual_connection = f;
+    init_connection(p);
+}
+
+struct mp_filter *mp_pin_get_manual_connection(struct mp_pin *p)
+{
+    return p->manual_connection;
+}
+
+static void deinit_connection(struct mp_pin *p)
+{
+    if (p->dir == MP_PIN_OUT)
+        p = p->other;
+
+    p = find_connected_end(p);
+
+    while (p) {
+        p->conn = p->other->conn = NULL;
+        p->within_conn = p->other->within_conn = false;
+        assert(!p->other->data_requested); // unused for in pins
+        assert(!p->other->data.type); // unused for in pins
+        p->data_requested = false;
+        if (p->data.type)
+            MP_WARN(p->owner, "dropping frame due to pin disconnect\n");
+        if (p->data_requested)
+            MP_WARN(p->owner, "dropping request due to pin disconnect\n");
+        mp_frame_unref(&p->data);
+        p = p->other->user_conn;
+    }
+}
+
+void mp_pin_disconnect(struct mp_pin *p)
+{
+    if (!mp_pin_is_connected(p))
+        return;
+
+    p->manual_connection = NULL;
+
+    struct mp_pin *conn = p->user_conn;
+    if (conn) {
+        p->user_conn = NULL;
+        conn->user_conn = NULL;
+        deinit_connection(conn);
+    }
+
+    deinit_connection(p);
+}
+
+bool mp_pin_is_connected(struct mp_pin *p)
+{
+    return p->user_conn || p->manual_connection;
+}
+
+const char *mp_pin_get_name(struct mp_pin *p)
+{
+    return p->name;
+}
+
+enum mp_pin_dir mp_pin_get_dir(struct mp_pin *p)
+{
+    return p->dir;
+}
+
+const char *mp_filter_get_name(struct mp_filter *f)
+{
+    return f->in->name;
+}
+
+void mp_filter_set_name(struct mp_filter *f, const char *name)
+{
+    talloc_free(f->in->name);
+    f->in->name = talloc_strdup(f, name);
+}
+
+struct mp_pin *mp_filter_get_named_pin(struct mp_filter *f, const char *name)
+{
+    for (int n = 0; n < f->num_pins; n++) {
+        if (name && strcmp(f->pins[n]->name, name) == 0)
+            return f->pins[n];
+    }
+    return NULL;
+}
+
+void mp_filter_set_error_handler(struct mp_filter *f, struct mp_filter *handler)
+{
+    f->in->error_handler = handler;
+}
+
+void mp_filter_internal_mark_failed(struct mp_filter *f)
+{
+    while (f) {
+        f->in->failed = true;
+        if (f->in->error_handler) {
+            add_pending(f->in->error_handler);
+            break;
+        }
+        f = f->in->parent;
+    }
+}
+
+bool mp_filter_has_failed(struct mp_filter *filter)
+{
+    bool failed = filter->in->failed;
+    filter->in->failed = false;
+    return failed;
+}
+
+static void reset_pin(struct mp_pin *p)
+{
+    if (!p->conn || p->dir != MP_PIN_OUT) {
+        assert(!p->data.type);
+        assert(!p->data_requested);
+    }
+    mp_frame_unref(&p->data);
+    p->data_requested = false;
+}
+
+void mp_filter_reset(struct mp_filter *filter)
+{
+    for (int n = 0; n < filter->in->num_children; n++)
+        mp_filter_reset(filter->in->children[n]);
+
+    for (int n = 0; n < filter->num_pins; n++) {
+        struct mp_pin *p = filter->ppins[n];
+        reset_pin(p);
+        reset_pin(p->other);
+    }
+
+    if (filter->in->info->reset)
+        filter->in->info->reset(filter);
+}
+
+struct mp_pin *mp_filter_add_pin(struct mp_filter *f, enum mp_pin_dir dir,
+                                 const char *name)
+{
+    assert(dir == MP_PIN_IN || dir == MP_PIN_OUT);
+    assert(name && name[0]);
+    assert(!mp_filter_get_named_pin(f, name));
+
+    // "Public" pin
+    struct mp_pin *p = talloc_ptrtype(NULL, p);
+    *p = (struct mp_pin){
+        .name = talloc_strdup(p, name),
+        .dir = dir,
+        .owner = f,
+        .manual_connection = f->in->parent,
+    };
+
+    // "Private" paired pin
+    p->other = talloc_ptrtype(NULL, p);
+    *p->other = (struct mp_pin){
+        .name = p->name,
+        .dir = p->dir == MP_PIN_IN ? MP_PIN_OUT : MP_PIN_IN,
+        .owner = f,
+        .other = p,
+        .manual_connection = f,
+    };
+
+    MP_TARRAY_GROW(f, f->pins, f->num_pins);
+    MP_TARRAY_GROW(f, f->ppins, f->num_pins);
+    f->pins[f->num_pins] = p;
+    f->ppins[f->num_pins] = p->other;
+    f->num_pins += 1;
+
+    init_connection(p);
+
+    return p->other;
+}
+
+void mp_filter_remove_pin(struct mp_filter *f, struct mp_pin *p)
+{
+    if (!p)
+        return;
+
+    assert(p->owner == f);
+    mp_pin_disconnect(p);
+    mp_pin_disconnect(p->other);
+
+    int index = -1;
+    for (int n = 0; n < f->num_pins; n++) {
+        if (f->ppins[n] == p) {
+            index = n;
+            break;
+        }
+    }
+    assert(index >= 0);
+
+    talloc_free(f->pins[index]);
+    talloc_free(f->ppins[index]);
+
+    int count = f->num_pins;
+    MP_TARRAY_REMOVE_AT(f->pins, count, index);
+    count = f->num_pins;
+    MP_TARRAY_REMOVE_AT(f->ppins, count, index);
+    f->num_pins -= 1;
+}
+
+bool mp_filter_command(struct mp_filter *f, struct mp_filter_command *cmd)
+{
+    return f->in->info->command ? f->in->info->command(f, cmd) : false;
+}
+
+struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f)
+{
+    while (f) {
+        if (f->stream_info)
+            return f->stream_info;
+        f = f->in->parent;
+    }
+    return NULL;
+}
+
+struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int avtype)
+{
+    struct mp_stream_info *info = mp_filter_find_stream_info(f);
+    if (!info || !info->hwdec_devs)
+        return NULL;
+
+    hwdec_devices_request_all(info->hwdec_devs);
+
+    return hwdec_devices_get_lavc(info->hwdec_devs, avtype);
+}
+
+static void filter_wakeup(struct mp_filter *f, bool mark_only)
+{
+    struct filter_runner *r = f->in->runner;
+    pthread_mutex_lock(&r->async_lock);
+    if (!f->in->async_pending) {
+        f->in->async_pending = true;
+        // (not using a talloc parent for thread safety reasons)
+        MP_TARRAY_APPEND(NULL, r->async_pending, r->num_async_pending, f);
+        if (!mark_only && !r->async_wakeup_sent) {
+            if (r->wakeup_cb)
+                r->wakeup_cb(r->wakeup_ctx);
+            r->async_wakeup_sent = true;
+        }
+    }
+    pthread_mutex_unlock(&r->async_lock);
+}
+
+void mp_filter_wakeup(struct mp_filter *f)
+{
+    filter_wakeup(f, false);
+}
+
+void mp_filter_mark_async_progress(struct mp_filter *f)
+{
+    filter_wakeup(f, true);
+}
+
+void mp_filter_free_children(struct mp_filter *f)
+{
+    while(f->in->num_children)
+        talloc_free(f->in->children[0]);
+}
+
+static void filter_destructor(void *p)
+{
+    struct mp_filter *f = p;
+    struct filter_runner *r = f->in->runner;
+
+    if (f->in->info->destroy)
+        f->in->info->destroy(f);
+
+    // For convenience, free child filters.
+    mp_filter_free_children(f);
+
+    while (f->num_pins)
+        mp_filter_remove_pin(f, f->ppins[0]);
+
+    // Just make sure the filter is not still in the async notifications set.
+    // There will be no more new notifications at this point (due to destroy()).
+    flush_async_notifications(r, false);
+
+    for (int n = 0; n < r->num_pending; n++) {
+        if (r->pending[n] == f) {
+            MP_TARRAY_REMOVE_AT(r->pending, r->num_pending, n);
+            break;
+        }
+    }
+
+    if (f->in->parent) {
+        struct mp_filter_internal *p_in = f->in->parent->in;
+        for (int n = 0; n < p_in->num_children; n++) {
+            if (p_in->children[n] == f) {
+                MP_TARRAY_REMOVE_AT(p_in->children, p_in->num_children, n);
+                break;
+            }
+        }
+    }
+
+    if (r->root_filter == f) {
+        assert(!f->in->parent);
+        pthread_mutex_destroy(&r->async_lock);
+        talloc_free(r->async_pending);
+        talloc_free(r);
+    }
+}
+
+
+struct mp_filter *mp_filter_create_with_params(struct mp_filter_params *params)
+{
+    struct mp_filter *f = talloc(NULL, struct mp_filter);
+    talloc_set_destructor(f, filter_destructor);
+    *f = (struct mp_filter){
+        .priv = params->info->priv_size ?
+                    talloc_zero_size(f, params->info->priv_size) : NULL,
+        .global = params->global,
+        .in = talloc(f, struct mp_filter_internal),
+    };
+    *f->in = (struct mp_filter_internal){
+        .info = params->info,
+        .parent = params->parent,
+        .runner = params->parent ? params->parent->in->runner : NULL,
+    };
+
+    if (!f->in->runner) {
+        assert(params->global);
+
+        f->in->runner = talloc(NULL, struct filter_runner);
+        *f->in->runner = (struct filter_runner){
+            .global = params->global,
+            .root_filter = f,
+        };
+        pthread_mutex_init(&f->in->runner->async_lock, NULL);
+    }
+
+    if (!f->global)
+        f->global = f->in->runner->global;
+
+    if (f->in->parent) {
+        struct mp_filter_internal *parent = f->in->parent->in;
+        MP_TARRAY_APPEND(parent, parent->children, parent->num_children, f);
+    }
+
+    f->log = mp_log_new(f, f->global->log, params->info->name);
+
+    if (f->in->info->init) {
+        if (!f->in->info->init(f, params)) {
+            talloc_free(f);
+            return NULL;
+        }
+    }
+
+    return f;
+}
+
+struct mp_filter *mp_filter_create(struct mp_filter *parent,
+                                   const struct mp_filter_info *info)
+{
+    assert(parent);
+    assert(info);
+    struct mp_filter_params params = {
+        .info = info,
+        .parent = parent,
+    };
+    return mp_filter_create_with_params(&params);
+}
+
+// (the root filter is just a dummy filter - nothing special about it, except
+// that it has no parent, and serves as manual connection for "external" pins)
+static const struct mp_filter_info filter_root = {
+    .name = "root",
+};
+
+struct mp_filter *mp_filter_create_root(struct mpv_global *global)
+{
+    struct mp_filter_params params = {
+        .info = &filter_root,
+        .global = global,
+    };
+    return mp_filter_create_with_params(&params);
+}
+
+void mp_filter_root_set_wakeup_cb(struct mp_filter *root,
+                                  void (*wakeup_cb)(void *ctx), void *ctx)
+{
+    struct filter_runner *r = root->in->runner;
+    pthread_mutex_lock(&r->async_lock);
+    r->wakeup_cb = wakeup_cb;
+    r->wakeup_ctx = ctx;
+    pthread_mutex_unlock(&r->async_lock);
+}
+
+static const char *filt_name(struct mp_filter *f)
+{
+    return f ? f->in->info->name : "-";
+}
+
+static void dump_pin_state(struct mp_filter *f, struct mp_pin *pin)
+{
+    MP_WARN(f, "  [%p] %s %s c=%s[%p] f=%s[%p] m=%s[%p] %s %s %s\n",
+        pin, pin->name, pin->dir == MP_PIN_IN ? "->" : "<-",
+        pin->user_conn ? filt_name(pin->user_conn->owner) : "-", pin->user_conn,
+        pin->conn ? filt_name(pin->conn->owner) : "-", pin->conn,
+        filt_name(pin->manual_connection), pin->manual_connection,
+        pin->within_conn ? "(within)" : "",
+        pin->data_requested ? "(request)" : "",
+        mp_frame_type_str(pin->data.type));
+}
+
+void mp_filter_dump_states(struct mp_filter *f)
+{
+    MP_WARN(f, "%s[%p] (%s[%p])\n", filt_name(f), f,
+            filt_name(f->in->parent), f->in->parent);
+    for (int n = 0; n < f->num_pins; n++) {
+        dump_pin_state(f, f->pins[n]);
+        dump_pin_state(f, f->ppins[n]);
+    }
+
+    for (int n = 0; n < f->in->num_children; n++)
+        mp_filter_dump_states(f->in->children[n]);
+}
diff --git a/filters/filter.h b/filters/filter.h
new file mode 100644
index 0000000000..3fd8af9195
--- /dev/null
+++ b/filters/filter.h
@@ -0,0 +1,379 @@
+#pragma once
+
+#include <stdbool.h>
+
+#include "frame.h"
+
+struct mpv_global;
+struct mp_filter;
+
+// A filter input or output. These always come in pairs: one mp_pin is for
+// input, the other is for output. (The separation is mostly for checking
+// their API use, and for the connection functions.)
+// Effectively, this is a 1-frame queue. The data flow rules have the goal to
+// reduce the number of buffered frames and the amount of time they are
+// buffered.
+// A mp_pin must be connected to be usable. The default state of a mp_pin is
+// a manual connection, which means you use the mp_pin_*() functions to
+// manually read or write data.
+struct mp_pin;
+
+enum mp_pin_dir {
+    MP_PIN_INVALID = 0, // used as a placeholder value
+    MP_PIN_IN,          // you write data to the pin
+    MP_PIN_OUT,         // you read data from the pin
+};
+
+// The established direction for this pin. The direction of a pin is immutable.
+// You must use the mp_pin_in_*() and mp_pin_out_*() functions on the correct
+// pin type - mismatching it is an API violation.
+enum mp_pin_dir mp_pin_get_dir(struct mp_pin *p);
+
+// True if a new frame should be written to the pin.
+bool mp_pin_in_needs_data(struct mp_pin *p);
+
+// Write a frame to the pin. If the input was not accepted, false is returned
+// (does not normally happen, as long as mp_pin_in_needs_data() returned true).
+// The callee owns the reference to the frame data, even on failure.
+// Writing a MP_FRAME_NONE has no effect (and returns false).
+bool mp_pin_in_write(struct mp_pin *p, struct mp_frame frame);
+
+// True if a frame is actually available for reading right now, and
+// mp_pin_out_read() will return success. If this returns false, the pin is
+// flagged for needing data (the filter might either produce output the next
+// time it's run, or request new input).
+// You should call this only if you can immediately consume the data. The goal
+// is to have no redundant buffering in the filter graph, and leaving frames
+// buffered in mp_pins goes against this.
+bool mp_pin_out_request_data(struct mp_pin *p);
+
+// Same as mp_pin_out_request_data(), but does not attempt to procure new frames
+// if the return value is false.
+bool mp_pin_out_has_data(struct mp_pin *p);
+
+// Read a frame. Returns MP_FRAME_NONE if currently no frame is available.
+// You need to call mp_pin_out_request_data() and wait until the frame is ready
+// to be sure this returns a frame. (This call implicitly calls _request if no
+// frame is available, but to get proper data flow in filters, you should
+// probably follow the preferred conventions.)
+// If no frame is returned, a frame is automatically requested via
+// mp_pin_out_request_data() (so it might be retuned in the future).
+// If a frame is returned, no new frame is automatically requested (this is
+// usually not wanted, because it could lead to additional buffering).
+// This is guaranteed to return a non-NONE frame if mp_pin_out_has_data()
+// returned true and no other filter functions were called.
+// The caller owns the reference to the returned data.
+struct mp_frame mp_pin_out_read(struct mp_pin *p);
+
+// Undo mp_pin_out_read(). This should be only used in special cases. Normally,
+// you should make an effort to reduce buffering, which means you signal that
+// you need a frame only once you know that you can use it (meaning you'll
+// really use it and have no need to "undo" the read). But in special cases,
+// especially if the behavior depends on the exact frame data, using this might
+// be justified.
+// If this is called, the next mp_pin_out_read() call will return the same frame
+// again. You must not have called mp_pin_out_request_data() on this pin and
+// you must not have disconnected or changed the pin in any way.
+// This does not mark the filter for progress, i.e. the filter's process()
+// function won't be repeated (unless other pins change). If you really need
+// that, call mp_filter_internal_mark_progress() manually in addition.
+void mp_pin_out_unread(struct mp_pin *p, struct mp_frame frame);
+
+// A helper to make draining on MP_FRAME_EOF frames easier. For filters which
+// buffer data, but have no easy way to buffer MP_FRAME_EOF frames natively.
+// This is to be used as follows:
+//  1. caller receives MP_FRAME_EOF
+//  2. initiates draining (or continues, see step 4.)
+//  2b. if there are no more buffered frames, just propagates the EOF frame and
+//      exits
+//  3. calls mp_pin_out_repeat_eof(pin)
+//  4. returns a buffered frame normally, and continues normally
+//  4b. pin returns "repeated" MP_FRAME_EOF, jump to 1.
+//  5. if there's nothing more to do, stop
+//  5b. there might be a sporadic wakeup, and an unwanted wait for output (in
+//      a typical filter implementation)
+// You must not have requested data before calling this. (Usually you'd call
+// this after mp_pin_out_read(). Requesting data after queuing the repeat EOF
+// is OK and idempotent.)
+// This is equivalent to mp_pin_out_unread(p, MP_EOF_FRAME). See that function
+// for further remarks.
+void mp_pin_out_repeat_eof(struct mp_pin *p);
+
+// Trivial helper to determine whether src is readable and dst is writable right
+// now. Defers or requests new data if not ready. This means it has the side
+// effect of telling the filters that you want to transfer data.
+// You use this in a filter process() function. If the result is false, it will
+// have requested new output from src, and your process() function will be
+// called again once src has output and dst is accepts input (the latest).
+bool mp_pin_can_transfer_data(struct mp_pin *dst, struct mp_pin *src);
+
+// Trivial helper to copy data between two manual pins. This uses filter data
+// flow - so if data can't be copied, it requests the pins to make it possible
+// on the next filter run. This implies you call this either from a filter
+// process() function, or call it manually when needed. Also see
+// mp_pin_can_transfer_data(). Returns whether a transfer happened.
+bool mp_pin_transfer_data(struct mp_pin *dst, struct mp_pin *src);
+
+// Connect src and dst, for automatic data flow. pin a will reflect the request
+// state of pin src, and accept and pass down frames to dst when appropriate.
+// src must be MP_PIN_OUT, dst must be MP_PIN_IN.
+// Previous connections are always removed. If the pins were already connected,
+// no action is taken.
+// Creating circular connections will just cause infinite recursion or such.
+// Both API user and filter implementations can use this, but always only on
+// the pins they're allowed to access.
+void mp_pin_connect(struct mp_pin *dst, struct mp_pin *src);
+
+// Enable manual filter access. This means you want to directly use the
+// mp_pin_in*() and mp_pin_out_*() functions for data flow.
+// Always severs previous connections.
+void mp_pin_set_manual_connection(struct mp_pin *p, bool connected);
+
+// Enable manual filter access, like mp_pin_set_manual_connection(). In
+// addition, this specifies which filter's process function should be invoked
+// on pin state changes. Using mp_pin_set_manual_connection() will default to
+// the parent filter for this.
+// Passing f=NULL disconnects.
+void mp_pin_set_manual_connection_for(struct mp_pin *p, struct mp_filter *f);
+
+// Return the manual connection for this pin, or NULL if none.
+struct mp_filter *mp_pin_get_manual_connection(struct mp_pin *p);
+
+// If not connected, this will produce EOF for MP_PIN_IN, and never request
+// data for MP_PIN_OUT.
+void mp_pin_disconnect(struct mp_pin *p);
+
+// Return whether a connection was set on this pin. Note that this is not
+// transitive (if the pin is connected to an pin with no further connections,
+// there is no active connection, but this still returns true).
+bool mp_pin_is_connected(struct mp_pin *p);
+
+// Return a symbolic name of the pin. Usually it will be something redundant
+// (like "in" or "out"), or something the user set.
+// The returned pointer is valid as long as the mp_pin is allocated.
+const char *mp_pin_get_name(struct mp_pin *p);
+
+/**
+ * A filter converts input frames to output frames (mp_frame, usually audio or
+ * video data). It can support multiple inputs and outputs. Data always flows
+ * through mp_pin instances.
+ *
+ * --- General rules for data flow:
+ *
+ * All data goes through mp_pin (present in the mp_filter inputs/outputs list).
+ * Actual work is done in the filter's process() function. This function
+ * queries whether input mp_pins have data and output mp_pins require data. If
+ * both is the case, a frame is read, filtered, and written to the output.
+ * Depending on the filter type, the filter might internally buffer data (e.g.
+ * things that require readahead). But in general, a filter should not request
+ * input before output is needed.
+ *
+ * The general goal is to reduce the amount of data buffered. If buffering is
+ * actually desired, explicit filters for buffering have to be introduced into
+ * the filter chain.
+ *
+ * Multiple filters are driven by letting mp_pin flag filters which need
+ * process() to be called. The process starts by requesting output from the
+ * last filter. The requests will "bubble up" by iteratively calling process()
+ * on each filter, which will request further input, until input on the first
+ * filter's input pin is requested. The API user feeds it a frame, which will
+ * call the first filter's process() function, which will filter and output
+ * the frame, and the frame is iteratively filtered until it reaches the output.
+ * (Depending on implementation, some if this wil be recursive not iterative.)
+ *
+ * --- General rules for thread safety:
+ *
+ * Filters are by default not thread safe. However, some filters can be
+ * partially thread safe and allow certain functions to be accessed from
+ * foreign threads. The common filter code itself is not thread safe, except
+ * for some utility functions explicitly marked as such, and which are meant
+ * to make implementing threaded filters easier.
+ *
+ * --- Rules for manual connections:
+ *
+ * A pin can be marked for manual connection via mp_pin_set_manual_connection().
+ * It's also the default. These have two uses:
+ *
+ *      1. filter internal (the filter actually does something with a frame)
+ *      2. filter user manually feeding/retrieving frames
+ *
+ * Basically, a manual connection means someone uses the mp_pin_in_*() or
+ * mp_pin_out_*() functions on a pin. The alternative is an automatic connection
+ * made via mp_pin_connect(). Manual connections need special considerations
+ * for wakeups:
+ *
+ * Internal manual pins (within a filter) will invoke the filter's process()
+ * function, and the filter polls the state of all pins to see if anything
+ * needs to be filtered or requested.
+ *
+ * External manual pins (filter user) require the user to poll all manual pins
+ * that are part of the graph. In addition, the filter's wakeup callback must be
+ * set, and trigger repolling all pins. This is needed in case any filters do
+ * async filtering internally.
+ *
+ * --- Rules for filters with multiple inputs or outputs:
+ *
+ * The generic filter code does not do any kind of scheduling. It's the filter's
+ * responsibility to request frames from input when needed, and to avoid
+ * internal excessive buffering if outputs aren't read.
+ *
+ * --- Rules for async filters:
+ *
+ * Async filters will have a synchronous interface with asynchronous waiting.
+ * They change mp_pin data flow to being poll based, with a wakeup mechanism to
+ * avoid active waiting. Once polling results in no change, the API user can go
+ * to sleep, and wait until the wakeup callback set via mp_filter_create_root()
+ * is invoked. Then it can poll the filters again. Internally, filters use
+ * mp_filter_wakeup() to get their process() function invoked on the user
+ * thread, and update the mp_pin states.
+ *
+ * For running parts of a filter graph on a different thread, f_async_queue.h
+ * can be used.
+ *
+ * --- Format conversions and mid-stream format changes:
+ *
+ * Generally, all filters must support all formats, as well as mid-stream
+ * format changes. If they don't, they will have to error out. There are some
+ * helpers for dealing with these two things.
+ *
+ * mp_pin_out_unread() can temporarily put back an input frame. If the input
+ * format changed, and you have to drain buffered data, you can put back the
+ * frame every time you output a buffered frame. Once all buffered data is
+ * drained this way, you can actually change the internal filter state to the
+ * new format, and actually consume the input frame.
+ *
+ * There is an f_autoconvert filter, which lets you transparently convert to
+ * a set of target formats (and which passes through the data if no conversion
+ * is needed).
+ *
+ * --- Rules for format negotiation:
+ *
+ * Since libavfilter does not provide _any_ kind of format negotiation to the
+ * user, and most filters use the libavfilter wrapper anyway, this is pretty
+ * broken and rudimentary. (The only thing libavfilter provides is that you
+ * can try to create a filter with a specific input format. Then you get
+ * either failure, or an output format. It involves actually initializing all
+ * filters, so a try run is not cheap or even side effect free.)
+ */
+struct mp_filter {
+    // Private state for the filter implementation. API users must not access
+    // this.
+    void *priv;
+
+    struct mpv_global *global;
+    struct mp_log *log;
+
+    // Array of public pins. API users can read this, but are not allowed to
+    // modify the array. Filter implementations use mp_filter_add_pin() to add
+    // pins to the array. The array is in order of the add calls.
+    // Most filters will use pins[0] for input (MP_PIN_IN), and pins[1] for
+    // output (MP_PIN_OUT). This is the default convention for filters. Some
+    // filters may have more complex usage, and assign pin entries with
+    // different meanings.
+    // The filter implementation must not use this. It must access ppins[]
+    // instead.
+    struct mp_pin **pins;
+    int num_pins;
+
+    // Internal pins, for access by the filter implementation. The meaning of
+    // in/out is swapped from the public interface: inputs use MP_PIN_OUT,
+    // because the filter reads from the inputs, and outputs use MP_PIN_IN,
+    // because the filter writes to them. ppins[n] always corresponds to pin[n],
+    // with swapped direction, and implicit data flow between the two.
+    // Outside API users must not access this.
+    struct mp_pin **ppins;
+
+    // Dumb garbage.
+    struct mp_stream_info *stream_info;
+
+    // Private state for the generic filter code.
+    struct mp_filter_internal *in;
+};
+
+// Return a symbolic name, which is set at init time. NULL if no name.
+// Valid until filter is destroyed or next mp_filter_set_name() call.
+const char *mp_filter_get_name(struct mp_filter *f);
+
+// Change mp_filter_get_name() return value.
+void mp_filter_set_name(struct mp_filter *f, const char *name);
+
+// Get a pin from f->pins[] for which mp_pin_get_name() returns the same name.
+// If name is NULL, always return NULL.
+struct mp_pin *mp_filter_get_named_pin(struct mp_filter *f, const char *name);
+
+// Return true if the filter has failed in some fatal way that does not allow
+// it to continue. This resets the error state (but does not reset the child
+// failed status on any parent filter).
+bool mp_filter_has_failed(struct mp_filter *filter);
+
+// Invoke mp_filter_info.reset on this filter and all children (but not
+// other filters connected via pins).
+void mp_filter_reset(struct mp_filter *filter);
+
+enum mp_filter_command_type {
+    MP_FILTER_COMMAND_TEXT = 1,
+    MP_FILTER_COMMAND_GET_META,
+};
+
+struct mp_filter_command {
+    enum mp_filter_command_type type;
+
+    // For MP_FILTER_COMMAND_TEXT
+    const char *cmd;
+    const char *arg;
+
+    // For MP_FILTER_COMMAND_GET_META
+    void *res; // must point to struct mp_tags*, will be set to new instance
+};
+
+// Run a command on the filter. Returns success. For libavfilter.
+bool mp_filter_command(struct mp_filter *f, struct mp_filter_command *cmd);
+
+// Specific information about a sub-tree in a filter graph. Currently, this is
+// mostly used to give filters access to VO mechanisms and capabilities.
+struct mp_stream_info {
+    void *priv; // for use by whoever implements the callbacks
+
+    double (*get_display_fps)(struct mp_stream_info *i);
+    double (*get_container_fps)(struct mp_stream_info *i);
+
+    struct mp_hwdec_devices *hwdec_devs;
+    struct osd_state *osd;
+    bool rotate90;
+};
+
+// Search for a parent filter (including f) that has this set, and return it.
+struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f);
+
+struct AVBufferRef;
+struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int avtype);
+
+// Perform filtering. This runs until the filter graph is blocked (due to
+// missing external input or unread output). It returns whether any outside
+// pins have changed state.
+// Does not perform recursive filtering to connected filters with different
+// root filter, though it notifies them.
+bool mp_filter_run(struct mp_filter *f);
+
+// Create a root dummy filter with no inputs or outputs. This fulfills the
+// following functions:
+// - passing it as parent filter to top-level filters
+// - driving the filter loop between the shared filters
+// - setting the wakeup callback for async filtering
+// - implicitly passing down global data like mpv_global and keeping filter
+//   constructor functions simple
+// Note that you can still connect pins of filters with different parents or
+// root filters, but then you may have to manually invoke mp_filter_run() on
+// the root filters of the connected filters to drive data flow.
+struct mp_filter *mp_filter_create_root(struct mpv_global *global);
+
+// Asynchronous filters may need to wakeup the user thread if the status of any
+// mp_pin has changed. If this is called, the callback provider should get the
+// user's thread to call mp_filter_run() again.
+// The wakeup callback must not recursively call into any filter APIs, or do
+// blocking waits on the filter API (deadlocks will happen).
+void mp_filter_root_set_wakeup_cb(struct mp_filter *root,
+                                  void (*wakeup_cb)(void *ctx), void *ctx);
+
+// Debugging internal stuff.
+void mp_filter_dump_states(struct mp_filter *f);
diff --git a/filters/filter_internal.h b/filters/filter_internal.h
new file mode 100644
index 0000000000..bd901db7d8
--- /dev/null
+++ b/filters/filter_internal.h
@@ -0,0 +1,144 @@
+#pragma once
+
+#include <stddef.h>
+
+#include "filter.h"
+
+// Flag the thread as needing mp_filter_process() to be called. Useful for
+// (some) async filters only. Idempotent.
+// Explicitly thread-safe.
+void mp_filter_wakeup(struct mp_filter *f);
+
+// Same as mp_filter_wakeup(), but skip the wakeup, and only mark the filter
+// as requiring processing to possibly update pin states changed due to async
+// processing.
+// Explicitly thread-safe.
+void mp_filter_mark_async_progress(struct mp_filter *f);
+
+// Flag the thread as needing mp_filter_process() to be called. Unlike
+// mp_filter_wakeup(), not thread-safe, and must be called from the process()
+// function of f (in exchange this is very light-weight).
+// In practice, this means process() is repeated.
+void mp_filter_internal_mark_progress(struct mp_filter *f);
+
+// Flag the filter as having failed, and propagate the error to the parent
+// filter. The error propagation stops either at the root filter, or if a filter
+// has an error handler set.
+// Must be called from f's process function.
+void mp_filter_internal_mark_failed(struct mp_filter *f);
+
+// If handler is not NULL, then if filter f errors, don't propagate the error
+// flag to its parent. Also invoke the handler's process() function, which is
+// supposed to use mp_filter_has_failed(f) to check any filters for which it has
+// set itself as error handler.
+// A filter must manually unset itself as error handler if it gets destroyed
+// before the filter f, otherwise dangling pointers will occur.
+void mp_filter_set_error_handler(struct mp_filter *f, struct mp_filter *handler);
+
+// Add a pin. Returns the private handle (same as f->ppins[f->num_pins-1]).
+// The name must be unique across all filter pins (you must verify this
+// yourself if filter names are from user input). name=="" is not allowed.
+// Never returns NULL. dir should be the external filter direction (a filter
+// input will use dir==MP_PIN_IN, and the returned pin will use MP_PIN_OUT,
+// because the internal pin is the opposite end of the external one).
+struct mp_pin *mp_filter_add_pin(struct mp_filter *f, enum mp_pin_dir dir,
+                                 const char *name);
+
+// Remove and deallocate a pin. The caller must be sure that nothing else
+// references the pin anymore. You must pass the private pin (from
+// mp_filter.ppin). This removes/deallocates the public paired pin as well.
+void mp_filter_remove_pin(struct mp_filter *f, struct mp_pin *p);
+
+// Free all filters which have f as parent. (This has nothing to do with
+// talloc.)
+void mp_filter_free_children(struct mp_filter *f);
+
+struct mp_filter_params;
+
+struct mp_filter_info {
+    // Informational name, in some cases might be used for filter discovery.
+    const char *name;
+
+    // mp_filter.priv is set to memory allocated with this size (if > 0)
+    size_t priv_size;
+
+    // Called during mp_filter_create(). Optional, can be NULL if use of a
+    // constructor function is required, which sets up the real filter after
+    // creation. Actually turns out nothing uses this.
+    bool (*init)(struct mp_filter *f, struct mp_filter_params *params);
+
+    // Free internal resources. Optional.
+    void (*destroy)(struct mp_filter *f);
+
+    // Called if any mp_pin was signalled (i.e. likely new data to process), or
+    // an async wakeup was received some time earlier.
+    // Generally, the implementation would consist of 2 stages:
+    //  1. check for the pin states, possibly request/probe for input/output
+    //  2. if data flow can happen, read a frame, perform actual work, write
+    //     result
+    // The process function will usually run very often, when pin states are
+    // updated, so the generic code can determine where data flow can happen.
+    // The common case will be that process() is called running stage 1 a bunch
+    // of times, until it finally can run stage 2 too.
+    // Optional.
+    void (*process)(struct mp_filter *f);
+
+    // Clear internal state and buffers (e.g. on seeks). Filtering can restart
+    // after this, and all settings are preserved. It makes sense to preserve
+    // internal resources for further filtering as well if you can.
+    // Input/output pins are always cleared by the common code before invoking
+    // this callback.
+    // Optional, can be NULL for filters without state.
+    // Don't create or destroy filters in this function, don't reconnect pins,
+    // don't access pins.
+    void (*reset)(struct mp_filter *f);
+
+    // Send a command to the filter. Highly implementation specific, usually
+    // user-initiated. Optional.
+    bool (*command)(struct mp_filter *f, struct mp_filter_command *cmd);
+};
+
+// Create a filter instance. Returns NULL on failure.
+// Destroy/free with talloc_free().
+// This is for filter implementers only. Filters are created with their own
+// constructor functions (instead of a generic one), which call this function
+// to create the filter itself.
+// parent is never NULL; use mp_filter_create_root() to create a top most
+// filter.
+// The parent does not imply anything about the position of the filter in
+// the dataflow (only the mp_pin connections matter). The parent exists for
+// convenience, which includes:
+//  - passing down implicit and explicit parameters (such as the filter driver
+//    loop)
+//  - auto freeing child filters if the parent is free'd
+//  - propagating errors
+//  - setting the parent as default manual connection for new external filter
+//    pins
+// The parent filter stays valid for the lifetime of any filters having it
+// directly or indirectly as parent. If the parent is free'd, all children are
+// automatically free'd.
+// All filters in the same parent tree must be driven in the same thread (or be
+// explicitly synchronized otherwise).
+// Driving the parent (or root) filter with mp_filter_run() will make sure this
+// filter is driven too, without having to resort to recursion.
+struct mp_filter *mp_filter_create(struct mp_filter *parent,
+                                   const struct mp_filter_info *info);
+
+struct mp_filter_params {
+    // Identifies the filter and its implementation. The pointer must stay
+    // valid for the life time of the created filter instance.
+    const struct mp_filter_info *info;
+
+    // Must be set if global==NULL. See mp_filter_create() for remarks.
+    struct mp_filter *parent;
+
+    // Must be set if parent==NULL, can otherwise be NULL.
+    struct mpv_global *global;
+
+    // Filter specific parameters. Most filters will have a constructor
+    // function, and pass in something internal.
+    void *params;
+};
+
+// Same as mp_filter_create(), but technically more flexible.
+struct mp_filter *mp_filter_create_with_params(struct mp_filter_params *params);
diff --git a/filters/frame.c b/filters/frame.c
new file mode 100644
index 0000000000..6c5b28e77a
--- /dev/null
+++ b/filters/frame.c
@@ -0,0 +1,179 @@
+#include <libavutil/frame.h>
+
+#include "audio/aframe.h"
+#include "common/av_common.h"
+#include "video/mp_image.h"
+
+#include "frame.h"
+
+struct frame_handler {
+    const char *name;
+    bool is_data;
+    bool is_signaling;
+    void *(*new_ref)(void *data);
+    // The following must be non-NULL if new_ref is non-NULL.
+    double (*get_pts)(void *data);
+    void (*set_pts)(void *data, double pts);
+    AVFrame *(*new_av_ref)(void *data);
+    void *(*from_av_ref)(AVFrame *data);
+    void (*free)(void *data);
+};
+
+static void *video_ref(void *data)
+{
+    return mp_image_new_ref(data);
+}
+
+static double video_get_pts(void *data)
+{
+    return ((struct mp_image *)data)->pts;
+}
+
+static void video_set_pts(void *data, double pts)
+{
+    ((struct mp_image *)data)->pts = pts;
+}
+
+static AVFrame *video_new_av_ref(void *data)
+{
+    return mp_image_to_av_frame(data);
+}
+
+static void *video_from_av_ref(AVFrame *data)
+{
+    return mp_image_from_av_frame(data);
+}
+
+static void *audio_ref(void *data)
+{
+    return mp_aframe_new_ref(data);
+}
+
+static double audio_get_pts(void *data)
+{
+    return mp_aframe_get_pts(data);
+}
+
+static void audio_set_pts(void *data, double pts)
+{
+    mp_aframe_set_pts(data, pts);
+}
+
+static AVFrame *audio_new_av_ref(void *data)
+{
+    return mp_aframe_to_avframe(data);
+}
+
+static void *audio_from_av_ref(AVFrame *data)
+{
+    return mp_aframe_from_avframe(data);
+}
+
+static const struct frame_handler frame_handlers[] = {
+    [MP_FRAME_NONE] = {
+        .name = "none",
+    },
+    [MP_FRAME_EOF] = {
+        .name = "eof",
+        .is_signaling = true,
+    },
+    [MP_FRAME_VIDEO] = {
+        .name = "video",
+        .is_data = true,
+        .new_ref = video_ref,
+        .get_pts = video_get_pts,
+        .set_pts = video_set_pts,
+        .new_av_ref = video_new_av_ref,
+        .from_av_ref = video_from_av_ref,
+        .free = talloc_free,
+    },
+    [MP_FRAME_AUDIO] = {
+        .name = "audio",
+        .is_data = true,
+        .new_ref = audio_ref,
+        .get_pts = audio_get_pts,
+        .set_pts = audio_set_pts,
+        .new_av_ref = audio_new_av_ref,
+        .from_av_ref = audio_from_av_ref,
+        .free = talloc_free,
+    },
+};
+
+const char *mp_frame_type_str(enum mp_frame_type t)
+{
+    return frame_handlers[t].name;
+}
+
+bool mp_frame_is_data(struct mp_frame frame)
+{
+    return frame_handlers[frame.type].is_data;
+}
+
+bool mp_frame_is_signaling(struct mp_frame frame)
+{
+    return frame_handlers[frame.type].is_signaling;
+}
+
+void mp_frame_unref(struct mp_frame *frame)
+{
+    if (!frame)
+        return;
+
+    if (frame_handlers[frame->type].free)
+        frame_handlers[frame->type].free(frame->data);
+
+    *frame = (struct mp_frame){0};
+}
+
+struct mp_frame mp_frame_ref(struct mp_frame frame)
+{
+    if (frame_handlers[frame.type].new_ref) {
+        assert(frame.data);
+        frame.data = frame_handlers[frame.type].new_ref(frame.data);
+        if (!frame.data)
+            frame.type = MP_FRAME_NONE;
+    }
+    return frame;
+}
+
+double mp_frame_get_pts(struct mp_frame frame)
+{
+    if (frame_handlers[frame.type].get_pts)
+        return frame_handlers[frame.type].get_pts(frame.data);
+    return MP_NOPTS_VALUE;
+}
+
+void mp_frame_set_pts(struct mp_frame frame, double pts)
+{
+    if (frame_handlers[frame.type].get_pts)
+        frame_handlers[frame.type].set_pts(frame.data, pts);
+}
+
+AVFrame *mp_frame_to_av(struct mp_frame frame, struct AVRational *tb)
+{
+    if (!frame_handlers[frame.type].new_av_ref)
+        return NULL;
+
+    AVFrame *res = frame_handlers[frame.type].new_av_ref(frame.data);
+    if (!res)
+        return NULL;
+
+    res->pts = mp_pts_to_av(mp_frame_get_pts(frame), tb);
+    return res;
+}
+
+struct mp_frame mp_frame_from_av(enum mp_frame_type type, struct AVFrame *frame,
+                                 struct AVRational *tb)
+{
+    struct mp_frame res = {type};
+
+    if (!frame_handlers[res.type].from_av_ref)
+        return MP_NO_FRAME;
+
+    res.data = frame_handlers[res.type].from_av_ref(frame);
+    if (!res.data)
+        return MP_NO_FRAME;
+
+    mp_frame_set_pts(res, mp_pts_from_av(frame->pts, tb));
+    return res;
+}
diff --git a/filters/frame.h b/filters/frame.h
new file mode 100644
index 0000000000..135920dea7
--- /dev/null
+++ b/filters/frame.h
@@ -0,0 +1,55 @@
+#pragma once
+
+#include <stdbool.h>
+
+enum mp_frame_type {
+    MP_FRAME_NONE = 0,  // NULL, placeholder, no frame available (_not_ EOF)
+    MP_FRAME_VIDEO,     // struct mp_image*
+    MP_FRAME_AUDIO,     // struct mp_aframe*
+    MP_FRAME_EOF,       // NULL, signals end of stream (but frames after it can
+                        // resume filtering!)
+};
+
+const char *mp_frame_type_str(enum mp_frame_type t);
+
+// Generic container for a piece of data, such as a video frame, or a collection
+// of audio samples. Wraps an actual media-specific frame data types in a
+// generic way. Also can be an empty frame for signaling (MP_FRAME_EOF and
+// possibly others).
+// This struct is usually allocated on the stack and can be copied by value.
+// You need to consider that the underlying pointer is ref-counted, and that
+// the _unref/_ref functions must be used accordingly.
+struct mp_frame {
+    enum mp_frame_type type;
+    void *data;
+};
+
+// Return whether the frame contains actual data (audio, video, ...). If false,
+// it's either signaling, or MP_FRAME_NONE.
+bool mp_frame_is_data(struct mp_frame frame);
+
+// Return whether the frame is for signaling (data flow commands like
+// MP_FRAME_EOF). If false, it's either data (mp_frame_is_data()), or
+// MP_FRAME_NONE.
+bool mp_frame_is_signaling(struct mp_frame frame);
+
+// Unreferences any frame data, and sets *frame to MP_FRAME_NONE. (It does
+// _not_ deallocate the memory block the parameter points to, only frame->data.)
+void mp_frame_unref(struct mp_frame *frame);
+
+// Return a new reference to the given frame. The caller owns the returned
+// frame. On failure returns a MP_FRAME_NONE.
+struct mp_frame mp_frame_ref(struct mp_frame frame);
+
+double mp_frame_get_pts(struct mp_frame frame);
+void mp_frame_set_pts(struct mp_frame frame, double pts);
+
+struct AVFrame;
+struct AVRational;
+struct AVFrame *mp_frame_to_av(struct mp_frame frame, struct AVRational *tb);
+struct mp_frame mp_frame_from_av(enum mp_frame_type type, struct AVFrame *frame,
+                                 struct AVRational *tb);
+
+#define MAKE_FRAME(type, frame) ((struct mp_frame){(type), (frame)})
+#define MP_NO_FRAME MAKE_FRAME(0, 0)
+#define MP_EOF_FRAME MAKE_FRAME(MP_FRAME_EOF, 0)
diff --git a/filters/user_filters.c b/filters/user_filters.c
new file mode 100644
index 0000000000..f59dd22d76
--- /dev/null
+++ b/filters/user_filters.c
@@ -0,0 +1,119 @@
+#include "config.h"
+
+#include "common/common.h"
+#include "common/msg.h"
+#include "options/m_config.h"
+
+#include "f_lavfi.h"
+#include "user_filters.h"
+
+static bool get_desc_from(const struct mp_user_filter_entry **list, int num,
+                          struct m_obj_desc *dst, int index)
+{
+    if (index >= num)
+        return false;
+    const struct mp_user_filter_entry *entry = list[index];
+    *dst = entry->desc;
+    dst->p = entry;
+    return true;
+}
+
+// --vf option
+
+const struct mp_user_filter_entry *vf_list[] = {
+    &vf_format,
+    &vf_lavfi,
+    &vf_lavfi_bridge,
+    &vf_sub,
+#if HAVE_VAPOURSYNTH_CORE && HAVE_VAPOURSYNTH
+    &vf_vapoursynth,
+#endif
+#if HAVE_VAPOURSYNTH_CORE && HAVE_VAPOURSYNTH_LAZY
+    &vf_vapoursynth_lazy,
+#endif
+#if HAVE_VDPAU
+    &vf_vdpaupp,
+#endif
+#if HAVE_VAAPI
+    &vf_vavpp,
+#endif
+#if HAVE_D3D_HWACCEL
+    &vf_d3d11vpp,
+#endif
+};
+
+static bool get_vf_desc(struct m_obj_desc *dst, int index)
+{
+    return get_desc_from(vf_list, MP_ARRAY_SIZE(vf_list), dst, index);
+}
+
+const struct m_obj_list vf_obj_list = {
+    .get_desc = get_vf_desc,
+    .description = "video filters",
+    .allow_disable_entries = true,
+    .allow_unknown_entries = true,
+};
+
+// Create a bidir, single-media filter from command line arguments.
+struct mp_filter *mp_create_user_filter(struct mp_filter *parent,
+                                        enum mp_output_chain_type type,
+                                        const char *name, char **args)
+{
+    const struct m_obj_list *obj_list = NULL;
+    const char *defs_name = NULL;
+    enum mp_frame_type frame_type = 0;
+    if (type == MP_OUTPUT_CHAIN_VIDEO) {
+        frame_type = MP_FRAME_VIDEO;
+        obj_list = &vf_obj_list;
+        defs_name = "vf-defaults";
+    }
+    assert(frame_type && obj_list);
+
+    struct mp_filter *f = NULL;
+
+    struct m_obj_desc desc;
+    if (!m_obj_list_find(&desc, obj_list, bstr0(name))) {
+        // Generic lavfi bridge.
+        if (strncmp(name, "lavfi-", 6) == 0)
+            name += 6;
+        struct mp_lavfi *l =
+            mp_lavfi_create_filter(parent, frame_type, true, NULL, name, args);
+        if (l)
+            f = l->f;
+        goto done;
+    }
+
+    void *options = NULL;
+    if (desc.options) {
+        struct m_obj_settings *defs = NULL;
+        if (defs_name) {
+            mp_read_option_raw(parent->global, defs_name,
+                                &m_option_type_obj_settings_list, &defs);
+        }
+
+        struct m_config *config =
+            m_config_from_obj_desc_and_args(NULL, parent->log, parent->global,
+                                            &desc, name, defs, args);
+
+        struct m_option dummy = {.type = &m_option_type_obj_settings_list};
+        m_option_free(&dummy, &defs);
+
+        if (!config)
+            goto done;
+
+        options = config->optstruct;
+        // Free config when options is freed.
+        ta_set_parent(options, NULL);
+        ta_set_parent(config, options);
+    }
+
+    const struct mp_user_filter_entry *entry = desc.p;
+    f = entry->create(parent, options);
+
+done:
+    if (!f) {
+        MP_ERR(parent, "Creating filter '%s' failed.\n", name);
+        return NULL;
+    }
+    return f;
+}
diff --git a/filters/user_filters.h b/filters/user_filters.h
new file mode 100644
index 0000000000..8e1e3e380d
--- /dev/null
+++ b/filters/user_filters.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include "options/m_option.h"
+
+#include "f_output_chain.h"
+
+// For creating filters from command line. Strictly for --vf/--af.
+struct mp_user_filter_entry {
+    // Name and sub-option description.
+    struct m_obj_desc desc;
+    // Create a filter. The option pointer is non-NULL if desc implies a priv
+    // struct to be allocated; then options are parsed into it. The callee
+    // must always free options (but can reparent it with talloc to keep it).
+    struct mp_filter *(*create)(struct mp_filter *parent, void *options);
+};
+
+struct mp_filter *mp_create_user_filter(struct mp_filter *parent,
+                                        enum mp_output_chain_type type,
+                                        const char *name, char **args);
+
+extern const struct mp_user_filter_entry vf_lavfi;
+extern const struct mp_user_filter_entry vf_lavfi_bridge;
+extern const struct mp_user_filter_entry vf_sub;
+extern const struct mp_user_filter_entry vf_vapoursynth;
+extern const struct mp_user_filter_entry vf_vapoursynth_lazy;
+extern const struct mp_user_filter_entry vf_format;
+extern const struct mp_user_filter_entry vf_vdpaupp;
+extern const struct mp_user_filter_entry vf_vavpp;
+extern const struct mp_user_filter_entry vf_d3d11vpp;