feat: setup filters (wip)

Cargo.lock

@@ -11,6 +11,55 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.6.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125"
+dependencies = [
+ "anstyle",
+ "windows-sys",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.91"
@@ -88,19 +137,50 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "colorchoice"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
+
 [[package]]
 name = "either"
 version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
+[[package]]
+name = "env_filter"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
+dependencies = [
+ "log",
+ "regex",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.11.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "env_filter",
+ "humantime",
+ "log",
+]
+
 [[package]]
 name = "ffmpeg-rs-raw"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "env_logger",
  "ffmpeg-sys-the-third",
  "libc",
+ "log",
  "slimbox",
 ]
 
@@ -123,6 +203,18 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
 [[package]]
 name = "itertools"
 version = "0.12.1"
@@ -160,6 +252,12 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "log"
+version = "0.4.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
+
 [[package]]
 name = "memchr"
 version = "2.7.4"
@@ -270,12 +368,27 @@ version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets",
+]
+
 [[package]]
 name = "windows-targets"
 version = "0.52.6"

Cargo.toml

@@ -3,9 +3,7 @@ name = "ffmpeg-rs-raw"
 version = "0.1.0"
 edition = "2021"
 repository = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git"
-authors = [
+authors = ["Kieran"]
-    "Kieran"
-]
 
 [lib]
 crate-type = ["lib", "cdylib"]
@@ -15,3 +13,7 @@ anyhow = "1.0.91"
 ffmpeg-sys-the-third = { git = "https://github.com/shssoichiro/ffmpeg-the-third.git", branch = "master", package = "ffmpeg-sys-the-third" }
 libc = { version = "0.2.160" }
 slimbox = "0.1.0"
+log = "0.4.22"
+
+[dev-dependencies]
+env_logger = "0.11.5"

examples/main.rs

@@ -1,23 +1,26 @@
-use ffmpeg_rs_raw::{Decoder, Demuxer, DemuxerInfo};
+use ffmpeg_rs_raw::{Decoder, Demuxer, DemuxerInfo, Filter, Scaler};
-use ffmpeg_sys_the_third::AVHWDeviceType::{AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_VDPAU};
+use ffmpeg_sys_the_third::AVHWDeviceType::{
-use ffmpeg_sys_the_third::{av_frame_free, av_packet_free};
+    AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_D3D12VA, AV_HWDEVICE_TYPE_MEDIACODEC,
+    AV_HWDEVICE_TYPE_OPENCL, AV_HWDEVICE_TYPE_VDPAU, AV_HWDEVICE_TYPE_VULKAN,
+};
+use ffmpeg_sys_the_third::AVPixelFormat::AV_PIX_FMT_BGR24;
+use ffmpeg_sys_the_third::{av_frame_free, av_packet_free, AVMediaType};
+use log::{error, info};
 use std::env::args;
 use std::fs::File;
 use std::io::{Cursor, Read};
 use std::path::PathBuf;
 
 fn main() {
+    env_logger::init();
     let name = args().next().unwrap_or("main".to_string());
     let path = if let Some(path) = args().skip(1).next() {
         PathBuf::from(path)
     } else {
-        eprintln!("Usage: {} <path>", name);
+        error!("Usage: {} <path>", name);
         std::process::exit(1);
     };
 
-    let cd = read_as_custom_io(path.clone());
-    scan_input(cd);
-
     let fd = read_as_file(path.clone());
     scan_input(fd);
 }
@@ -36,33 +39,48 @@ fn read_as_file(path_buf: PathBuf) -> Demuxer {
 fn scan_input(mut demuxer: Demuxer) {
     unsafe {
         let info = demuxer.probe_input().expect("demuxer failed");
-        println!("{}", info);
+        info!("{}", info);
         decode_input(demuxer, info);
     }
 }
 
 unsafe fn decode_input(demuxer: Demuxer, info: DemuxerInfo) {
     let mut decoder = Decoder::new();
-    decoder.enable_hw_decoder(AV_HWDEVICE_TYPE_VDPAU);
+    //decoder.enable_hw_decoder_any();
     decoder.enable_hw_decoder(AV_HWDEVICE_TYPE_CUDA);
     for ref stream in info.channels {
         decoder
             .setup_decoder(stream, None)
             .expect("decoder setup failed");
     }
-    println!("{}", decoder);
+    info!("{}", decoder);
     loop_decoder(demuxer, decoder);
 }
 
 unsafe fn loop_decoder(mut demuxer: Demuxer, mut decoder: Decoder) {
+    let mut filter =
+        Filter::parse(&format!("scale_cuda=w={}:h={}", -2, 1080)).expect("filter add failed");
+
     loop {
         let (mut pkt, stream) = demuxer.get_packet().expect("demuxer failed");
         if pkt.is_null() {
             break; // EOF
         }
+        let media_type = (*(*stream).codecpar).codec_type;
+        // only decode audio/video
+        if media_type != AVMediaType::AVMEDIA_TYPE_VIDEO
+            && media_type != AVMediaType::AVMEDIA_TYPE_AUDIO
+        {
+            av_packet_free(&mut pkt);
+            continue;
+        }
         if let Ok(frames) = decoder.decode_pkt(pkt, stream) {
             for (mut frame, _stream) in frames {
                 // do nothing but decode entire stream
+                if media_type == AVMediaType::AVMEDIA_TYPE_VIDEO {
+                    let mut new_frame = filter.process_frame(frame).expect("scale failed");
+                    av_frame_free(&mut new_frame);
+                }
                 av_frame_free(&mut frame);
             }
         }

src/decode.rs

@@ -7,13 +7,14 @@ use std::ptr;
 use anyhow::Error;
 use ffmpeg_sys_the_third::AVPictureType::AV_PICTURE_TYPE_NONE;
 use ffmpeg_sys_the_third::{
-    av_buffer_ref, av_frame_alloc, av_frame_copy_props, av_frame_free, av_hwdevice_ctx_create,
+    av_buffer_ref, av_frame_alloc, av_hwdevice_ctx_create, av_hwdevice_get_type_name,
-    av_hwdevice_get_type_name, av_hwframe_transfer_data, avcodec_alloc_context3,
+    av_hwdevice_iterate_types, avcodec_alloc_context3, avcodec_find_decoder, avcodec_free_context,
-    avcodec_find_decoder, avcodec_free_context, avcodec_get_hw_config, avcodec_get_name,
+    avcodec_get_hw_config, avcodec_get_name, avcodec_open2, avcodec_parameters_to_context,
-    avcodec_open2, avcodec_parameters_to_context, avcodec_receive_frame, avcodec_send_packet,
+    avcodec_receive_frame, avcodec_send_packet, AVCodec, AVCodecContext, AVCodecHWConfig, AVFrame,
-    AVCodec, AVCodecContext, AVCodecHWConfig, AVFrame, AVHWDeviceType, AVPacket, AVStream, AVERROR,
+    AVHWDeviceType, AVPacket, AVStream, AVERROR, AVERROR_EOF,
-    AVERROR_EOF, AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX,
+    AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX,
 };
+use log::debug;
 
 pub struct DecoderCodecContext {
     pub context: *mut AVCodecContext,
@@ -100,6 +101,22 @@ impl Decoder {
         }
     }
 
+    /// Enable hardware decoding
+    pub fn enable_hw_decoder_any(&mut self) {
+        let mut res = HashSet::new();
+        let mut hwt = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE;
+        unsafe {
+            loop {
+                hwt = av_hwdevice_iterate_types(hwt);
+                if hwt == AVHWDeviceType::AV_HWDEVICE_TYPE_NONE {
+                    break;
+                }
+                res.insert(hwt);
+            }
+        }
+        self.hw_decoder_types = Some(res);
+    }
+
     /// Set up a decoder for a given channel
     pub fn setup_decoder(
         &mut self,
@@ -142,6 +159,7 @@ impl Decoder {
             let mut ret = avcodec_parameters_to_context(context, (*stream).codecpar);
             return_ffmpeg_error!(ret, "Failed to copy codec parameters to context");
 
+            let codec_name = CStr::from_ptr(avcodec_get_name((*codec).id)).to_str()?;
             // try use HW decoder
             let mut hw_config = ptr::null();
             if let Some(ref hw_types) = self.hw_decoder_types {
@@ -153,7 +171,11 @@ impl Decoder {
                     if hw_config.is_null() {
                         break;
                     }
+                    let hw_name =
+                        CStr::from_ptr(av_hwdevice_get_type_name((*hw_config).device_type))
+                            .to_str()?;
                     if !hw_types.contains(&(*hw_config).device_type) {
+                        debug!("skipping hwaccel={}_{}", codec_name, hw_name);
                         continue;
                     }
                     let hw_flag = AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX as libc::c_int;
@@ -167,6 +189,7 @@ impl Decoder {
                         );
                         return_ffmpeg_error!(ret, "Failed to create HW ctx");
                         (*context).hw_device_ctx = av_buffer_ref(hw_buf_ref);
+                        debug!("using hwaccel={}_{}", codec_name, hw_name);
                         break;
                     }
                 }
@@ -180,6 +203,7 @@ impl Decoder {
             ret = avcodec_open2(context, codec, &mut dict);
             return_ffmpeg_error!(ret, "Failed to open codec");
 
+            debug!("opened decoder={}", codec_name);
             Ok(e.insert(DecoderCodecContext {
                 context,
                 codec,
@@ -219,17 +243,6 @@ impl Decoder {
                     return Err(Error::msg(format!("Failed to decode {}", ret)));
                 }
 
-                // copy frame from GPU
-                if !ctx.hw_config.is_null() {
-                    let sw_frame = av_frame_alloc();
-                    ret = av_hwframe_transfer_data(sw_frame, frame, 0);
-                    return_ffmpeg_error!(ret, "Failed to transfer data from GPU");
-
-                    av_frame_copy_props(sw_frame, frame);
-                    av_frame_free(&mut frame);
-                    frame = sw_frame;
-                }
-
                 (*frame).pict_type = AV_PICTURE_TYPE_NONE; // encoder prints warnings
                 pkgs.push((frame, stream));
             }

src/filter.rs

@@ -0,0 +1,110 @@
+use crate::{cstr, return_ffmpeg_error, set_opts};
+use anyhow::Error;
+use ffmpeg_sys_the_third::{
+    av_free, av_strdup, avfilter_get_by_name, avfilter_graph_alloc, avfilter_graph_alloc_filter,
+    avfilter_graph_config, avfilter_graph_create_filter, avfilter_graph_dump, avfilter_graph_parse,
+    avfilter_graph_parse2, avfilter_graph_parse_ptr, avfilter_inout_alloc, AVFilterContext,
+    AVFilterGraph, AVFrame,
+};
+use log::debug;
+use std::collections::HashMap;
+use std::ffi::CStr;
+use std::ptr;
+
+pub struct Filter {
+    graph: *mut AVFilterGraph,
+}
+
+impl Filter {
+    pub fn new() -> Self {
+        Self {
+            graph: unsafe { avfilter_graph_alloc() },
+        }
+    }
+
+    /// Parse filter from string using [avfilter_graph_parse2]
+    ///
+    /// https://ffmpeg.org/ffmpeg-filters.html
+    pub unsafe fn parse(graph: &str) -> Result<Self, Error> {
+        let mut ctx = avfilter_graph_alloc();
+        let inputs = avfilter_inout_alloc();
+        let outputs = avfilter_inout_alloc();
+        let src = avfilter_get_by_name(cstr!("buffer"));
+        let dst = avfilter_get_by_name(cstr!("buffersink"));
+        let mut src_ctx = ptr::null_mut();
+        let mut dst_ctx = ptr::null_mut();
+        let ret = avfilter_graph_create_filter(
+            &mut src_ctx,
+            src,
+            cstr!("in"),
+            ptr::null_mut(),
+            ptr::null_mut(),
+            ctx,
+        );
+        return_ffmpeg_error!(ret, "Failed to parse graph");
+
+        let ret = avfilter_graph_create_filter(
+            &mut dst_ctx,
+            dst,
+            cstr!("out"),
+            ptr::null_mut(),
+            ptr::null_mut(),
+            ctx,
+        );
+        return_ffmpeg_error!(ret, "Failed to parse graph");
+
+        (*outputs).name = av_strdup((*dst).name);
+        (*outputs).filter_ctx = dst_ctx;
+        (*outputs).pad_idx = 0;
+        (*outputs).next = ptr::null_mut();
+
+        (*inputs).name = av_strdup((*src).name);
+        (*inputs).filter_ctx = src_ctx;
+        (*inputs).pad_idx = 0;
+        (*inputs).next = ptr::null_mut();
+
+        let ret = avfilter_graph_parse(ctx, cstr!(graph), inputs, outputs, ptr::null_mut());
+        return_ffmpeg_error!(ret, "Failed to parse graph");
+        let mut ret = Self { graph: ctx };
+        ret.build()?;
+        Ok(ret)
+    }
+
+    pub fn add_filter(
+        &mut self,
+        name: &str,
+        options: Option<HashMap<String, String>>,
+    ) -> Result<*mut AVFilterContext, Error> {
+        if self.graph.is_null() {
+            anyhow::bail!("Filter graph is null.");
+        }
+        unsafe {
+            let filter = avfilter_get_by_name(cstr!(name));
+            if filter.is_null() {
+                anyhow::bail!("Filter {} not found", name);
+            }
+            let flt = avfilter_graph_alloc_filter(self.graph, filter, ptr::null_mut());
+            if flt.is_null() {
+                anyhow::bail!("Filter {} not found", name);
+            }
+            if let Some(opt) = options {
+                set_opts(flt as *mut libc::c_void, opt)?;
+            }
+            Ok(flt)
+        }
+    }
+
+    pub unsafe fn build(&mut self) -> Result<(), Error> {
+        let mut d = avfilter_graph_dump(self.graph, ptr::null_mut());
+        debug!("{}", CStr::from_ptr(d).to_string_lossy());
+        av_free(d as *mut _);
+
+        let ret = avfilter_graph_config(self.graph, ptr::null_mut());
+        return_ffmpeg_error!(ret, "Failed to build filter");
+        Ok(())
+    }
+
+    pub unsafe fn process_frame(&mut self, frame: *mut AVFrame) -> Result<*mut AVFrame, Error> {
+        todo!();
+    }
+}

src/lib.rs

@@ -9,6 +9,7 @@ use std::ptr;
 
 mod decode;
 mod demux;
+mod filter;
 mod resample;
 mod scale;
 mod stream_info;
@@ -27,6 +28,13 @@ macro_rules! return_ffmpeg_error {
     };
 }
 
+#[macro_export]
+macro_rules! cstr {
+    ($str:expr) => {
+        format!("{}\0", $str).as_ptr() as *const libc::c_char
+    };
+}
+
 fn get_ffmpeg_error_msg(ret: libc::c_int) -> String {
     unsafe {
         const BUF_SIZE: usize = 512;
@@ -112,6 +120,7 @@ fn set_opts(ctx: *mut libc::c_void, options: HashMap<String, String>) -> Result<
 pub use decode::*;
 pub use demux::*;
 pub use ffmpeg_sys_the_third;
+pub use filter::*;
 pub use resample::*;
 pub use scale::*;
 pub use stream_info::*;

src/stream_info.rs

@@ -68,7 +68,7 @@ impl Display for DemuxerInfo {
 
         write!(
             f,
-            "Demuxer Info: duration={}, bitrate={}k",
+            "Demuxer Info: duration={}, bitrate={}",
             format_time(self.duration),
             bitrate_str
         )?;