From 8edfaf9478d967536034ed26e614c0f410f80030 Mon Sep 17 00:00:00 2001 From: kieran Date: Wed, 6 Nov 2024 09:34:37 +0000 Subject: [PATCH] feat: setup filters (wip) --- Cargo.lock | 113 +++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 8 ++-- examples/main.rs | 38 +++++++++++---- src/decode.rs | 47 ++++++++++++------- src/filter.rs | 110 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 9 ++++ src/stream_info.rs | 2 +- 7 files changed, 296 insertions(+), 31 deletions(-) create mode 100644 src/filter.rs diff --git a/Cargo.lock b/Cargo.lock index 3e72d61..e6d9cfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,55 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.91" @@ -88,19 +137,50 @@ dependencies = [ "libloading", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "ffmpeg-rs-raw" version = "0.1.0" dependencies = [ "anyhow", + "env_logger", "ffmpeg-sys-the-third", "libc", + "log", "slimbox", ] @@ -123,6 +203,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -160,6 +252,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "memchr" version = "2.7.4" @@ -270,12 +368,27 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index e12bd76..2173f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,9 +3,7 @@ name = "ffmpeg-rs-raw" version = "0.1.0" edition = "2021" repository = "https://git.v0l.io/Kieran/ffmpeg-rs-raw.git" -authors = [ - "Kieran" -] +authors = ["Kieran"] [lib] crate-type = ["lib", "cdylib"] @@ -15,3 +13,7 @@ anyhow = "1.0.91" ffmpeg-sys-the-third = { git = "https://github.com/shssoichiro/ffmpeg-the-third.git", branch = "master", package = "ffmpeg-sys-the-third" } libc = { version = "0.2.160" } slimbox = "0.1.0" +log = "0.4.22" + +[dev-dependencies] +env_logger = "0.11.5" diff --git a/examples/main.rs b/examples/main.rs index 2d12060..a81b472 100644 --- a/examples/main.rs +++ b/examples/main.rs @@ -1,23 +1,26 @@ -use ffmpeg_rs_raw::{Decoder, Demuxer, DemuxerInfo}; -use ffmpeg_sys_the_third::AVHWDeviceType::{AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_VDPAU}; -use ffmpeg_sys_the_third::{av_frame_free, av_packet_free}; +use ffmpeg_rs_raw::{Decoder, Demuxer, DemuxerInfo, Filter, Scaler}; +use ffmpeg_sys_the_third::AVHWDeviceType::{ + AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_D3D12VA, AV_HWDEVICE_TYPE_MEDIACODEC, + AV_HWDEVICE_TYPE_OPENCL, AV_HWDEVICE_TYPE_VDPAU, AV_HWDEVICE_TYPE_VULKAN, +}; +use ffmpeg_sys_the_third::AVPixelFormat::AV_PIX_FMT_BGR24; +use ffmpeg_sys_the_third::{av_frame_free, av_packet_free, AVMediaType}; +use log::{error, info}; use std::env::args; use std::fs::File; use std::io::{Cursor, Read}; use std::path::PathBuf; fn main() { + env_logger::init(); let name = args().next().unwrap_or("main".to_string()); let path = if let Some(path) = args().skip(1).next() { PathBuf::from(path) } else { - eprintln!("Usage: {} ", name); + error!("Usage: {} ", name); std::process::exit(1); }; - let cd = read_as_custom_io(path.clone()); - scan_input(cd); - let fd = read_as_file(path.clone()); scan_input(fd); } @@ -36,33 +39,48 @@ fn read_as_file(path_buf: PathBuf) -> Demuxer { fn scan_input(mut demuxer: Demuxer) { unsafe { let info = demuxer.probe_input().expect("demuxer failed"); - println!("{}", info); + info!("{}", info); decode_input(demuxer, info); } } unsafe fn decode_input(demuxer: Demuxer, info: DemuxerInfo) { let mut decoder = Decoder::new(); - decoder.enable_hw_decoder(AV_HWDEVICE_TYPE_VDPAU); + //decoder.enable_hw_decoder_any(); decoder.enable_hw_decoder(AV_HWDEVICE_TYPE_CUDA); for ref stream in info.channels { decoder .setup_decoder(stream, None) .expect("decoder setup failed"); } - println!("{}", decoder); + info!("{}", decoder); loop_decoder(demuxer, decoder); } unsafe fn loop_decoder(mut demuxer: Demuxer, mut decoder: Decoder) { + let mut filter = + Filter::parse(&format!("scale_cuda=w={}:h={}", -2, 1080)).expect("filter add failed"); + loop { let (mut pkt, stream) = demuxer.get_packet().expect("demuxer failed"); if pkt.is_null() { break; // EOF } + let media_type = (*(*stream).codecpar).codec_type; + // only decode audio/video + if media_type != AVMediaType::AVMEDIA_TYPE_VIDEO + && media_type != AVMediaType::AVMEDIA_TYPE_AUDIO + { + av_packet_free(&mut pkt); + continue; + } if let Ok(frames) = decoder.decode_pkt(pkt, stream) { for (mut frame, _stream) in frames { // do nothing but decode entire stream + if media_type == AVMediaType::AVMEDIA_TYPE_VIDEO { + let mut new_frame = filter.process_frame(frame).expect("scale failed"); + av_frame_free(&mut new_frame); + } av_frame_free(&mut frame); } } diff --git a/src/decode.rs b/src/decode.rs index 367f31a..e3d2347 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -7,13 +7,14 @@ use std::ptr; use anyhow::Error; use ffmpeg_sys_the_third::AVPictureType::AV_PICTURE_TYPE_NONE; use ffmpeg_sys_the_third::{ - av_buffer_ref, av_frame_alloc, av_frame_copy_props, av_frame_free, av_hwdevice_ctx_create, - av_hwdevice_get_type_name, av_hwframe_transfer_data, avcodec_alloc_context3, - avcodec_find_decoder, avcodec_free_context, avcodec_get_hw_config, avcodec_get_name, - avcodec_open2, avcodec_parameters_to_context, avcodec_receive_frame, avcodec_send_packet, - AVCodec, AVCodecContext, AVCodecHWConfig, AVFrame, AVHWDeviceType, AVPacket, AVStream, AVERROR, - AVERROR_EOF, AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX, + av_buffer_ref, av_frame_alloc, av_hwdevice_ctx_create, av_hwdevice_get_type_name, + av_hwdevice_iterate_types, avcodec_alloc_context3, avcodec_find_decoder, avcodec_free_context, + avcodec_get_hw_config, avcodec_get_name, avcodec_open2, avcodec_parameters_to_context, + avcodec_receive_frame, avcodec_send_packet, AVCodec, AVCodecContext, AVCodecHWConfig, AVFrame, + AVHWDeviceType, AVPacket, AVStream, AVERROR, AVERROR_EOF, + AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX, }; +use log::debug; pub struct DecoderCodecContext { pub context: *mut AVCodecContext, @@ -100,6 +101,22 @@ impl Decoder { } } + /// Enable hardware decoding + pub fn enable_hw_decoder_any(&mut self) { + let mut res = HashSet::new(); + let mut hwt = AVHWDeviceType::AV_HWDEVICE_TYPE_NONE; + unsafe { + loop { + hwt = av_hwdevice_iterate_types(hwt); + if hwt == AVHWDeviceType::AV_HWDEVICE_TYPE_NONE { + break; + } + res.insert(hwt); + } + } + self.hw_decoder_types = Some(res); + } + /// Set up a decoder for a given channel pub fn setup_decoder( &mut self, @@ -142,6 +159,7 @@ impl Decoder { let mut ret = avcodec_parameters_to_context(context, (*stream).codecpar); return_ffmpeg_error!(ret, "Failed to copy codec parameters to context"); + let codec_name = CStr::from_ptr(avcodec_get_name((*codec).id)).to_str()?; // try use HW decoder let mut hw_config = ptr::null(); if let Some(ref hw_types) = self.hw_decoder_types { @@ -153,7 +171,11 @@ impl Decoder { if hw_config.is_null() { break; } + let hw_name = + CStr::from_ptr(av_hwdevice_get_type_name((*hw_config).device_type)) + .to_str()?; if !hw_types.contains(&(*hw_config).device_type) { + debug!("skipping hwaccel={}_{}", codec_name, hw_name); continue; } let hw_flag = AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX as libc::c_int; @@ -167,6 +189,7 @@ impl Decoder { ); return_ffmpeg_error!(ret, "Failed to create HW ctx"); (*context).hw_device_ctx = av_buffer_ref(hw_buf_ref); + debug!("using hwaccel={}_{}", codec_name, hw_name); break; } } @@ -180,6 +203,7 @@ impl Decoder { ret = avcodec_open2(context, codec, &mut dict); return_ffmpeg_error!(ret, "Failed to open codec"); + debug!("opened decoder={}", codec_name); Ok(e.insert(DecoderCodecContext { context, codec, @@ -219,17 +243,6 @@ impl Decoder { return Err(Error::msg(format!("Failed to decode {}", ret))); } - // copy frame from GPU - if !ctx.hw_config.is_null() { - let sw_frame = av_frame_alloc(); - ret = av_hwframe_transfer_data(sw_frame, frame, 0); - return_ffmpeg_error!(ret, "Failed to transfer data from GPU"); - - av_frame_copy_props(sw_frame, frame); - av_frame_free(&mut frame); - frame = sw_frame; - } - (*frame).pict_type = AV_PICTURE_TYPE_NONE; // encoder prints warnings pkgs.push((frame, stream)); } diff --git a/src/filter.rs b/src/filter.rs new file mode 100644 index 0000000..65096d2 --- /dev/null +++ b/src/filter.rs @@ -0,0 +1,110 @@ +use crate::{cstr, return_ffmpeg_error, set_opts}; +use anyhow::Error; +use ffmpeg_sys_the_third::{ + av_free, av_strdup, avfilter_get_by_name, avfilter_graph_alloc, avfilter_graph_alloc_filter, + avfilter_graph_config, avfilter_graph_create_filter, avfilter_graph_dump, avfilter_graph_parse, + avfilter_graph_parse2, avfilter_graph_parse_ptr, avfilter_inout_alloc, AVFilterContext, + AVFilterGraph, AVFrame, +}; +use log::debug; +use std::collections::HashMap; +use std::ffi::CStr; +use std::ptr; + +pub struct Filter { + graph: *mut AVFilterGraph, +} + +impl Filter { + pub fn new() -> Self { + Self { + graph: unsafe { avfilter_graph_alloc() }, + } + } + + /// Parse filter from string using [avfilter_graph_parse2] + /// + /// https://ffmpeg.org/ffmpeg-filters.html + pub unsafe fn parse(graph: &str) -> Result { + let mut ctx = avfilter_graph_alloc(); + let inputs = avfilter_inout_alloc(); + let outputs = avfilter_inout_alloc(); + let src = avfilter_get_by_name(cstr!("buffer")); + let dst = avfilter_get_by_name(cstr!("buffersink")); + let mut src_ctx = ptr::null_mut(); + let mut dst_ctx = ptr::null_mut(); + let ret = avfilter_graph_create_filter( + &mut src_ctx, + src, + cstr!("in"), + ptr::null_mut(), + ptr::null_mut(), + ctx, + ); + return_ffmpeg_error!(ret, "Failed to parse graph"); + + let ret = avfilter_graph_create_filter( + &mut dst_ctx, + dst, + cstr!("out"), + ptr::null_mut(), + ptr::null_mut(), + ctx, + ); + return_ffmpeg_error!(ret, "Failed to parse graph"); + + (*outputs).name = av_strdup((*dst).name); + (*outputs).filter_ctx = dst_ctx; + (*outputs).pad_idx = 0; + (*outputs).next = ptr::null_mut(); + + (*inputs).name = av_strdup((*src).name); + (*inputs).filter_ctx = src_ctx; + (*inputs).pad_idx = 0; + (*inputs).next = ptr::null_mut(); + + let ret = avfilter_graph_parse(ctx, cstr!(graph), inputs, outputs, ptr::null_mut()); + return_ffmpeg_error!(ret, "Failed to parse graph"); + let mut ret = Self { graph: ctx }; + ret.build()?; + Ok(ret) + } + + pub fn add_filter( + &mut self, + name: &str, + options: Option>, + ) -> Result<*mut AVFilterContext, Error> { + if self.graph.is_null() { + anyhow::bail!("Filter graph is null."); + } + unsafe { + let filter = avfilter_get_by_name(cstr!(name)); + if filter.is_null() { + anyhow::bail!("Filter {} not found", name); + } + let flt = avfilter_graph_alloc_filter(self.graph, filter, ptr::null_mut()); + if flt.is_null() { + anyhow::bail!("Filter {} not found", name); + } + if let Some(opt) = options { + set_opts(flt as *mut libc::c_void, opt)?; + } + Ok(flt) + } + } + + pub unsafe fn build(&mut self) -> Result<(), Error> { + let mut d = avfilter_graph_dump(self.graph, ptr::null_mut()); + debug!("{}", CStr::from_ptr(d).to_string_lossy()); + av_free(d as *mut _); + + let ret = avfilter_graph_config(self.graph, ptr::null_mut()); + return_ffmpeg_error!(ret, "Failed to build filter"); + Ok(()) + } + + pub unsafe fn process_frame(&mut self, frame: *mut AVFrame) -> Result<*mut AVFrame, Error> { + todo!(); + } +} diff --git a/src/lib.rs b/src/lib.rs index 3c55f15..0802428 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ use std::ptr; mod decode; mod demux; +mod filter; mod resample; mod scale; mod stream_info; @@ -27,6 +28,13 @@ macro_rules! return_ffmpeg_error { }; } +#[macro_export] +macro_rules! cstr { + ($str:expr) => { + format!("{}\0", $str).as_ptr() as *const libc::c_char + }; +} + fn get_ffmpeg_error_msg(ret: libc::c_int) -> String { unsafe { const BUF_SIZE: usize = 512; @@ -112,6 +120,7 @@ fn set_opts(ctx: *mut libc::c_void, options: HashMap) -> Result< pub use decode::*; pub use demux::*; pub use ffmpeg_sys_the_third; +pub use filter::*; pub use resample::*; pub use scale::*; pub use stream_info::*; diff --git a/src/stream_info.rs b/src/stream_info.rs index 6e09f00..632c5b2 100644 --- a/src/stream_info.rs +++ b/src/stream_info.rs @@ -68,7 +68,7 @@ impl Display for DemuxerInfo { write!( f, - "Demuxer Info: duration={}, bitrate={}k", + "Demuxer Info: duration={}, bitrate={}", format_time(self.duration), bitrate_str )?;