Skip to content

Commit

Permalink
mux mjpeg to mp4
Browse files Browse the repository at this point in the history 
Fixes #101
  • Loading branch information
@scottlamb
scottlamb committed Apr 22, 2024
1 parent 64e275b commit 623c0c1
Show file tree
Hide file tree
Showing 5 changed files with 217 additions and 131 deletions.
44 changes: 8 additions & 36 deletions examples/client/src/mp4.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,13 @@ impl<W: AsyncWrite + AsyncSeek + Send + Unpin> Mp4Writer<W> {
buf.put_u32(0); // version
buf.put_u32(u32::try_from(self.video_params.len())?); // entry_count
for p in &self.video_params {
self.write_video_sample_entry(buf, p)?;
let e = p.sample_entry().ok_or_else(|| {
anyhow!(
"unable to produce VisualSampleEntry for {} stream",
p.rfc6381_codec()
)
})?;
buf.extend_from_slice(e);
}
});
self.video_trak.write_common_stbl_parts(buf)?;
Expand Down Expand Up @@ -499,40 +505,6 @@ impl<W: AsyncWrite + AsyncSeek + Send + Unpin> Mp4Writer<W> {
Ok(())
}

fn write_video_sample_entry(
&self,
buf: &mut BytesMut,
parameters: &VideoParameters,
) -> Result<(), Error> {
// TODO: this should move to client::VideoParameters::sample_entry() or some such.
write_box!(buf, b"avc1", {
buf.put_u32(0);
buf.put_u32(1); // data_reference_index = 1
buf.extend_from_slice(&[0; 16]);
buf.put_u16(u16::try_from(parameters.pixel_dimensions().0)?);
buf.put_u16(u16::try_from(parameters.pixel_dimensions().1)?);
buf.extend_from_slice(&[
0x00, 0x48, 0x00, 0x00, // horizresolution
0x00, 0x48, 0x00, 0x00, // vertresolution
0x00, 0x00, 0x00, 0x00, // reserved
0x00, 0x01, // frame count
0x00, 0x00, 0x00, 0x00, // compressorname
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x00, 0x00, 0x00, //
0x00, 0x18, 0xff, 0xff, // depth + pre_defined
]);
write_box!(buf, b"avcC", {
buf.extend_from_slice(parameters.extra_data());
});
});
Ok(())
}

async fn video(
&mut self,
stream: &retina::client::Stream,
Expand Down Expand Up @@ -737,7 +709,7 @@ pub async fn run(opts: Opts) -> Result<(), Error> {
let video_stream_i = if !opts.no_video {
let s = session.streams().iter().position(|s| {
if s.media() == "video" {
if s.encoding_name() == "h264" {
if s.encoding_name() == "h264" || s.encoding_name() == "jpeg" {
log::info!("Using h264 video stream");
return true;
}
Expand Down
97 changes: 6 additions & 91 deletions src/codec/aac.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,91 +174,6 @@ impl AudioSpecificConfig {
}
}

/// Overwrites a buffer with a varint length, returning the length of the length.
/// See ISO/IEC 14496-1 section 8.3.3.
fn set_length(len: usize, data: &mut [u8]) -> Result<usize, String> {
if len < 1 << 7 {
data[0] = len as u8;
Ok(1)
} else if len < 1 << 14 {
data[0] = ((len & 0x7F) | 0x80) as u8;
data[1] = (len >> 7) as u8;
Ok(2)
} else if len < 1 << 21 {
data[0] = ((len & 0x7F) | 0x80) as u8;
data[1] = (((len >> 7) & 0x7F) | 0x80) as u8;
data[2] = (len >> 14) as u8;
Ok(3)
} else if len < 1 << 28 {
data[0] = ((len & 0x7F) | 0x80) as u8;
data[1] = (((len >> 7) & 0x7F) | 0x80) as u8;
data[2] = (((len >> 14) & 0x7F) | 0x80) as u8;
data[3] = (len >> 21) as u8;
Ok(4)
} else {
// BaseDescriptor sets a maximum length of 2**28 - 1.
Err(format!("length {len} too long"))
}
}

/// Writes a box length and type (four-character code) for everything appended
/// in the supplied scope.
macro_rules! write_box {
($buf:expr, $fourcc:expr, $b:block) => {
// The caller uses `&mut buf`. clippy likes to complain about the `&mut`
// being unnecessary for len(), but it is necessary for other things.
// The macro also can't store `$buf` in its own local, because `$b`
// is expected to reference `$buf` via the original name.
#[allow(clippy::unnecessary_mut_passed)]
{
let _: &mut Vec<u8> = $buf; // type-check.

let pos_start = $buf.len();
let fourcc: &[u8; 4] = $fourcc;
$buf.extend_from_slice(&[0, 0, 0, 0, fourcc[0], fourcc[1], fourcc[2], fourcc[3]]);
let r = {
$b;
};
let pos_end = $buf.len();
let len = pos_end.checked_sub(pos_start).unwrap();
$buf[pos_start..pos_start + 4].copy_from_slice(
&u32::try_from(len)
.map_err(|_| format!("box length {} exceeds u32::MAX", len))?
.to_be_bytes()[..],
);
r
}
};
}

/// Writes a descriptor tag and length for everything appended in the supplied
/// scope. See ISO/IEC 14496-1 Table 1 for the `tag`.
macro_rules! write_descriptor {
($buf:expr, $tag:expr, $b:block) => {{
let _: &mut Vec<u8> = $buf; // type-check.
let _: u8 = $tag;
let pos_start = $buf.len();

// Overallocate room for the varint length and append the body.
$buf.extend_from_slice(&[$tag, 0, 0, 0, 0]);
let r = {
$b;
};
let pos_end = $buf.len();

// Then fix it afterward: write the correct varint length and move
// the body backward. This approach seems better than requiring the
// caller to first prepare the body in a separate allocation (and
// awkward code ordering), or (as ffmpeg does) writing a "varint"
// which is padded with leading 0x80 bytes.
let len = pos_end.checked_sub(pos_start + 5).unwrap();
let len_len = set_length(len, &mut $buf[pos_start + 1..pos_start + 4])?;
$buf.copy_within(pos_start + 5..pos_end, pos_start + 1 + len_len);
$buf.truncate(pos_end + len_len - 4);
r
}};
}

/// Returns an MP4AudioSampleEntry (`mp4a`) box as in ISO/IEC 14496-14 section 5.6.1.
/// `config` should be a raw AudioSpecificConfig.
fn make_sample_entry(
Expand All @@ -271,7 +186,7 @@ fn make_sample_entry(
// Write an MP4AudioSampleEntry (`mp4a`), as in ISO/IEC 14496-14 section 5.6.1.
// It's based on AudioSampleEntry, ISO/IEC 14496-12 section 12.2.3.2,
// in turn based on SampleEntry, ISO/IEC 14496-12 section 8.5.2.2.
write_box!(&mut buf, b"mp4a", {
write_mp4_box!(&mut buf, b"mp4a", {
buf.extend_from_slice(&[
0, 0, 0, 0, // SampleEntry.reserved
0, 0, 0, 1, // SampleEntry.reserved, SampleEntry.data_reference_index (1)
Expand All @@ -294,10 +209,10 @@ fn make_sample_entry(
buf.put_u32(u32::from(sampling_frequency) << 16);

// Write the embedded ESDBox (`esds`), as in ISO/IEC 14496-14 section 5.6.1.
write_box!(&mut buf, b"esds", {
write_mp4_box!(&mut buf, b"esds", {
buf.put_u32(0); // version

write_descriptor!(&mut buf, 0x03 /* ES_DescrTag */, {
write_mpeg4_descriptor!(&mut buf, 0x03 /* ES_DescrTag */, {
// The ESDBox contains an ES_Descriptor, defined in ISO/IEC 14496-1 section 8.3.3.
// ISO/IEC 14496-14 section 3.1.2 has advice on how to set its
// fields within the scope of a .mp4 file.
Expand All @@ -307,7 +222,7 @@ fn make_sample_entry(
]);

// DecoderConfigDescriptor, defined in ISO/IEC 14496-1 section 7.2.6.6.
write_descriptor!(&mut buf, 0x04 /* DecoderConfigDescrTag */, {
write_mpeg4_descriptor!(&mut buf, 0x04 /* DecoderConfigDescrTag */, {
buf.extend_from_slice(&[
0x40, // objectTypeIndication = Audio ISO/IEC 14496-3
0x15, // streamType = audio, upstream = false, reserved = 1
Expand All @@ -333,13 +248,13 @@ fn make_sample_entry(
buf.put_u32(0);

// AudioSpecificConfiguration, ISO/IEC 14496-3 subpart 1 section 1.6.2.
write_descriptor!(&mut buf, 0x05 /* DecSpecificInfoTag */, {
write_mpeg4_descriptor!(&mut buf, 0x05 /* DecSpecificInfoTag */, {
buf.extend_from_slice(config);
});
});

// SLConfigDescriptor, ISO/IEC 14496-1 section 7.3.2.3.1.
write_descriptor!(&mut buf, 0x06 /* SLConfigDescrTag */, {
write_mpeg4_descriptor!(&mut buf, 0x06 /* SLConfigDescrTag */, {
buf.put_u8(2); // predefined = reserved for use in MP4 files
});
});
Expand Down
19 changes: 19 additions & 0 deletions src/codec/h264.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use h264_reader::nal::{NalHeader, UnitType};
use log::{debug, log_enabled, trace};

use crate::{
codec::write_visual_sample_entry_body,
rtp::{ReceivedPacket, ReceivedPacketBuilder},
Error, Timestamp,
};
Expand Down Expand Up @@ -615,6 +616,22 @@ struct InternalParameters {
pps_nal: Bytes,
}

/// Writes an `avc1` / `AVCSampleEntry` as in ISO/IEC 14496-15 section 5.4.2.1.
fn make_video_sample_entry(pixel_dimensions: (u32, u32), extra_data: &[u8]) -> Option<Vec<u8>> {
let pixel_dimensions = (
u16::try_from(pixel_dimensions.0).ok()?,
u16::try_from(pixel_dimensions.1).ok()?,
);
let mut buf = Vec::new();
write_mp4_box!(&mut buf, b"avc1", {
write_visual_sample_entry_body(&mut buf, pixel_dimensions);
write_mp4_box!(&mut buf, b"avcC", {
buf.extend_from_slice(extra_data);
});
});
Some(buf)
}

impl InternalParameters {
/// Parses metadata from the `format-specific-params` of a SDP `fmtp` media attribute.
fn parse_format_specific_params(format_specific_params: &str) -> Result<Self, String> {
Expand Down Expand Up @@ -742,13 +759,15 @@ impl InternalParameters {
let avc_decoder_config = avc_decoder_config.freeze();
let sps_nal = avc_decoder_config.slice(sps_nal_start..sps_nal_end);
let pps_nal = avc_decoder_config.slice(pps_nal_start..pps_nal_end);
let sample_entry = make_video_sample_entry(pixel_dimensions, &avc_decoder_config);
Ok(InternalParameters {
generic_parameters: super::VideoParameters {
rfc6381_codec,
pixel_dimensions,
pixel_aspect_ratio,
frame_rate,
extra_data: avc_decoder_config,
sample_entry,
},
sps_nal,
pps_nal,
Expand Down
67 changes: 64 additions & 3 deletions src/codec/jpeg.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,23 @@

//! [JPEG](https://www.itu.int/rec/T-REC-T.81-199209-I/en)-encoded video.
//! [RTP Payload Format for JPEG-compressed Video](https://datatracker.ietf.org/doc/html/rfc2435)
//!
//! # Representation in `.mp4` (ISO BMFF) container files
//!
//! ISO/IEC 14496-12 section C.5 ("Storage of new media types") says that media
//! codecs should be represented either via "MPEG-4 systems constructs" and
//! `mp4v` sample entries or via dedicated sample entry fourccs. Only the former
//! appears to be defined for MJPEG. Other software appears to support this
//! representation:
//!
//! * encoding in FFmpeg and MediaMTX
//! * decoding in FFmpeg, VLC, and Apple QuickTime Player
//!
//! Retina matches this.

use bytes::{Buf, Bytes};

use crate::{rtp::ReceivedPacket, PacketContext, Timestamp};
use crate::{codec::write_visual_sample_entry_body, rtp::ReceivedPacket, PacketContext, Timestamp};

use super::{VideoFrame, VideoParameters};

Expand Down Expand Up @@ -433,11 +446,12 @@ impl Depacketizer {
start_ctx: ctx,
timestamp,
parameters: Some(VideoParameters {
pixel_dimensions: (width as u32, height as u32),
rfc6381_codec: "".to_string(), // RFC 6381 is not applicable to MJPEG
pixel_dimensions: (u32::from(width), u32::from(height)),
rfc6381_codec: "mp4v.6C".to_owned(),
pixel_aspect_ratio: None,
frame_rate: None,
extra_data: Bytes::new(),
sample_entry: Some(make_video_sample_entry(width, height)),
}),
});
}
Expand Down Expand Up @@ -510,6 +524,53 @@ impl Depacketizer {
}
}

fn make_video_sample_entry(width: u16, height: u16) -> Vec<u8> {
let mut buf = Vec::new();

// Write an MP4VisualSampleEntry (`mp4v`), as in ISO/IEC 14496-14 section 5.6.1.
// It's based on VisualSampleEntry, ISO/IEC 14496-12 section 12.1.3.
// in turn based on SampleEntry, ISO/IEC 14496-12 section 8.5.2.2.
write_mp4_box!(&mut buf, b"mp4v", {
write_visual_sample_entry_body(&mut buf, (width, height));

// Write the embedded ESDBox (`esds`), as in ISO/IEC 14496-14 section 5.6.1.
write_mp4_box!(&mut buf, b"esds", {
buf.extend_from_slice(&0u32.to_be_bytes()[..]); // version
write_mpeg4_descriptor!(&mut buf, 0x03 /* ES_DescrTag */, {
// The ESDBox contains an ES_Descriptor, defined in ISO/IEC 14496-1 section 8.3.3.
// ISO/IEC 14496-14 section 3.1.2 has advice on how to set its
// fields within the scope of a .mp4 file.
buf.extend_from_slice(&[
0, 0, // ES_ID=0
0x00, // streamDependenceFlag, URL_Flag, OCRStreamFlag, streamPriority.
]);

// DecoderConfigDescriptor, defined in ISO/IEC 14496-1 section 7.2.6.6.
write_mpeg4_descriptor!(&mut buf, 0x04 /* DecoderConfigDescrTag */, {
buf.extend_from_slice(&[
0x6C, // objectTypeIndication = Visual ISO/IEC 10918-1 (aka JPEG)
0x11, // streamType = visual, upstream = false, reserved = 1
// XXX: does any reader expect valid values here? They wouldn't be
// trivial to calculate ahead of time.
0x00, 0x00, 0x00, // bufferSizeDB
0x00, 0x00, 0x00, 0x00, // maxBitrate
0x00, 0x00, 0x00, 0x00, // avgBitrate
]);
// No DecoderSpecificInfo.
// DecoderSpecificInfo, 2 of them?
// No profileLevelIndicatorIndexDescr.
});

// SLConfigDescriptor, ISO/IEC 14496-1 section 7.3.2.3.1.
write_mpeg4_descriptor!(&mut buf, 0x06 /* SLConfigDescrTag */, {
buf.push(2); // predefined = reserved for use in MP4 files
});
});
});
});
buf
}

impl Default for Depacketizer {
fn default() -> Self {
Self::new()
Expand Down
Loading

0 comments on commit 623c0c1

Please sign in to comment.