fix: improve RTP depacketizer robustness for H264 and H265 (#2032)

- Improved H264/H265 depacketizers to properly cache decoding parameter sets (SPS/PPS/VPS) when they are delivered via FU-A/FU fragmentation, which is uncommon but valid per RFC 6184/7798
- Fixed minor bugs in H264 and H265 depacketizers
  - Removed the redundant `start_payload` flag in H264. First fragment detection now relies solely on the FU-A S-bit (RFC 6184)
  - Fixed an incorrect DONL size check in H265 that silently dropped valid FU packets when `sprop-max-don-diff=0`

Author: getroot

src/projects/modules/rtp_rtcp/rtp_depacketizer_h264.cpp

@@ -15,8 +15,10 @@ std::shared_ptr<ov::Data> RtpDepacketizerH264::ParseAndAssembleFrame(std::vector
 		reserve_size += 16; // spare
 	}
 
+	// Reset FU-A DPS reassembly buffer at frame boundary
+	_fua_dps_buffer = nullptr;
+
 	auto bitstream = std::make_shared<ov::Data>(reserve_size);
-	bool start_payload = true;
 	for(const auto &payload : payload_list)
 	{
 		if (payload->GetLength() < NAL_HEADER_SIZE)
@@ -30,7 +32,7 @@ std::shared_ptr<ov::Data> RtpDepacketizerH264::ParseAndAssembleFrame(std::vector
 		// Fragmented NAL units
 		if(nal_type == NaluType::kFuA)
 		{
-			result = ParseFuaAndConvertAnnexB(payload, start_payload);
+			result = ParseFuaAndConvertAnnexB(payload);
 			if(result == nullptr)
 			{
 				return nullptr;
@@ -58,32 +60,61 @@ std::shared_ptr<ov::Data> RtpDepacketizerH264::ParseAndAssembleFrame(std::vector
 
 			bitstream->Append(result);
 		}
-
-		start_payload = false;
 	}
 
 	return bitstream;
 }
 
-std::shared_ptr<ov::Data> RtpDepacketizerH264::ParseFuaAndConvertAnnexB(const std::shared_ptr<ov::Data> &payload, bool start)
+std::shared_ptr<ov::Data> RtpDepacketizerH264::ParseFuaAndConvertAnnexB(const std::shared_ptr<ov::Data> &payload)
 {
 	auto bitstream = std::make_shared<ov::Data>(payload->GetLength() + 16);
 
-	if(payload->GetLength() < FUA_HEADER_SIZE)
+	if (payload->GetLength() < FUA_HEADER_SIZE)
 	{
 		// Invalid Data
 		return nullptr;
 	}
 
 	auto buffer = payload->GetDataAs<uint8_t>();
 	auto fnri = buffer[0] & (NAL_FBIT | NAL_NRI_MASK);
-	auto original_nal_type = buffer[1] & NAL_TYPE_MASK;
+	uint8_t original_nal_type = buffer[1] & NAL_TYPE_MASK;
 	bool first_fragment = (buffer[1] & FUA_SBIT) > 0;
+	bool last_fragment  = (buffer[1] & FUA_EBIT) > 0;
+	uint8_t original_nal_header = fnri | original_nal_type;
 
-	if(first_fragment == true || start == true)
+	// Cache SPS/PPS carried over FU-A (non-standard but possible encoders)
+	if(first_fragment)
+	{
+		_fua_dps_buffer = nullptr;
+
+		if (IsDecodingParmeterSets(original_nal_type))
+		{
+			_fua_dps_nal_type = original_nal_type;
+			_fua_dps_buffer = std::make_shared<ov::Data>();
+			_fua_dps_buffer->Append(&original_nal_header, NAL_HEADER_SIZE);
+			_fua_dps_buffer->Append(payload->Subdata(FUA_HEADER_SIZE));
+
+			// Single-fragment FU-A (S+E both set): flush immediately
+			if(last_fragment)
+			{
+				AddDecodingParameterSet(_fua_dps_nal_type, _fua_dps_buffer);
+				_fua_dps_buffer = nullptr;
+			}
+		}
+	}
+	else if (_fua_dps_buffer != nullptr)
+	{
+		_fua_dps_buffer->Append(payload->Subdata(FUA_HEADER_SIZE));
+		if (last_fragment)
+		{
+			AddDecodingParameterSet(_fua_dps_nal_type, _fua_dps_buffer);
+			_fua_dps_buffer = nullptr;
+		}
+	}
+
+	if (first_fragment == true)
 	{
 		uint8_t	start_prefix_and_nal_header[ANNEXB_START_PREFIX_LENGTH + NAL_HEADER_SIZE];
-		uint8_t original_nal_header = fnri | original_nal_type;
 
 		start_prefix_and_nal_header[0] = 0;
 		start_prefix_and_nal_header[1] = 0;
@@ -93,7 +124,7 @@ std::shared_ptr<ov::Data> RtpDepacketizerH264::ParseFuaAndConvertAnnexB(const st
 
 		bitstream->Append(start_prefix_and_nal_header, ANNEXB_START_PREFIX_LENGTH + NAL_HEADER_SIZE);
 	}
-	
+
 	bitstream->Append(payload->Subdata(FUA_HEADER_SIZE));
 
 	return bitstream;

src/projects/modules/rtp_rtcp/rtp_depacketizer_h264.h

@@ -24,10 +24,14 @@ class RtpDepacketizerH264 : public RtpDepacketizingManager
 	std::shared_ptr<ov::Data> GetDecodingParameterSetsToAnnexB() override;
 
 private:
-	std::shared_ptr<ov::Data> ParseFuaAndConvertAnnexB(const std::shared_ptr<ov::Data> &payload, bool start=false);
+	std::shared_ptr<ov::Data> ParseFuaAndConvertAnnexB(const std::shared_ptr<ov::Data> &payload);
 	std::shared_ptr<ov::Data> ParseStapAAndConvertToAnnexB(const std::shared_ptr<ov::Data> &payload);
 	std::shared_ptr<ov::Data> ConvertSingleNaluToAnnexB(const std::shared_ptr<ov::Data> &payload);
 
 	bool IsDecodingParmeterSets(uint8_t nal_unit_type);
 
+	// For FU-A SPS/PPS reassembly
+	std::shared_ptr<ov::Data> _fua_dps_buffer;
+	uint8_t _fua_dps_nal_type = 0;
+
 };

src/projects/modules/rtp_rtcp/rtp_depacketizer_h265.cpp

@@ -25,6 +25,9 @@ std::shared_ptr<ov::Data> RtpDepacketizerH265::ParseAndAssembleFrame(std::vector
 		reserve_size += 16;	 // spare
 	}
 
+	// Reset FU DPS reassembly buffer at frame boundary
+	_fu_dps_buffer = nullptr;
+
 	auto bitstream = std::make_shared<ov::Data>(reserve_size);
 	for (const auto &payload : payload_list)
 	{
@@ -149,7 +152,7 @@ std::shared_ptr<ov::Data> RtpDepacketizerH265::ParseFUsAndConvertAnnexB(const st
 	// FU header
 	uint8_t fu_hdr = buffer[offset];
 	uint8_t fu_s = FUH_SBIT(fu_hdr);
-	[[maybe_unused]] uint8_t fu_e = FUH_EBIT(fu_hdr);
+	uint8_t fu_e = FUH_EBIT(fu_hdr);
 	uint8_t fu_type = FUH_TYPE(fu_hdr);
 	offset += FU_HEADER_SIZE;
 
@@ -160,12 +163,10 @@ std::shared_ptr<ov::Data> RtpDepacketizerH265::ParseFUsAndConvertAnnexB(const st
 		return nullptr;
 	}
 
-	// If the S field is 1, the FU header must include the DONL(Conditional) field.
-	if (fu_s == 1 && payload->GetLength() < PAYLOAD_HEADER_SIZE + FU_HEADER_SIZE + DONL_FIELD_SIZE)
-	{
-		// Invalid Data
-		return nullptr;
-	}
+	// DONL is conditionally present only when S=1 and sprop-max-don-diff > 0 (SDP negotiated).
+	// sprop-max-don-diff > 0 is not used in WebRTC (browsers do not support it) and is
+	// virtually never seen in real RTSP sources either. OvenMediaEngine assumes
+	// sprop-max-don-diff=0 (the default), so DONL is treated as absent.
 
 	// If the S field is 1, the NAL header must be attached.
 	if (fu_s == 1)
@@ -179,8 +180,35 @@ std::shared_ptr<ov::Data> RtpDepacketizerH265::ParseFUsAndConvertAnnexB(const st
 		bitstream->Append(nal_header, PAYLOAD_HEADER_SIZE);
 	}
 
+	// Cache VPS/SPS/PPS carried over FU (non-standard but possible encoders)
+	const size_t fu_payload_length = payload->GetLength() - PAYLOAD_HEADER_SIZE - FU_HEADER_SIZE;
+	if (fu_s == 1)
+	{
+		_fu_dps_buffer = nullptr;
+
+		if (IsDecodingParmeterSets(fu_type))
+		{
+			_fu_dps_nal_type = fu_type;
+			_fu_dps_buffer = std::make_shared<ov::Data>();
+
+			uint8_t nal_header[2];
+			ByteWriter<uint16_t>::WriteBigEndian((uint8_t *)&nal_header[0], NUH_HEADER(fu_type, nuh_layer_id, nuh_temporal_id));
+			_fu_dps_buffer->Append(nal_header, PAYLOAD_HEADER_SIZE);
+			_fu_dps_buffer->Append(&buffer[offset], fu_payload_length);
+		}
+	}
+	else if (_fu_dps_buffer != nullptr)
+	{
+		_fu_dps_buffer->Append(&buffer[offset], fu_payload_length);
+		if (fu_e == 1)
+		{
+			AddDecodingParameterSet(_fu_dps_nal_type, _fu_dps_buffer);
+			_fu_dps_buffer = nullptr;
+		}
+	}
+
 	// Set FU Payload
-	bitstream->Append(&buffer[offset], payload->GetLength() - PAYLOAD_HEADER_SIZE - FU_HEADER_SIZE);
+	bitstream->Append(&buffer[offset], fu_payload_length);
 
 	return bitstream;
 }

src/projects/modules/rtp_rtcp/rtp_depacketizer_h265.h

@@ -44,4 +44,8 @@ class RtpDepacketizerH265 : public RtpDepacketizingManager
 	void AppendBitstream(std::shared_ptr<ov::Data>&bitstream, uint8_t nal_type, uint8_t nal_header[2], const void *payload, size_t payload_length);
 	bool IsDecodingParmeterSets(uint8_t nal_unit_type);
 
+	// For FU DPS reassembly (VPS/SPS/PPS carried over FU units)
+	std::shared_ptr<ov::Data> _fu_dps_buffer;
+	uint8_t _fu_dps_nal_type = 0;
+
 };