diff --git a/libavformat/whip.c b/libavformat/whip.c index 710f24fc5ab97..ab1b7be7d0d8f 100644 --- a/libavformat/whip.c +++ b/libavformat/whip.c @@ -42,7 +42,7 @@ #include "network.h" #include "srtp.h" #include "tls.h" - +#include /** * Maximum size limit of a Session Description Protocol (SDP), * be it an offer or answer. @@ -114,6 +114,7 @@ /* Referring to Chrome's definition of RTP payload types. */ #define WHIP_RTP_PAYLOAD_TYPE_H264 106 #define WHIP_RTP_PAYLOAD_TYPE_OPUS 111 +#define WHIP_RTP_PAYLOAD_TYPE_RTX 105 /** * The STUN message header, which is 20 bytes long, comprises the @@ -143,6 +144,12 @@ #define WHIP_RTCP_PT_START 192 #define WHIP_RTCP_PT_END 223 +/** + * Consent-freshness constants ����������������������������������������� + */ +#define WHIP_CONSENT_DEF_INTERVAL 15000 /* ms � RFC 7675 default */ +#define WHIP_CONSENT_MAX_FAILURES 3 + /** * In the case of ICE-LITE, these fields are not used; instead, they are defined * as constant values. @@ -150,15 +157,22 @@ #define WHIP_SDP_SESSION_ID "4489045141692799359" #define WHIP_SDP_CREATOR_IP "127.0.0.1" +/** + * Retransmission / NACK support +*/ +#define HISTORY_SIZE_DEFAULT 512 + /* Calculate the elapsed time from starttime to endtime in milliseconds. */ #define ELAPSED(starttime, endtime) ((int)(endtime - starttime) / 1000) /* STUN Attribute, comprehension-required range (0x0000-0x7FFF) */ enum STUNAttr { STUN_ATTR_USERNAME = 0x0006, /// shared secret response/bind request + STUN_ATTR_PRIORITY = 0x0024, /// ICE controlling/controlled STUN_ATTR_USE_CANDIDATE = 0x0025, /// bind request STUN_ATTR_MESSAGE_INTEGRITY = 0x0008, /// bind request/response STUN_ATTR_FINGERPRINT = 0x8028, /// rfc5389 + STUN_ATTR_ICE_CONTROLLING = 0x802A, /// full agent talking to ice-lite }; enum WHIPState { @@ -193,6 +207,15 @@ enum WHIPState { WHIP_STATE_FAILED, }; +typedef struct RtpHistoryItem { + /* original RTP seq */ + uint16_t seq; + /* length in bytes */ + int size; + /* malloc-ed copy */ + uint8_t* pkt; +} RtpHistoryItem; + typedef struct WHIPContext { AVClass *av_class; @@ -303,6 +326,25 @@ typedef struct WHIPContext { /* The certificate and private key used for DTLS handshake. */ char* cert_file; char* key_file; + + /* Consent-freshness state */ + int consent_interval; + int64_t last_consent_tx; + int consent_failures; + + /* ICE-lite support */ + int ice_lite_remote; + uint64_t ice_tie_breaker; /* random 64-bit, for ICE-CONTROLLING */ + + /* RTX / NACK */ + uint8_t rtx_payload_type; + uint32_t video_rtx_ssrc; + uint16_t rtx_seq; + int history_size; + RtpHistoryItem * history; /* ring buffer */ + int hist_head; + int enable_nack_rtx; + } WHIPContext; /** @@ -412,6 +454,15 @@ static av_cold int initialize(AVFormatContext *s) seed = av_get_random_seed(); av_lfg_init(&whip->rnd, seed); + /* 64-bit tie-breaker for ICE-CONTROLLING (RFC 8445 6.1.1) */ + whip->ice_tie_breaker = ((uint64_t)av_lfg_get(&whip->rnd) << 32) | (uint64_t)av_lfg_get(&whip->rnd); + + /* Initialise consent-freshness timers */ + if (whip->consent_interval <= 0) + whip->consent_interval = WHIP_CONSENT_DEF_INTERVAL; + whip->last_consent_tx = av_gettime(); + whip->consent_failures = 0; + if (whip->pkt_size < ideal_pkt_size) av_log(whip, AV_LOG_WARNING, "WHIP: pkt_size=%d(<%d) is too small, may cause packet loss\n", whip->pkt_size, ideal_pkt_size); @@ -615,6 +666,17 @@ static int generate_sdp_offer(AVFormatContext *s) whip->audio_payload_type = WHIP_RTP_PAYLOAD_TYPE_OPUS; whip->video_payload_type = WHIP_RTP_PAYLOAD_TYPE_H264; + /* RTX / NACK init */ + whip->rtx_payload_type = WHIP_RTP_PAYLOAD_TYPE_RTX; + whip->video_rtx_ssrc = av_lfg_get(&whip->rnd); + whip->rtx_seq = 0; + whip->hist_head = 0; + whip->history_size = FFMAX(64, whip->history_size); + whip->history = av_calloc(whip->history_size, sizeof(*whip->history)); + if (!whip->history) + return AVERROR(ENOMEM); + whip->enable_nack_rtx = 1; + av_bprintf(&bp, "" "v=0\r\n" "o=FFmpeg %s 2 IN IP4 %s\r\n" @@ -679,8 +741,14 @@ static int generate_sdp_offer(AVFormatContext *s) "a=rtcp-rsize\r\n" "a=rtpmap:%u %s/90000\r\n" "a=fmtp:%u level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=%02x%02x%02x\r\n" + "a=rtcp-fb:%u nack\r\n" + "a=rtpmap:%u rtx/90000\r\n" + "a=fmtp:%u apt=%u\r\n" + "a=ssrc:%u cname:FFmpeg\r\n" + "a=ssrc:%u msid:FFmpeg video\r\n" "a=ssrc:%u cname:FFmpeg\r\n" - "a=ssrc:%u msid:FFmpeg video\r\n", + "a=ssrc:%u msid:FFmpeg video\r\n" + "a=ssrc-group:FID %u %u\r\n", whip->video_payload_type, whip->ice_ufrag_local, whip->ice_pwd_local, @@ -691,8 +759,16 @@ static int generate_sdp_offer(AVFormatContext *s) profile, profile_iop, level, + whip->video_payload_type, + whip->rtx_payload_type, + whip->rtx_payload_type, + whip->video_payload_type, + whip->video_ssrc, whip->video_ssrc, - whip->video_ssrc); + whip->video_rtx_ssrc, + whip->video_rtx_ssrc, + whip->video_ssrc, + whip->video_rtx_ssrc); } if (!av_bprint_is_complete(&bp)) { @@ -894,6 +970,8 @@ static int parse_answer(AVFormatContext *s) goto end; } } + } else if (av_strstart(line, "a=ice-lite", NULL)) { + whip->ice_lite_remote = 1; } } @@ -985,6 +1063,22 @@ static int ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, in avio_wb16(pb, STUN_ATTR_USE_CANDIDATE); /* attribute type use-candidate */ avio_wb16(pb, 0); /* size of use-candidate */ + /** + * For ICE-lite peers we are *always* the controlling agent (RFC 8445 6.1.3.1). + * Add PRIORITY + ICE-CONTROLLING attributes. + */ + if (whip->ice_lite_remote) { + /* we are controlling, use host-candidate priority 126 << 24 | 65535 << 8 | 255 = 2130706431 */ + avio_wb16(pb, STUN_ATTR_PRIORITY); + avio_wb16(pb, 4); + avio_wb32(pb, 2130706431); + + avio_wb16(pb, STUN_ATTR_ICE_CONTROLLING); + avio_wb16(pb, 8); + avio_wb32(pb, (uint32_t)(whip->ice_tie_breaker >> 32)); + avio_wb32(pb, (uint32_t)(whip->ice_tie_breaker & 0xffffffff)); + } + /* Build and update message integrity */ avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */ avio_wb16(pb, 20); /* size of message integrity */ @@ -1421,6 +1515,37 @@ static int setup_srtp(AVFormatContext *s) return ret; } + +/** + * RTX history helpers + */ + static void rtp_history_store(WHIPContext *whip, const uint8_t *pkt, int size) +{ + int pos = whip->hist_head % whip->history_size; + RtpHistoryItem * it = &whip->history[pos]; + /* free older entry */ + av_free(it->pkt); + it->pkt = av_malloc(size); + if (!it->pkt) + return; + + memcpy(it->pkt, pkt, size); + it->size = size; + it->seq = AV_RB16(pkt + 2); + + whip->hist_head++; +} + +static const RtpHistoryItem* rtp_history_find(const WHIPContext *whip, uint16_t seq) +{ + for (int i = 0; i < whip->history_size; i++) { + const RtpHistoryItem * it = &whip->history[i]; + if (it->pkt && it->seq == seq) + return it; + } + return NULL; +} + /** * Callback triggered by the RTP muxer when it creates and sends out an RTP packet. * @@ -1457,6 +1582,10 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size) return 0; } + /* Store only ORIGINAL video packets (non-RTX, non-RTCP) */ + if (!is_rtcp && is_video) + rtp_history_store(whip, buf, buf_size); + ret = ffurl_write(whip->udp, whip->buf, cipher_size); if (ret < 0) { av_log(whip, AV_LOG_ERROR, "WHIP: Failed to write packet=%dB, ret=%d\n", cipher_size, ret); @@ -1465,6 +1594,45 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size) return ret; } +/** + * Build and send a single RTX packet +*/ +static int send_rtx_packet(AVFormatContext *s, const uint8_t * orig_pkt, int orig_size) +{ + WHIPContext * whip = s->priv_data; + int new_size, cipher_size; + /* skip if no RTX PT configured */ + if (!whip->enable_nack_rtx) + return 0; + + /* allocate new buffer: header + 2 + payload */ + if (orig_size + 2 > sizeof(whip->buf)) + return 0; + + memcpy(whip->buf, orig_pkt, orig_size); + + uint8_t * hdr = whip->buf; + uint16_t orig_seq = AV_RB16(hdr + 2); + + /* rewrite header */ + hdr[1] = (hdr[1] & 0x80) | whip->rtx_payload_type; /* keep M bit */ + AV_WB16(hdr + 2, whip->rtx_seq++); + AV_WB32(hdr + 8, whip->video_rtx_ssrc); + + /* shift payload 2 bytes */ + memmove(hdr + 12 + 2, hdr + 12, orig_size - 12); + AV_WB16(hdr + 12, orig_seq); + + new_size = orig_size + 2; + + /* Encrypt by SRTP and send out. */ + cipher_size = ff_srtp_encrypt(&whip->srtp_video_send, whip->buf, new_size, whip->buf, sizeof(whip->buf)); + if (cipher_size <= 0 || cipher_size < new_size) { + av_log(whip, AV_LOG_WARNING, "WHIP: Failed to encrypt packet=%dB, cipher=%dB\n", new_size, cipher_size); + return 0; + } + return ffurl_write(whip->udp, whip->buf, cipher_size); +} /** * Creates dedicated RTP muxers for each stream in the AVFormatContext to build RTP @@ -1775,7 +1943,16 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt) AVStream *st = s->streams[pkt->stream_index]; AVFormatContext *rtp_ctx = st->priv_data; - /* TODO: Send binding request every 1s as WebRTC heartbeat. */ + /* Periodic consent-freshness STUN Binding Request */ + int64_t now = av_gettime(); + if (now - whip->last_consent_tx >= (int64_t)whip->consent_interval * 1000) { + int req_sz; + if (ice_create_request(s, whip->buf, sizeof(whip->buf), &req_sz) >= 0 && ffurl_write(whip->udp, whip->buf, req_sz) == req_sz) { + whip->consent_failures++; + whip->last_consent_tx = now; + av_log(whip, AV_LOG_VERBOSE, "WHIP: consent-freshness request %d sent\n", whip->consent_failures); + } + } /** * Receive packets from the server such as ICE binding requests, DTLS messages, @@ -1783,16 +1960,58 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt) */ ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf)); if (ret > 0) { + if (ice_is_binding_response(whip->buf, ret)) { + whip->consent_failures = 0; + av_log(whip, AV_LOG_VERBOSE, "WHIP: consent-freshness response received, counter reset\n"); + } if (is_dtls_packet(whip->buf, ret)) { if ((ret = ffurl_write(whip->dtls_uc, whip->buf, ret)) < 0) { av_log(whip, AV_LOG_ERROR, "WHIP: Failed to handle DTLS message\n"); goto end; } } + /* Handle RTCP NACK ( RTPFB / FMT=1 ) -------------- */ + if (media_is_rtcp(whip->buf, ret)) { + int ptr = 0; + while (ptr + 4 <= ret) { + uint8_t pt = whip->buf[ptr + 1]; + int len = (AV_RB16(&whip->buf[ptr + 2]) + 1) * 4; + if (ptr + len > ret) break; + + if (pt == 205) { /* RTPFB */ + uint8_t fmt = (whip->buf[ptr] & 0x1f); + if (fmt == 1 && len >= 12) { + uint16_t pid = AV_RB16(&whip->buf[ptr + 12 - 4]); + uint16_t blp = AV_RB16(&whip->buf[ptr + 14 - 4]); + + /* retransmit pid + any bit set in blp */ + for (int bit = -1; bit < 16; bit++) { + uint16_t seq = (bit < 0) ? pid : pid + bit + 1; + if (bit >= 0 && !(blp & (1 << bit))) + continue; + + const RtpHistoryItem * it = rtp_history_find(whip, seq); + if (it) + send_rtx_packet(s, it->pkt, it->size); + + } + } + + } + ptr += len; + } + + } } else if (ret != AVERROR(EAGAIN)) { av_log(whip, AV_LOG_ERROR, "WHIP: Failed to read from UDP socket\n"); goto end; } + /* Check consent freshness consecutive failures */ + if (whip->consent_failures >= WHIP_CONSENT_MAX_FAILURES) { + av_log(whip, AV_LOG_ERROR, "WHIP: No consent-freshness response after %d attempts, closing\n", WHIP_CONSENT_MAX_FAILURES); + ret = AVERROR(EHOSTUNREACH); + goto end; + } if (whip->h264_annexb_insert_sps_pps && st->codecpar->codec_id == AV_CODEC_ID_H264) { if ((ret = h264_annexb_insert_sps_pps(s, pkt)) < 0) { @@ -1891,7 +2110,9 @@ static const AVOption options[] = { { "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, DEC }, { "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC }, { "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC }, - { "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC }, + { "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC }, + { "consent_interval", "STUN consent refresh interval in ms (RFC 7675)", OFFSET(consent_interval), AV_OPT_TYPE_INT, { .i64 = WHIP_CONSENT_DEF_INTERVAL }, 5000, 30000, DEC }, + { "rtx_history", "Packet history size", OFFSET(history_size), AV_OPT_TYPE_INT, { .i64 = HISTORY_SIZE_DEFAULT }, 64, 2048, DEC }, { NULL }, };