avformat/tls: move DTLS packet detection into ff_is_dtls_packet()

Move the DTLS packet detection logic from whip.c into a shared
ff_is_dtls_packet() function in tls.c, with its declaration and
related macros in tls.h. Update whip.c to use the new shared function.

Signed-off-by: Nariman-Sayed <narimansayed28@gmail.com>

libavformat/tls.c

@@ -28,6 +28,7 @@
 #include "tls.h"
 #include "libavutil/avstring.h"
 #include "libavutil/getenv_utf8.h"
+#include "libavutil/intreadwrite.h"
 #include "libavutil/mem.h"
 #include "libavutil/parseutils.h"
 
@@ -151,3 +152,13 @@ end:
     av_dict_free(&opts);
     return ret;
 }
+
+int ff_is_dtls_packet(const uint8_t *buf, int size)
+{
+    if (size > DTLS_RECORD_LAYER_HEADER_LEN) {
+        uint16_t version = AV_RB16(&buf[1]);
+        return buf[0] >= DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC &&
+               (version == DTLS_VERSION_10 || version == DTLS_VERSION_12);
+    }
+    return 0;
+}

libavformat/tls.h

@@ -34,6 +34,26 @@
  */
 #define MAX_CERTIFICATE_SIZE 8192
 
+/**
+ * The DTLS content type.
+ * See https://tools.ietf.org/html/rfc2246#section-6.2.1
+ * change_cipher_spec(20), alert(21), handshake(22), application_data(23)
+ */
+#define DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC 20
+/**
+ * The DTLS record layer header has a total size of 13 bytes, consisting of
+ * ContentType (1 byte), ProtocolVersion (2 bytes), Epoch (2 bytes),
+ * SequenceNumber (6 bytes), and Length (2 bytes).
+ * See https://datatracker.ietf.org/doc/html/rfc9147#section-4
+ */
+#define DTLS_RECORD_LAYER_HEADER_LEN 13
+/**
+ * The DTLS version number, which is 0xfeff for DTLS 1.0, or 0xfefd for DTLS 1.2.
+ * See https://datatracker.ietf.org/doc/html/rfc9147#name-the-dtls-record-layer
+ */
+#define DTLS_VERSION_10 0xfeff
+#define DTLS_VERSION_12 0xfefd
+
 typedef struct TLSShared {
     const AVClass *class;
     char *ca_file;
@@ -113,4 +133,9 @@ void ff_gnutls_deinit(void);
 int ff_openssl_init(void);
 void ff_openssl_deinit(void);
 
+/**
+ * Whether the packet is a DTLS packet, as defined by RFC 5764 Section 5.1.2.
+ */
+int ff_is_dtls_packet(const uint8_t *buf, int size);
+
 #endif /* AVFORMAT_TLS_H */

libavformat/whip.c

@@ -87,28 +87,6 @@
  */
 #define STUN_HOST_CANDIDATE_PRIORITY 126 << 24 | 65535 << 8 | 255
 
-/**
- * The DTLS content type.
- * See https://tools.ietf.org/html/rfc2246#section-6.2.1
- * change_cipher_spec(20), alert(21), handshake(22), application_data(23)
- */
-#define DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC 20
-
-/**
- * The DTLS record layer header has a total size of 13 bytes, consisting of
- * ContentType (1 byte), ProtocolVersion (2 bytes), Epoch (2 bytes),
- * SequenceNumber (6 bytes), and Length (2 bytes).
- * See https://datatracker.ietf.org/doc/html/rfc9147#section-4
- */
-#define DTLS_RECORD_LAYER_HEADER_LEN 13
-
-/**
- * The DTLS version number, which is 0xfeff for DTLS 1.0, or 0xfefd for DTLS 1.2.
- * See https://datatracker.ietf.org/doc/html/rfc9147#name-the-dtls-record-layer
- */
-#define DTLS_VERSION_10 0xfeff
-#define DTLS_VERSION_12 0xfefd
-
 /**
  * Maximum size of the buffer for sending and receiving UDP packets.
  * Please note that this size does not limit the size of the UDP packet that can be sent.
@@ -362,21 +340,6 @@ typedef struct WHIPContext {
     int hist_head;
 } WHIPContext;
 
-/**
- * Whether the packet is a DTLS packet.
- */
-static int is_dtls_packet(uint8_t *b, int size)
-{
-    int ret = 0;
-    if (size > DTLS_RECORD_LAYER_HEADER_LEN) {
-        uint16_t version = AV_RB16(&b[1]);
-        ret = b[0] >= DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC &&
-            (version == DTLS_VERSION_10 || version == DTLS_VERSION_12);
-    }
-    return ret;
-}
-
-
 /**
  * Get or Generate a self-signed certificate and private key for DTLS,
  * fingerprint for SDP
@@ -1386,7 +1349,7 @@ next_packet:
         }
 
         /* Handle DTLS handshake */
-        if (is_dtls_packet(whip->buf, ret) || is_dtls_active) {
+        if (ff_is_dtls_packet(whip->buf, ret) || is_dtls_active) {
             whip->whip_ice_time = av_gettime_relative();
             /* Start consent timer when ICE selected */
             whip->whip_last_consent_tx_time = whip->whip_last_consent_rx_time = whip->whip_ice_time;
@@ -2068,7 +2031,7 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
         whip->whip_last_consent_rx_time = av_gettime_relative();
         av_log(whip, AV_LOG_DEBUG, "Consent Freshness check received\n");
     }
-    if (is_dtls_packet(whip->buf, ret)) {
+    if (ff_is_dtls_packet(whip->buf, ret)) {
         if ((ret = ffurl_write(whip->dtls_uc, whip->buf, ret)) < 0) {
             av_log(whip, AV_LOG_ERROR, "Failed to handle DTLS message\n");
             goto end;