[2/3] Integrate XSK maps into the core XDP datapath

src/core/library.c

@@ -915,6 +915,8 @@ QuicLibraryLazyInitialize(
 
     CXPLAT_DATAPATH_INIT_CONFIG InitConfig = {0};
     InitConfig.EnableDscpOnRecv = MsQuicLib.EnableDscpOnRecv;
+    InitConfig.XdpMapConfigs = MsQuicLib.XdpMapConfigs;
+    InitConfig.XdpMapConfigCount = MsQuicLib.XdpMapConfigCount;
 
     Status =
         CxPlatDataPathInitialize(

src/generated/linux/datapath_raw.c.clog.h

@@ -45,10 +45,10 @@ tracepoint(CLOG_DATAPATH_RAW_C, AllocFailure , arg2, arg3);\
 // Decoder Ring for LibraryErrorStatus
 // [ lib] ERROR, %u, %s.
 // QuicTraceEvent(
-            LibraryErrorStatus,
-            "[ lib] ERROR, %u, %s.",
-            PlumbStatus,
-            "CxPlatDpRawPlumbRulesOnSocket (delete)");
+                LibraryErrorStatus,
+                "[ lib] ERROR, %u, %s.",
+                PlumbStatus,
+                "CxPlatDpRawPlumbRulesOnSocket (delete)");
 // arg2 = arg2 = PlumbStatus = arg2
 // arg3 = arg3 = "CxPlatDpRawPlumbRulesOnSocket (delete)" = arg3
 ----------------------------------------------------------*/

src/generated/linux/datapath_raw.c.clog.h.lttng.h

@@ -28,10 +28,10 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_C, AllocFailure,
 // Decoder Ring for LibraryErrorStatus
 // [ lib] ERROR, %u, %s.
 // QuicTraceEvent(
-            LibraryErrorStatus,
-            "[ lib] ERROR, %u, %s.",
-            PlumbStatus,
-            "CxPlatDpRawPlumbRulesOnSocket (delete)");
+                LibraryErrorStatus,
+                "[ lib] ERROR, %u, %s.",
+                PlumbStatus,
+                "CxPlatDpRawPlumbRulesOnSocket (delete)");
 // arg2 = arg2 = PlumbStatus = arg2
 // arg3 = arg3 = "CxPlatDpRawPlumbRulesOnSocket (delete)" = arg3
 ----------------------------------------------------------*/

src/generated/linux/datapath_raw_xdp_win.c.clog.h

@@ -332,6 +332,50 @@ tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdownComplete , arg2);\
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for XdpMapModeConfigured
+// [ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)
+// QuicTraceLogVerbose(
+                XdpMapModeConfigured,
+                "[ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)",
+                Interface,
+                Interface->IfIndex,
+                XskMap);
+// arg2 = arg2 = Interface = arg2
+// arg3 = arg3 = Interface->IfIndex = arg3
+// arg4 = arg4 = XskMap = arg4
+----------------------------------------------------------*/
+#ifndef _clog_5_ARGS_TRACE_XdpMapModeConfigured
+#define _clog_5_ARGS_TRACE_XdpMapModeConfigured(uniqueId, encoded_arg_string, arg2, arg3, arg4)\
+tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapModeConfigured , arg2, arg3, arg4);\
+
+#endif
+
+
+
+
+/*----------------------------------------------------------
+// Decoder Ring for XdpMapModeInserted
+// [ixdp][%p] Map mode: inserted XSK for queue %u (IfIndex=%u)
+// QuicTraceLogVerbose(
+                    XdpMapModeInserted,
+                    "[ixdp][%p] Map mode: inserted XSK for queue %u (IfIndex=%u)",
+                    Interface,
+                    j,
+                    Interface->IfIndex);
+// arg2 = arg2 = Interface = arg2
+// arg3 = arg3 = j = arg3
+// arg4 = arg4 = Interface->IfIndex = arg4
+----------------------------------------------------------*/
+#ifndef _clog_5_ARGS_TRACE_XdpMapModeInserted
+#define _clog_5_ARGS_TRACE_XdpMapModeInserted(uniqueId, encoded_arg_string, arg2, arg3, arg4)\
+tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapModeInserted , arg2, arg3, arg4);\
+
+#endif
+
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for LibraryErrorStatus
 // [ lib] ERROR, %u, %s.

src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h

@@ -336,6 +336,60 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdownComplete,
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for XdpMapModeConfigured
+// [ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)
+// QuicTraceLogVerbose(
+                XdpMapModeConfigured,
+                "[ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)",
+                Interface,
+                Interface->IfIndex,
+                XskMap);
+// arg2 = arg2 = Interface = arg2
+// arg3 = arg3 = Interface->IfIndex = arg3
+// arg4 = arg4 = XskMap = arg4
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapModeConfigured,
+    TP_ARGS(
+        const void *, arg2,
+        unsigned int, arg3,
+        const void *, arg4), 
+    TP_FIELDS(
+        ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2)
+        ctf_integer(unsigned int, arg3, arg3)
+        ctf_integer_hex(uint64_t, arg4, (uint64_t)arg4)
+    )
+)
+
+
+
+/*----------------------------------------------------------
+// Decoder Ring for XdpMapModeInserted
+// [ixdp][%p] Map mode: inserted XSK for queue %u (IfIndex=%u)
+// QuicTraceLogVerbose(
+                    XdpMapModeInserted,
+                    "[ixdp][%p] Map mode: inserted XSK for queue %u (IfIndex=%u)",
+                    Interface,
+                    j,
+                    Interface->IfIndex);
+// arg2 = arg2 = Interface = arg2
+// arg3 = arg3 = j = arg3
+// arg4 = arg4 = Interface->IfIndex = arg4
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapModeInserted,
+    TP_ARGS(
+        const void *, arg2,
+        unsigned int, arg3,
+        unsigned int, arg4), 
+    TP_FIELDS(
+        ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2)
+        ctf_integer(unsigned int, arg3, arg3)
+        ctf_integer(unsigned int, arg4, arg4)
+    )
+)
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for LibraryErrorStatus
 // [ lib] ERROR, %u, %s.

src/generated/linux/datapath_xplat.c.clog.h

@@ -26,6 +26,10 @@
 #define _clog_MACRO_QuicTraceLogError  1
 #define QuicTraceLogError(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__)))
 #endif
+#ifndef _clog_MACRO_QuicTraceEvent
+#define _clog_MACRO_QuicTraceEvent  1
+#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__)))
+#endif
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -63,12 +67,46 @@ tracepoint(CLOG_DATAPATH_XPLAT_C, WarnNoXdpForCibirSockets );\
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for DatapathRawInitFailMapMode
+// [  dp] XDP map mode: raw datapath required but failed to initialize
+// QuicTraceLogVerbose(
+                DatapathRawInitFailMapMode,
+                "[  dp] XDP map mode: raw datapath required but failed to initialize");
+----------------------------------------------------------*/
+#ifndef _clog_2_ARGS_TRACE_DatapathRawInitFailMapMode
+#define _clog_2_ARGS_TRACE_DatapathRawInitFailMapMode(uniqueId, encoded_arg_string)\
+tracepoint(CLOG_DATAPATH_XPLAT_C, DatapathRawInitFailMapMode );\
+
+#endif
+
+
+
+
+/*----------------------------------------------------------
+// Decoder Ring for DatapathRawMapInsertFail
+// [  dp] XDP map mode: failed to insert XSK sockets into map, status:%d
+// QuicTraceLogVerbose(
+                DatapathRawMapInsertFail,
+                "[  dp] XDP map mode: failed to insert XSK sockets into map, status:%d",
+                Status);
+// arg2 = arg2 = Status = arg2
+----------------------------------------------------------*/
+#ifndef _clog_3_ARGS_TRACE_DatapathRawMapInsertFail
+#define _clog_3_ARGS_TRACE_DatapathRawMapInsertFail(uniqueId, encoded_arg_string, arg2)\
+tracepoint(CLOG_DATAPATH_XPLAT_C, DatapathRawMapInsertFail , arg2);\
+
+#endif
+
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for DatapathInitFail
 // [  dp] Failed to initialize datapath, status:%d
 // QuicTraceLogVerbose(
-            DatapathInitFail,
-            "[  dp] Failed to initialize datapath, status:%d", Status);
+                DatapathInitFail,
+                "[  dp] Failed to initialize datapath, status:%d", Status);
 // arg2 = arg2 = Status = arg2
 ----------------------------------------------------------*/
 #ifndef _clog_3_ARGS_TRACE_DatapathInitFail
@@ -130,6 +168,26 @@ tracepoint(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForQtip );\
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for AllocFailure
+// Allocation of '%s' failed. (%llu bytes)
+// QuicTraceEvent(
+                AllocFailure,
+                "Allocation of '%s' failed. (%llu bytes)",
+                "CXPLAT_DATAPATH (map mode)",
+                sizeof(CXPLAT_DATAPATH));
+// arg2 = arg2 = "CXPLAT_DATAPATH (map mode)" = arg2
+// arg3 = arg3 = sizeof(CXPLAT_DATAPATH) = arg3
+----------------------------------------------------------*/
+#ifndef _clog_4_ARGS_TRACE_AllocFailure
+#define _clog_4_ARGS_TRACE_AllocFailure(uniqueId, encoded_arg_string, arg2, arg3)\
+tracepoint(CLOG_DATAPATH_XPLAT_C, AllocFailure , arg2, arg3);\
+
+#endif
+
+
+
+
 #ifdef __cplusplus
 }
 #endif

src/generated/linux/datapath_xplat.c.clog.h.lttng.h

@@ -35,12 +35,47 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, WarnNoXdpForCibirSockets,
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for DatapathRawInitFailMapMode
+// [  dp] XDP map mode: raw datapath required but failed to initialize
+// QuicTraceLogVerbose(
+                DatapathRawInitFailMapMode,
+                "[  dp] XDP map mode: raw datapath required but failed to initialize");
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, DatapathRawInitFailMapMode,
+    TP_ARGS(
+), 
+    TP_FIELDS(
+    )
+)
+
+
+
+/*----------------------------------------------------------
+// Decoder Ring for DatapathRawMapInsertFail
+// [  dp] XDP map mode: failed to insert XSK sockets into map, status:%d
+// QuicTraceLogVerbose(
+                DatapathRawMapInsertFail,
+                "[  dp] XDP map mode: failed to insert XSK sockets into map, status:%d",
+                Status);
+// arg2 = arg2 = Status = arg2
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, DatapathRawMapInsertFail,
+    TP_ARGS(
+        int, arg2), 
+    TP_FIELDS(
+        ctf_integer(int, arg2, arg2)
+    )
+)
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for DatapathInitFail
 // [  dp] Failed to initialize datapath, status:%d
 // QuicTraceLogVerbose(
-            DatapathInitFail,
-            "[  dp] Failed to initialize datapath, status:%d", Status);
+                DatapathInitFail,
+                "[  dp] Failed to initialize datapath, status:%d", Status);
 // arg2 = arg2 = Status = arg2
 ----------------------------------------------------------*/
 TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, DatapathInitFail,
@@ -102,3 +137,26 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForQtip,
     TP_FIELDS(
     )
 )
+
+
+
+/*----------------------------------------------------------
+// Decoder Ring for AllocFailure
+// Allocation of '%s' failed. (%llu bytes)
+// QuicTraceEvent(
+                AllocFailure,
+                "Allocation of '%s' failed. (%llu bytes)",
+                "CXPLAT_DATAPATH (map mode)",
+                sizeof(CXPLAT_DATAPATH));
+// arg2 = arg2 = "CXPLAT_DATAPATH (map mode)" = arg2
+// arg3 = arg3 = sizeof(CXPLAT_DATAPATH) = arg3
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, AllocFailure,
+    TP_ARGS(
+        const char *, arg2,
+        unsigned long long, arg3), 
+    TP_FIELDS(
+        ctf_string(arg2, arg2)
+        ctf_integer(uint64_t, arg3, arg3)
+    )
+)

src/inc/quic_datapath.h

@@ -476,6 +476,18 @@ typedef struct CXPLAT_DATAPATH_INIT_CONFIG {
     // the Windows fast path causing a large performance regression.
     //
     BOOLEAN EnableDscpOnRecv;
+
+    //
+    // External XDP map configurations. When XdpMapConfigCount > 0 and
+    // XdpMapConfigs is non-NULL, the datapath operates in XDP map mode: the
+    // WinSock (normal) datapath is skipped, the raw (XDP) datapath is required
+    // to succeed, and XSK sockets are inserted into the provided XSKMAPs at
+    // init time.
+    // XdpMapConfigs is NULL if and only if XdpMapConfigCount == 0.
+    // The map configs must remain valid for the lifetime of the datapath.
+    //
+    const struct QUIC_XDP_MAP_CONFIG* XdpMapConfigs;
+    uint32_t XdpMapConfigCount;
 } CXPLAT_DATAPATH_INIT_CONFIG;
 
 //