diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 62bac83c91..17cbb4fdd8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -196,17 +196,17 @@ jobs: if: matrix.vec.os == 'WinServerPrerelease' shell: pwsh timeout-minutes: 120 - run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -GHA -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }} -GenerateXmlResults ${{ matrix.vec.xdp }} ${{ matrix.vec.qtip }} ${{ inputs.filter && '-Filter' }} ${{ inputs.filter || '' }} + run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -GHA -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }} -GenerateXmlResults ${{ matrix.vec.xdp }} ${{ matrix.vec.qtip }} ${{ matrix.vec.xdpmapmode }} ${{ (matrix.vec.testfilter || inputs.filter) && '-Filter' || '' }} ${{ matrix.vec.testfilter || inputs.filter || '' }} - name: Test if: matrix.vec.os != 'WinServerPrerelease' shell: pwsh timeout-minutes: 120 - run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -OsRunner ${{ matrix.vec.os }} -GHA -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }} -GenerateXmlResults ${{ matrix.vec.xdp }} ${{ matrix.vec.qtip }} ${{ inputs.filter && '-Filter' }} ${{ inputs.filter || '' }} + run: scripts/test.ps1 -Config ${{ matrix.vec.config }} -Arch ${{ matrix.vec.arch }} -Tls ${{ matrix.vec.tls }} -OsRunner ${{ matrix.vec.os }} -GHA -LogProfile ${{ matrix.vec.log || (inputs.log_level || 'Full.Light') }} -GenerateXmlResults ${{ matrix.vec.xdp }} ${{ matrix.vec.qtip }} ${{ matrix.vec.xdpmapmode }} ${{ (matrix.vec.testfilter || inputs.filter) && '-Filter' || '' }} ${{ matrix.vec.testfilter || inputs.filter || '' }} - name: Upload on Failure or Cancellation uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a if: failure() || cancelled() with: - name: BVT-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.iouring }}${{ matrix.vec.qtip }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }} + name: BVT-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.iouring }}${{ matrix.vec.qtip }}${{ matrix.vec.xdpmapmode }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }} path: artifacts bvt-kernel: diff --git a/scripts/run-gtest.ps1 b/scripts/run-gtest.ps1 index 31708b483f..fa47a98db5 100644 --- a/scripts/run-gtest.ps1 +++ b/scripts/run-gtest.ps1 @@ -62,6 +62,9 @@ as necessary. .PARAMETER DuoNic Uses DuoNic instead of loopback. +.PARAMETER XdpMapMode + Uses XDP map mode with DuoNic. + #> param ( @@ -133,6 +136,9 @@ param ( [Parameter(Mandatory = $false)] [switch]$DuoNic = $false, + [Parameter(Mandatory = $false)] + [switch]$XdpMapMode = $false, + [Parameter(Mandatory = $false)] [string]$OsRunner = "", @@ -403,6 +409,9 @@ function Start-TestCase([String]$Name) { if ($DuoNic) { $Arguments += " --duoNic" } + if ($XdpMapMode) { + $Arguments += " --xdpMapMode" + } if ($UseQtip) { $Arguments += " --useQTIP" } @@ -450,6 +459,9 @@ function Start-AllTestCases { if ($DuoNic) { $Arguments += " --duoNic" } + if ($XdpMapMode) { + $Arguments += " --xdpMapMode" + } if ($UseQtip) { $Arguments += " --useQTIP" } diff --git a/scripts/test.ps1 b/scripts/test.ps1 index 74da3b7bdd..8e960a5eed 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -70,6 +70,9 @@ This script runs the MsQuic tests. .Parameter DuoNic Uses DuoNic instead of loopback (DuoNic must already be installed via 'prepare-machine.ps1 -InstallDuoNic'). +.Parameter XdpMapMode + Uses XDP map mode with DuoNic (Windows user-mode only, requires XDP + DuoNic). + .Parameter NumIterations Number of times to run this particular command. Catches tricky edge cases due to random nature of networks. @@ -178,6 +181,9 @@ param ( [Parameter(Mandatory = $false)] [switch]$DuoNic = $false, + [Parameter(Mandatory = $false)] + [switch]$XdpMapMode = $false, + [Parameter(Mandatory = $false)] [switch]$UseXdp = $false, @@ -235,6 +241,11 @@ if ($UseXdp) { $DuoNic = $true } +if ($XdpMapMode) { + # Map mode implies DuoNic + $DuoNic = $true +} + # Root directory of the project. $RootDir = Split-Path $PSScriptRoot -Parent @@ -298,6 +309,9 @@ $TestArguments = "-IsolationMode $IsolationMode -PfxPath $PfxFile" if ($DuoNic) { $TestArguments += " -DuoNic" } +if ($XdpMapMode) { + $TestArguments += " -XdpMapMode" +} if ($Kernel) { $TestArguments += " -Kernel $KernelPath" } diff --git a/scripts/update-sidecar.ps1 b/scripts/update-sidecar.ps1 index ed8dbf8b1a..a600c90b34 100644 --- a/scripts/update-sidecar.ps1 +++ b/scripts/update-sidecar.ps1 @@ -33,10 +33,15 @@ if (Test-Path $OutputDir) { } } +$TmpOutputDir = Join-Path $RootDir "build" "tmp" + +# Clean stale generated files so removed/renamed sources don't leave orphans +if (Test-Path $TmpOutputDir) { + Remove-Item $TmpOutputDir -Recurse -Force +} + $Sidecar = Join-Path $SrcDir "manifest" "clog.sidecar" $ConfigFile = Join-Path $SrcDir "manifest" "msquic.clog_config" - -$TmpOutputDir = Join-Path $RootDir "build" "tmp" $ClogDir = Join-Path $RootDir "build" "clog" # Create directories diff --git a/src/core/library.c b/src/core/library.c index f7a938cc2c..b7e37ab24c 100644 --- a/src/core/library.c +++ b/src/core/library.c @@ -915,6 +915,8 @@ QuicLibraryLazyInitialize( CXPLAT_DATAPATH_INIT_CONFIG InitConfig = {0}; InitConfig.EnableDscpOnRecv = MsQuicLib.EnableDscpOnRecv; + InitConfig.XdpMapConfigs = MsQuicLib.XdpMapConfigs; + InitConfig.XdpMapConfigCount = MsQuicLib.XdpMapConfigCount; Status = CxPlatDataPathInitialize( diff --git a/src/core/settings.c b/src/core/settings.c index 814aed882c..e76b32cd2e 100644 --- a/src/core/settings.c +++ b/src/core/settings.c @@ -712,6 +712,18 @@ QuicSettingApply( Destination->IsSet.ReliableResetEnabled = TRUE; } + // + // If XDP map mode is active (XdpMapConfigCount > 0), XDP is implicitly + // enabled for all sockets. Reject an explicit XdpEnabled = FALSE since + // it contradicts map mode — there is no OS datapath to fall back to. + // + if (Source->IsSet.XdpEnabled && !Source->XdpEnabled && MsQuicLib.XdpMapConfigCount > 0) { + QuicTraceLogError( + SettingXdpDisabledInMapMode, + "[ lib] Error: XdpEnabled cannot be set to FALSE when XDP map mode is active."); + return FALSE; + } + if (Source->IsSet.XdpEnabled && (!Destination->IsSet.XdpEnabled || OverWrite)) { Destination->XdpEnabled = Source->XdpEnabled; Destination->IsSet.XdpEnabled = TRUE; diff --git a/src/generated/linux/datapath_raw.c.clog.h b/src/generated/linux/datapath_raw.c.clog.h index 35e4abef69..3f47da86a4 100644 --- a/src/generated/linux/datapath_raw.c.clog.h +++ b/src/generated/linux/datapath_raw.c.clog.h @@ -14,6 +14,10 @@ #include "datapath_raw.c.clog.h.lttng.h" #endif #include +#ifndef _clog_MACRO_QuicTraceLogVerbose +#define _clog_MACRO_QuicTraceLogVerbose 1 +#define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif #ifndef _clog_MACRO_QuicTraceEvent #define _clog_MACRO_QuicTraceEvent 1 #define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) @@ -21,6 +25,24 @@ #ifdef __cplusplus extern "C" { #endif +/*---------------------------------------------------------- +// Decoder Ring for DatapathRawMapInsertFail +// [ dp] XDP map mode: failed to insert XSK sockets into map, status:%d +// QuicTraceLogVerbose( + DatapathRawMapInsertFail, + "[ dp] XDP map mode: failed to insert XSK sockets into map, status:%d", + Status); +// arg2 = arg2 = Status = arg2 +----------------------------------------------------------*/ +#ifndef _clog_3_ARGS_TRACE_DatapathRawMapInsertFail +#define _clog_3_ARGS_TRACE_DatapathRawMapInsertFail(uniqueId, encoded_arg_string, arg2)\ +tracepoint(CLOG_DATAPATH_RAW_C, DatapathRawMapInsertFail , arg2);\ + +#endif + + + + /*---------------------------------------------------------- // Decoder Ring for AllocFailure // Allocation of '%s' failed. (%llu bytes) diff --git a/src/generated/linux/datapath_raw.c.clog.h.lttng.h b/src/generated/linux/datapath_raw.c.clog.h.lttng.h index 4edb7a0fb6..5031a7e859 100644 --- a/src/generated/linux/datapath_raw.c.clog.h.lttng.h +++ b/src/generated/linux/datapath_raw.c.clog.h.lttng.h @@ -1,6 +1,25 @@ +/*---------------------------------------------------------- +// Decoder Ring for DatapathRawMapInsertFail +// [ dp] XDP map mode: failed to insert XSK sockets into map, status:%d +// QuicTraceLogVerbose( + DatapathRawMapInsertFail, + "[ dp] XDP map mode: failed to insert XSK sockets into map, status:%d", + Status); +// arg2 = arg2 = Status = arg2 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_C, DatapathRawMapInsertFail, + TP_ARGS( + int, arg2), + TP_FIELDS( + ctf_integer(int, arg2, arg2) + ) +) + + + /*---------------------------------------------------------- // Decoder Ring for AllocFailure // Allocation of '%s' failed. (%llu bytes) diff --git a/src/generated/linux/datapath_raw_xdp_win.c.clog.h b/src/generated/linux/datapath_raw_xdp_win.c.clog.h index 3b4f03bf3b..1f0d10daac 100644 --- a/src/generated/linux/datapath_raw_xdp_win.c.clog.h +++ b/src/generated/linux/datapath_raw_xdp_win.c.clog.h @@ -332,6 +332,54 @@ tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdownComplete , arg2);\ +/*---------------------------------------------------------- +// Decoder Ring for XdpMapInsertFailed +// [ixdp][%p] XdpMapInsert failed for IfIndex=%u, QueueId=%u, XskMap=%p, RxXsk=%p +// QuicTraceLogVerbose( + XdpMapInsertFailed, + "[ixdp][%p] XdpMapInsert failed for IfIndex=%u, QueueId=%u, XskMap=%p, RxXsk=%p", + Interface, + Interface->IfIndex, + j, + XskMap, + Queue->RxXsk); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->IfIndex = arg3 +// arg4 = arg4 = j = arg4 +// arg5 = arg5 = XskMap = arg5 +// arg6 = arg6 = Queue->RxXsk = arg6 +----------------------------------------------------------*/ +#ifndef _clog_7_ARGS_TRACE_XdpMapInsertFailed +#define _clog_7_ARGS_TRACE_XdpMapInsertFailed(uniqueId, encoded_arg_string, arg2, arg3, arg4, arg5, arg6)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapInsertFailed , arg2, arg3, arg4, arg5, arg6);\ + +#endif + + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpMapModeConfigured +// [ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p) +// QuicTraceLogVerbose( + XdpMapModeConfigured, + "[ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)", + Interface, + Interface->IfIndex, + XskMap); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->IfIndex = arg3 +// arg4 = arg4 = XskMap = arg4 +----------------------------------------------------------*/ +#ifndef _clog_5_ARGS_TRACE_XdpMapModeConfigured +#define _clog_5_ARGS_TRACE_XdpMapModeConfigured(uniqueId, encoded_arg_string, arg2, arg3, arg4)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapModeConfigured , arg2, arg3, arg4);\ + +#endif + + + + /*---------------------------------------------------------- // Decoder Ring for LibraryErrorStatus // [ lib] ERROR, %u, %s. diff --git a/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h index a1850a527c..c4bc81b8f3 100644 --- a/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h +++ b/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h @@ -336,6 +336,68 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpPartitionShutdownComplete, +/*---------------------------------------------------------- +// Decoder Ring for XdpMapInsertFailed +// [ixdp][%p] XdpMapInsert failed for IfIndex=%u, QueueId=%u, XskMap=%p, RxXsk=%p +// QuicTraceLogVerbose( + XdpMapInsertFailed, + "[ixdp][%p] XdpMapInsert failed for IfIndex=%u, QueueId=%u, XskMap=%p, RxXsk=%p", + Interface, + Interface->IfIndex, + j, + XskMap, + Queue->RxXsk); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->IfIndex = arg3 +// arg4 = arg4 = j = arg4 +// arg5 = arg5 = XskMap = arg5 +// arg6 = arg6 = Queue->RxXsk = arg6 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapInsertFailed, + TP_ARGS( + const void *, arg2, + unsigned int, arg3, + unsigned int, arg4, + const void *, arg5, + const void *, arg6), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ctf_integer(unsigned int, arg4, arg4) + ctf_integer_hex(uint64_t, arg5, (uint64_t)arg5) + ctf_integer_hex(uint64_t, arg6, (uint64_t)arg6) + ) +) + + + +/*---------------------------------------------------------- +// Decoder Ring for XdpMapModeConfigured +// [ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p) +// QuicTraceLogVerbose( + XdpMapModeConfigured, + "[ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)", + Interface, + Interface->IfIndex, + XskMap); +// arg2 = arg2 = Interface = arg2 +// arg3 = arg3 = Interface->IfIndex = arg3 +// arg4 = arg4 = XskMap = arg4 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpMapModeConfigured, + TP_ARGS( + const void *, arg2, + unsigned int, arg3, + const void *, arg4), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ctf_integer_hex(uint64_t, arg4, (uint64_t)arg4) + ) +) + + + /*---------------------------------------------------------- // Decoder Ring for LibraryErrorStatus // [ lib] ERROR, %u, %s. diff --git a/src/generated/linux/datapath_xplat.c.clog.h b/src/generated/linux/datapath_xplat.c.clog.h index da7666512f..3c0ed3c61d 100644 --- a/src/generated/linux/datapath_xplat.c.clog.h +++ b/src/generated/linux/datapath_xplat.c.clog.h @@ -26,6 +26,10 @@ #define _clog_MACRO_QuicTraceLogError 1 #define QuicTraceLogError(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) #endif +#ifndef _clog_MACRO_QuicTraceEvent +#define _clog_MACRO_QuicTraceEvent 1 +#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__))) +#endif #ifdef __cplusplus extern "C" { #endif @@ -63,12 +67,28 @@ tracepoint(CLOG_DATAPATH_XPLAT_C, WarnNoXdpForCibirSockets );\ +/*---------------------------------------------------------- +// Decoder Ring for DatapathRawInitFailRawOnly +// [ dp] Raw-only mode: raw datapath required but failed to initialize +// QuicTraceLogVerbose( + DatapathRawInitFailRawOnly, + "[ dp] Raw-only mode: raw datapath required but failed to initialize"); +----------------------------------------------------------*/ +#ifndef _clog_2_ARGS_TRACE_DatapathRawInitFailRawOnly +#define _clog_2_ARGS_TRACE_DatapathRawInitFailRawOnly(uniqueId, encoded_arg_string)\ +tracepoint(CLOG_DATAPATH_XPLAT_C, DatapathRawInitFailRawOnly );\ + +#endif + + + + /*---------------------------------------------------------- // Decoder Ring for DatapathInitFail // [ dp] Failed to initialize datapath, status:%d // QuicTraceLogVerbose( - DatapathInitFail, - "[ dp] Failed to initialize datapath, status:%d", Status); + DatapathInitFail, + "[ dp] Failed to initialize datapath, status:%d", Status); // arg2 = arg2 = Status = arg2 ----------------------------------------------------------*/ #ifndef _clog_3_ARGS_TRACE_DatapathInitFail @@ -130,6 +150,26 @@ tracepoint(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForQtip );\ +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "CXPLAT_DATAPATH (raw-only)", + sizeof(CXPLAT_DATAPATH)); +// arg2 = arg2 = "CXPLAT_DATAPATH (raw-only)" = arg2 +// arg3 = arg3 = sizeof(CXPLAT_DATAPATH) = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_AllocFailure +#define _clog_4_ARGS_TRACE_AllocFailure(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_XPLAT_C, AllocFailure , arg2, arg3);\ + +#endif + + + + #ifdef __cplusplus } #endif diff --git a/src/generated/linux/datapath_xplat.c.clog.h.lttng.h b/src/generated/linux/datapath_xplat.c.clog.h.lttng.h index 45f0bcbaae..441de90373 100644 --- a/src/generated/linux/datapath_xplat.c.clog.h.lttng.h +++ b/src/generated/linux/datapath_xplat.c.clog.h.lttng.h @@ -35,12 +35,28 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, WarnNoXdpForCibirSockets, +/*---------------------------------------------------------- +// Decoder Ring for DatapathRawInitFailRawOnly +// [ dp] Raw-only mode: raw datapath required but failed to initialize +// QuicTraceLogVerbose( + DatapathRawInitFailRawOnly, + "[ dp] Raw-only mode: raw datapath required but failed to initialize"); +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, DatapathRawInitFailRawOnly, + TP_ARGS( +), + TP_FIELDS( + ) +) + + + /*---------------------------------------------------------- // Decoder Ring for DatapathInitFail // [ dp] Failed to initialize datapath, status:%d // QuicTraceLogVerbose( - DatapathInitFail, - "[ dp] Failed to initialize datapath, status:%d", Status); + DatapathInitFail, + "[ dp] Failed to initialize datapath, status:%d", Status); // arg2 = arg2 = Status = arg2 ----------------------------------------------------------*/ TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, DatapathInitFail, @@ -102,3 +118,26 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, ErrNoXdpForQtip, TP_FIELDS( ) ) + + + +/*---------------------------------------------------------- +// Decoder Ring for AllocFailure +// Allocation of '%s' failed. (%llu bytes) +// QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "CXPLAT_DATAPATH (raw-only)", + sizeof(CXPLAT_DATAPATH)); +// arg2 = arg2 = "CXPLAT_DATAPATH (raw-only)" = arg2 +// arg3 = arg3 = sizeof(CXPLAT_DATAPATH) = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_XPLAT_C, AllocFailure, + TP_ARGS( + const char *, arg2, + unsigned long long, arg3), + TP_FIELDS( + ctf_string(arg2, arg2) + ctf_integer(uint64_t, arg3, arg3) + ) +) diff --git a/src/generated/linux/settings.c.clog.h b/src/generated/linux/settings.c.clog.h index 6d721ac673..c39e2a3b8d 100644 --- a/src/generated/linux/settings.c.clog.h +++ b/src/generated/linux/settings.c.clog.h @@ -872,6 +872,22 @@ tracepoint(CLOG_SETTINGS_C, SettingStreamMultiReceiveEnabled , arg2);\ +/*---------------------------------------------------------- +// Decoder Ring for SettingXdpDisabledInMapMode +// [ lib] Error: XdpEnabled cannot be set to FALSE when XDP map mode is active. +// QuicTraceLogError( + SettingXdpDisabledInMapMode, + "[ lib] Error: XdpEnabled cannot be set to FALSE when XDP map mode is active."); +----------------------------------------------------------*/ +#ifndef _clog_2_ARGS_TRACE_SettingXdpDisabledInMapMode +#define _clog_2_ARGS_TRACE_SettingXdpDisabledInMapMode(uniqueId, encoded_arg_string)\ +tracepoint(CLOG_SETTINGS_C, SettingXdpDisabledInMapMode );\ + +#endif + + + + /*---------------------------------------------------------- // Decoder Ring for SettingsLoadInvalidAcceptableVersion // Invalid AcceptableVersion loaded from storage! 0x%x at position %d diff --git a/src/generated/linux/settings.c.clog.h.lttng.h b/src/generated/linux/settings.c.clog.h.lttng.h index f7e15549d7..9dbe097f25 100644 --- a/src/generated/linux/settings.c.clog.h.lttng.h +++ b/src/generated/linux/settings.c.clog.h.lttng.h @@ -906,6 +906,22 @@ TRACEPOINT_EVENT(CLOG_SETTINGS_C, SettingStreamMultiReceiveEnabled, +/*---------------------------------------------------------- +// Decoder Ring for SettingXdpDisabledInMapMode +// [ lib] Error: XdpEnabled cannot be set to FALSE when XDP map mode is active. +// QuicTraceLogError( + SettingXdpDisabledInMapMode, + "[ lib] Error: XdpEnabled cannot be set to FALSE when XDP map mode is active."); +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_SETTINGS_C, SettingXdpDisabledInMapMode, + TP_ARGS( +), + TP_FIELDS( + ) +) + + + /*---------------------------------------------------------- // Decoder Ring for SettingsLoadInvalidAcceptableVersion // Invalid AcceptableVersion loaded from storage! 0x%x at position %d diff --git a/src/inc/quic_datapath.h b/src/inc/quic_datapath.h index f98bef52ec..0c4757da30 100644 --- a/src/inc/quic_datapath.h +++ b/src/inc/quic_datapath.h @@ -476,6 +476,20 @@ typedef struct CXPLAT_DATAPATH_INIT_CONFIG { // the Windows fast path causing a large performance regression. // BOOLEAN EnableDscpOnRecv; + + // + // External XDP map configurations. When XdpMapConfigCount > 0 and + // XdpMapConfigs is non-NULL, the datapath operates in XDP map mode: the + // WinSock (normal) datapath is skipped, the raw (XDP) datapath is required + // to succeed, and XSK sockets are inserted into the provided XSKMAPs at + // init time. + // XdpMapConfigs is NULL if and only if XdpMapConfigCount == 0. + // The map configs must remain valid for the lifetime of the datapath. + // + // N.B. Currently only supported for Windows user-mode. + // + const struct QUIC_XDP_MAP_CONFIG* XdpMapConfigs; + uint32_t XdpMapConfigCount; } CXPLAT_DATAPATH_INIT_CONFIG; // diff --git a/src/manifest/clog.sidecar b/src/manifest/clog.sidecar index 540e4cb0fa..d77185ff1b 100644 --- a/src/manifest/clog.sidecar +++ b/src/manifest/clog.sidecar @@ -2807,6 +2807,25 @@ ], "macroName": "QuicTraceLogWarning" }, + "DatapathRawInitFailRawOnly": { + "ModuleProperites": {}, + "TraceString": "[ dp] Raw-only mode: raw datapath required but failed to initialize", + "UniqueId": "DatapathRawInitFailRawOnly", + "splitArgs": [], + "macroName": "QuicTraceLogVerbose" + }, + "DatapathRawMapInsertFail": { + "ModuleProperites": {}, + "TraceString": "[ dp] XDP map mode: failed to insert XSK sockets into map, status:%d", + "UniqueId": "DatapathRawMapInsertFail", + "splitArgs": [ + { + "DefinationEncoding": "d", + "MacroVariableName": "arg2" + } + ], + "macroName": "QuicTraceLogVerbose" + }, "DatapathRecv": { "ModuleProperites": {}, "TraceString": "[data][%p] Recv %u bytes (segment=%hu) Src=%!ADDR! Dst=%!ADDR!", @@ -10998,6 +11017,13 @@ ], "macroName": "QuicTraceLogVerbose" }, + "SettingXdpDisabledInMapMode": { + "ModuleProperites": {}, + "TraceString": "[ lib] Error: XdpEnabled cannot be set to FALSE when XDP map mode is active.", + "UniqueId": "SettingXdpDisabledInMapMode", + "splitArgs": [], + "macroName": "QuicTraceLogError" + }, "SettingXdpEnabled": { "ModuleProperites": {}, "TraceString": "[sett] XdpEnabled = %hhu", @@ -12347,6 +12373,82 @@ ], "macroName": "QuicTraceLogVerbose" }, + "XdpMapInsertFailed": { + "ModuleProperites": {}, + "TraceString": "[ixdp][%p] XdpMapInsert failed for IfIndex=%u, QueueId=%u, XskMap=%p, RxXsk=%p", + "UniqueId": "XdpMapInsertFailed", + "splitArgs": [ + { + "DefinationEncoding": "p", + "MacroVariableName": "arg2" + }, + { + "DefinationEncoding": "u", + "MacroVariableName": "arg3" + }, + { + "DefinationEncoding": "u", + "MacroVariableName": "arg4" + }, + { + "DefinationEncoding": "p", + "MacroVariableName": "arg5" + }, + { + "DefinationEncoding": "p", + "MacroVariableName": "arg6" + } + ], + "macroName": "QuicTraceLogVerbose" + }, + "XdpMapModeConfigured": { + "ModuleProperites": {}, + "TraceString": "[ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)", + "UniqueId": "XdpMapModeConfigured", + "splitArgs": [ + { + "DefinationEncoding": "p", + "MacroVariableName": "arg2" + }, + { + "DefinationEncoding": "u", + "MacroVariableName": "arg3" + }, + { + "DefinationEncoding": "p", + "MacroVariableName": "arg4" + } + ], + "macroName": "QuicTraceLogVerbose" + }, + "XdpMapModeInserted": { + "ModuleProperites": {}, + "TraceString": "[ixdp][%p] Map mode: inserted XSK for queue %u (IfIndex=%u, XskMap=%p, RxXsk=%p)", + "UniqueId": "XdpMapModeInserted", + "splitArgs": [ + { + "DefinationEncoding": "p", + "MacroVariableName": "arg2" + }, + { + "DefinationEncoding": "u", + "MacroVariableName": "arg3" + }, + { + "DefinationEncoding": "u", + "MacroVariableName": "arg4" + }, + { + "DefinationEncoding": "p", + "MacroVariableName": "arg5" + }, + { + "DefinationEncoding": "p", + "MacroVariableName": "arg6" + } + ], + "macroName": "QuicTraceLogVerbose" + }, "XdpPartitionShutdown": { "ModuleProperites": {}, "TraceString": "[ xdp][%p] XDP partition shutdown", @@ -13878,6 +13980,16 @@ "TraceID": "DatapathQueryUdpSendMsgFailedAsync", "EncodingString": "[data] Query for UDP_SEND_MSG_SIZE failed (async), 0x%x" }, + { + "UniquenessHash": "4001a3d6-c475-003a-33d3-0252c3c10417", + "TraceID": "DatapathRawInitFailRawOnly", + "EncodingString": "[ dp] Raw-only mode: raw datapath required but failed to initialize" + }, + { + "UniquenessHash": "38cb5c9d-e612-01a0-ec5f-6af064bb0724", + "TraceID": "DatapathRawMapInsertFail", + "EncodingString": "[ dp] XDP map mode: failed to insert XSK sockets into map, status:%d" + }, { "UniquenessHash": "3b8a506d-ede0-3c3b-7c31-83127acebfc6", "TraceID": "DatapathRecv", @@ -16398,6 +16510,11 @@ "TraceID": "SettingStreamMultiReceiveEnabled", "EncodingString": "[sett] StreamMultiReceiveEnabled = %hhu" }, + { + "UniquenessHash": "45180046-2197-4f5c-6e93-f5791314c245", + "TraceID": "SettingXdpDisabledInMapMode", + "EncodingString": "[ lib] Error: XdpEnabled cannot be set to FALSE when XDP map mode is active." + }, { "UniquenessHash": "3bdc4807-1d2f-01f2-3c8c-043542720899", "TraceID": "SettingXdpEnabled", @@ -16883,6 +17000,21 @@ "TraceID": "XdpInterfaceQueues", "EncodingString": "[ixdp][%p] Initializing %u queues on interface" }, + { + "UniquenessHash": "c69cd3d4-9f76-1bd6-efc3-ccc85d8b3f9d", + "TraceID": "XdpMapInsertFailed", + "EncodingString": "[ixdp][%p] XdpMapInsert failed for IfIndex=%u, QueueId=%u, XskMap=%p, RxXsk=%p" + }, + { + "UniquenessHash": "005c527e-db63-8dde-7829-b589b84080d9", + "TraceID": "XdpMapModeConfigured", + "EncodingString": "[ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)" + }, + { + "UniquenessHash": "ef9c8310-696c-a60e-7004-f26815bf4101", + "TraceID": "XdpMapModeInserted", + "EncodingString": "[ixdp][%p] Map mode: inserted XSK for queue %u (IfIndex=%u, XskMap=%p, RxXsk=%p)" + }, { "UniquenessHash": "18009e0d-e738-467e-35c0-55ebabcbf31f", "TraceID": "XdpPartitionShutdown", diff --git a/src/platform/datapath_raw.c b/src/platform/datapath_raw.c index 99d0ec67e1..b7746f659c 100644 --- a/src/platform/datapath_raw.c +++ b/src/platform/datapath_raw.c @@ -24,6 +24,7 @@ RawDataPathInitialize( _In_ uint32_t ClientRecvContextLength, _In_opt_ const CXPLAT_DATAPATH* ParentDataPath, _In_ CXPLAT_WORKER_POOL* WorkerPool, + _In_ CXPLAT_DATAPATH_INIT_CONFIG* InitConfig, _Outptr_result_maybenull_ CXPLAT_DATAPATH_RAW** NewDataPath ) { @@ -34,8 +35,8 @@ RawDataPathInitialize( *NewDataPath = NULL; - CXPLAT_DATAPATH_RAW* DataPath = CXPLAT_ALLOC_PAGED(DatapathSize, QUIC_POOL_DATAPATH); - if (DataPath == NULL) { + CXPLAT_DATAPATH_RAW* RawDataPath = CXPLAT_ALLOC_PAGED(DatapathSize, QUIC_POOL_DATAPATH); + if (RawDataPath == NULL) { QuicTraceEvent( AllocFailure, "Allocation of '%s' failed. (%llu bytes)", @@ -43,49 +44,84 @@ RawDataPathInitialize( DatapathSize); return; } - CxPlatZeroMemory(DataPath, DatapathSize); + CxPlatZeroMemory(RawDataPath, DatapathSize); CXPLAT_FRE_ASSERT(CxPlatWorkerPoolAddRef(WorkerPool, CXPLAT_WORKER_POOL_REF_RAW)); - DataPath->WorkerPool = WorkerPool; + RawDataPath->WorkerPool = WorkerPool; - if (!CxPlatSockPoolInitialize(&DataPath->SocketPool)) { + if (!CxPlatSockPoolInitialize(&RawDataPath->SocketPool)) { goto Error; } SockPoolInitialized = TRUE; - Status = CxPlatDpRawInitialize(DataPath, ClientRecvContextLength, WorkerPool); + Status = CxPlatDpRawInitialize(RawDataPath, ClientRecvContextLength, WorkerPool); if (QUIC_FAILED(Status)) { goto Error; } DpRawInitialized = TRUE; - Status = CxPlatDataPathRouteWorkerInitialize(DataPath); + Status = CxPlatDataPathRouteWorkerInitialize(RawDataPath); if (QUIC_FAILED(Status)) { goto Error; } - *NewDataPath = DataPath; - DataPath->ParentDataPath = ParentDataPath; - DataPath = NULL; + if (InitConfig->XdpMapConfigCount > 0) { + CXPLAT_DBG_ASSERT(InitConfig->XdpMapConfigs != NULL); + CxPlatDpRawEnableRawDatapathOnly(RawDataPath); + Status = + CxPlatDpRawInsertXskByMapConfigs( + RawDataPath, + InitConfig->XdpMapConfigs, + InitConfig->XdpMapConfigCount); + if (QUIC_FAILED(Status)) { + QuicTraceLogVerbose( + DatapathRawMapInsertFail, + "[ dp] XDP map mode: failed to insert XSK sockets into map, status:%d", + Status); + RawDataPathUninitialize(RawDataPath); + RawDataPath = NULL; + goto Error; + } + } + + *NewDataPath = RawDataPath; + RawDataPath->ParentDataPath = ParentDataPath; + RawDataPath = NULL; Error: - if (DataPath != NULL) { + if (RawDataPath != NULL) { #if DEBUG - DataPath->Uninitialized = TRUE; + RawDataPath->Uninitialized = TRUE; #endif if (DpRawInitialized) { - CxPlatDpRawUninitialize(DataPath); + CxPlatDpRawUninitialize(RawDataPath); } else { if (SockPoolInitialized) { - CxPlatSockPoolUninitialize(&DataPath->SocketPool); + CxPlatSockPoolUninitialize(&RawDataPath->SocketPool); } - CXPLAT_FREE(DataPath, QUIC_POOL_DATAPATH); + CXPLAT_FREE(RawDataPath, QUIC_POOL_DATAPATH); CxPlatWorkerPoolRelease(WorkerPool, CXPLAT_WORKER_POOL_REF_RAW); } } } +BOOLEAN +CxPlatDpRawIsRawDatapathOnly( + _In_opt_ const CXPLAT_DATAPATH_RAW* RawDataPath + ) +{ + return RawDataPath != NULL && RawDataPath->RawDatapathOnly; +} + +void +CxPlatDpRawEnableRawDatapathOnly( + _In_ CXPLAT_DATAPATH_RAW* RawDataPath + ) +{ + RawDataPath->RawDatapathOnly = TRUE; +} + _IRQL_requires_max_(PASSIVE_LEVEL) void RawDataPathUninitialize( diff --git a/src/platform/datapath_raw.h b/src/platform/datapath_raw.h index 8999779f7e..1dcce1da86 100644 --- a/src/platform/datapath_raw.h +++ b/src/platform/datapath_raw.h @@ -63,6 +63,7 @@ typedef struct CXPLAT_DATAPATH_RAW { BOOLEAN Freed : 1; #endif BOOLEAN ReserveAuxTcpSockForQtip; // Whether or not we create an auxiliary TCP socket. + BOOLEAN RawDatapathOnly; // Raw datapath is the only datapath; OS socket creation is skipped (e.g. XDP map mode, DPDK). } CXPLAT_DATAPATH_RAW; diff --git a/src/platform/datapath_raw_dummy.c b/src/platform/datapath_raw_dummy.c index 90e2a78c49..0ee1126448 100644 --- a/src/platform/datapath_raw_dummy.c +++ b/src/platform/datapath_raw_dummy.c @@ -56,12 +56,14 @@ RawDataPathInitialize( _In_ uint32_t ClientRecvContextLength, _In_opt_ const CXPLAT_DATAPATH* ParentDataPath, _In_ CXPLAT_WORKER_POOL* WorkerPool, + _In_ CXPLAT_DATAPATH_INIT_CONFIG* InitConfig, _Outptr_result_maybenull_ CXPLAT_DATAPATH_RAW** DataPath ) { UNREFERENCED_PARAMETER(ClientRecvContextLength); UNREFERENCED_PARAMETER(ParentDataPath); UNREFERENCED_PARAMETER(WorkerPool); + UNREFERENCED_PARAMETER(InitConfig); *DataPath = NULL; } @@ -74,6 +76,23 @@ RawDataPathUninitialize( UNREFERENCED_PARAMETER(Datapath); } +BOOLEAN +CxPlatDpRawIsRawDatapathOnly( + _In_opt_ const CXPLAT_DATAPATH_RAW* RawDataPath + ) +{ + UNREFERENCED_PARAMETER(RawDataPath); + return FALSE; // Dummy raw datapath is never raw-only. +} + +void +CxPlatDpRawEnableRawDatapathOnly( + _In_ CXPLAT_DATAPATH_RAW* RawDataPath + ) +{ + UNREFERENCED_PARAMETER(RawDataPath); +} + _IRQL_requires_max_(PASSIVE_LEVEL) void RawDataPathUpdatePollingIdleTimeout( @@ -85,6 +104,24 @@ RawDataPathUpdatePollingIdleTimeout( UNREFERENCED_PARAMETER(PollingIdleTimeoutUs); } +_IRQL_requires_max_(PASSIVE_LEVEL) +QUIC_STATUS +CxPlatDpRawInsertXskByMapConfigs( + _In_ CXPLAT_DATAPATH_RAW* RawDataPath, + _In_reads_(MapConfigCount) const QUIC_XDP_MAP_CONFIG* MapConfigs, + _In_ uint32_t MapConfigCount + ) +{ + // + // Stub for platforms without XDP support. Must return QUIC_STATUS_NOT_SUPPORTED + // so raw-only init fails gracefully on non-Windows platforms. + // + UNREFERENCED_PARAMETER(RawDataPath); + UNREFERENCED_PARAMETER(MapConfigs); + UNREFERENCED_PARAMETER(MapConfigCount); + return QUIC_STATUS_NOT_SUPPORTED; +} + _IRQL_requires_max_(DISPATCH_LEVEL) CXPLAT_DATAPATH_FEATURES RawDataPathGetSupportedFeatures( diff --git a/src/platform/datapath_raw_xdp_win.c b/src/platform/datapath_raw_xdp_win.c index c1c3e57278..7101570634 100644 --- a/src/platform/datapath_raw_xdp_win.c +++ b/src/platform/datapath_raw_xdp_win.c @@ -31,6 +31,7 @@ #define XDP_MAX_SYNC_WAIT_TIMEOUT_MS 1000 // Used for querying XDP RSS capabilities. + typedef struct XDP_DATAPATH { CXPLAT_DATAPATH_RAW; DECLSPEC_CACHEALIGN @@ -1467,6 +1468,14 @@ CxPlatDpRawPlumbRulesOnSocket( { QUIC_STATUS Status = QUIC_STATUS_SUCCESS; XDP_DATAPATH* Xdp = (XDP_DATAPATH*)Socket->RawDatapath; + + // + // In external map mode, the caller manages XDP rules; nothing to do here. + // + if (Xdp->RawDatapathOnly) { + return QUIC_STATUS_SUCCESS; + } + if (Socket->Wildcard) { XDP_RULE Rules[5] = {0}; uint8_t RulesSize = 0; @@ -2265,3 +2274,70 @@ CxPlatDataPathRssConfigFree( { CXPLAT_FREE(RssConfig, QUIC_POOL_DATAPATH_RSS_CONFIG); } + +_IRQL_requires_max_(PASSIVE_LEVEL) +static +QUIC_STATUS +CxPlatDpRawInsertXskInMap( + _In_ XDP_INTERFACE* Interface, + _In_ HANDLE XskMap + ) +{ + for (uint32_t j = 0; j < Interface->QueueCount; j++) { + CXPLAT_QUEUE* Queue = &Interface->Queues[j]; + if (Queue->RxXsk == NULL) { + continue; + } + HRESULT Hr = XdpMapInsert(XskMap, &j, &Queue->RxXsk); + if (FAILED(Hr)) { + QuicTraceLogVerbose( + XdpMapInsertFailed, + "[ixdp][%p] XdpMapInsert failed for IfIndex=%u, QueueId=%u, XskMap=%p, RxXsk=%p", + Interface, + Interface->IfIndex, + j, + XskMap, + Queue->RxXsk); + return (QUIC_STATUS)Hr; + } + } + return QUIC_STATUS_SUCCESS; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +QUIC_STATUS +CxPlatDpRawInsertXskByMapConfigs( + _In_ CXPLAT_DATAPATH_RAW* RawDataPath, + _In_reads_(MapConfigCount) const QUIC_XDP_MAP_CONFIG* MapConfigs, + _In_ uint32_t MapConfigCount + ) +{ + XDP_DATAPATH* Xdp = (XDP_DATAPATH*)RawDataPath; + QUIC_STATUS Status = QUIC_STATUS_SUCCESS; + + CXPLAT_LIST_ENTRY* Entry; + for (Entry = Xdp->Interfaces.Flink; Entry != &Xdp->Interfaces; Entry = Entry->Flink) { + XDP_INTERFACE* Interface = CONTAINING_RECORD(Entry, XDP_INTERFACE, Link); + + for (uint32_t i = 0; i < MapConfigCount; i++) { + if (MapConfigs[i].InterfaceIndex == Interface->IfIndex) { + HANDLE XskMap = (HANDLE)MapConfigs[i].MapHandle; + Status = CxPlatDpRawInsertXskInMap(Interface, XskMap); + if (QUIC_FAILED(Status)) { + goto Exit; + } + QuicTraceLogVerbose( + XdpMapModeConfigured, + "[ixdp][%p] Map mode configured for IfIndex=%u (MapHandle=%p)", + Interface, + Interface->IfIndex, + XskMap); + break; + } + } + } + +Exit: + + return Status; +} diff --git a/src/platform/datapath_winuser.c b/src/platform/datapath_winuser.c index 87ba64a8cc..04f3267205 100644 --- a/src/platform/datapath_winuser.c +++ b/src/platform/datapath_winuser.c @@ -1227,7 +1227,7 @@ SocketCreateUdp( int Result, Option; CXPLAT_DBG_ASSERT(Datapath->UdpHandlers.Receive != NULL || Config->Flags & CXPLAT_SOCKET_FLAG_PCP); - CXPLAT_DBG_ASSERT(IsServerSocket || Config->PartitionIndex < Datapath->PartitionCount); + CXPLAT_DBG_ASSERT(CxPlatDpRawIsRawDatapathOnly(Datapath->RawDataPath) || IsServerSocket || Config->PartitionIndex < Datapath->PartitionCount); CXPLAT_DBG_ASSERT(Config->CibirIdLength <= sizeof(Config->CibirId)); const uint32_t RawSocketLength = CxPlatGetRawSocketSize() + SocketCount * sizeof(CXPLAT_SOCKET_PROC); @@ -1277,6 +1277,46 @@ SocketCreateUdp( MAX_URO_PAYLOAD_LENGTH : Socket->Mtu - CXPLAT_MIN_IPV4_HEADER_SIZE - CXPLAT_UDP_HEADER_SIZE; + if (CxPlatDpRawIsRawDatapathOnly(Datapath->RawDataPath)) { + // + // There is no OS native datapath in raw-only mode. The application + // must specify the local port. Skip OS socket creation entirely and + // defer to the raw (XDP) datapath. + // + CXPLAT_DBG_ASSERT(Datapath->RawDataPath != NULL); + Socket->SkipCreatingOsSockets = TRUE; + CxPlatRefInitializeEx(&Socket->RefCount, 1); + if (Config->LocalAddress == NULL || Config->LocalAddress->Ipv4.sin_port == 0) { + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + (uint32_t)QUIC_STATUS_INVALID_PARAMETER, + "Raw-only datapath requires an explicit local port"); + Status = QUIC_STATUS_INVALID_PARAMETER; + goto Error; + } + // + // Client (connected) sockets are demuxed on local IP + port. In + // raw-only mode there is no OS bind to resolve a wildcard local IP + // into the concrete bound address, so a wildcard local IP would never + // match inbound packets and would silently black-hole all RX. Require + // a concrete local IP for connected sockets. Listeners legitimately + // stay wildcard and are demuxed on port alone. + // + if (!IsServerSocket && QuicAddrIsWildCard(Config->LocalAddress)) { + QuicTraceEvent( + DatapathErrorStatus, + "[data][%p] ERROR, %u, %s.", + Socket, + (uint32_t)QUIC_STATUS_INVALID_PARAMETER, + "Raw-only datapath requires an explicit local IP for connected sockets"); + Status = QUIC_STATUS_INVALID_PARAMETER; + goto Error; + } + goto Skip; + } + if (Socket->ReserveAuxTcpSockForQtip && !IsServerSocket) { // // QTIP clients will skip normal UDP socket reservation to use AuxSocket (TCP socket reservation) in raw socket. diff --git a/src/platform/datapath_xplat.c b/src/platform/datapath_xplat.c index f87cff50c6..f2dc40e4b8 100644 --- a/src/platform/datapath_xplat.c +++ b/src/platform/datapath_xplat.c @@ -32,29 +32,82 @@ CxPlatDataPathInitialize( goto Error; } - Status = - DataPathInitialize( + if (InitConfig->XdpMapConfigCount > 0) { + // + // Raw-only datapath: the raw datapath must initialize successfully + // since we are skipping OS platform specific initializations. + // + CXPLAT_DBG_ASSERT(InitConfig->XdpMapConfigs != NULL); + if (NewDataPath == NULL) { + return QUIC_STATUS_INVALID_PARAMETER; + } + if (UdpCallbacks != NULL) { + if (UdpCallbacks->Receive == NULL || UdpCallbacks->Unreachable == NULL) { + return QUIC_STATUS_INVALID_PARAMETER; + } + } + CXPLAT_DATAPATH* Datapath = + CXPLAT_ALLOC_PAGED(sizeof(CXPLAT_DATAPATH), QUIC_POOL_DATAPATH); + if (Datapath == NULL) { + QuicTraceEvent( + AllocFailure, + "Allocation of '%s' failed. (%llu bytes)", + "CXPLAT_DATAPATH (raw-only)", + sizeof(CXPLAT_DATAPATH)); + Status = QUIC_STATUS_OUT_OF_MEMORY; + goto Error; + } + CxPlatZeroMemory(Datapath, sizeof(CXPLAT_DATAPATH)); + if (UdpCallbacks) { + Datapath->UdpHandlers = *UdpCallbacks; + } + Datapath->WorkerPool = WorkerPool; + + RawDataPathInitialize( ClientRecvContextLength, - UdpCallbacks, - TcpCallbacks, + Datapath, WorkerPool, InitConfig, - NewDataPath); - if (QUIC_FAILED(Status)) { - QuicTraceLogVerbose( - DatapathInitFail, - "[ dp] Failed to initialize datapath, status:%d", Status); - goto Error; - } + &Datapath->RawDataPath); + if (Datapath->RawDataPath == NULL) { + QuicTraceLogVerbose( + DatapathRawInitFailRawOnly, + "[ dp] Raw-only mode: raw datapath required but failed to initialize"); + CXPLAT_FREE(Datapath, QUIC_POOL_DATAPATH); + *NewDataPath = NULL; + Status = QUIC_STATUS_NOT_SUPPORTED; + goto Error; + } + *NewDataPath = Datapath; + } else { + // + // OS platform-specific initializations. + // + Status = + DataPathInitialize( + ClientRecvContextLength, + UdpCallbacks, + TcpCallbacks, + WorkerPool, + InitConfig, + NewDataPath); + if (QUIC_FAILED(Status)) { + QuicTraceLogVerbose( + DatapathInitFail, + "[ dp] Failed to initialize datapath, status:%d", Status); + goto Error; + } - // - // Best effort try to initialize the raw datapath. - // - RawDataPathInitialize( - ClientRecvContextLength, - *NewDataPath, - WorkerPool, - &((*NewDataPath)->RawDataPath)); + // + // Best effort try to initialize the raw datapath. + // + RawDataPathInitialize( + ClientRecvContextLength, + *NewDataPath, + WorkerPool, + InitConfig, + &((*NewDataPath)->RawDataPath)); + } Error: @@ -67,10 +120,19 @@ CxPlatDataPathUninitialize( _In_ CXPLAT_DATAPATH* Datapath ) { + BOOLEAN IsRawDatapathOnly = CxPlatDpRawIsRawDatapathOnly(Datapath->RawDataPath); if (Datapath->RawDataPath) { RawDataPathUninitialize(Datapath->RawDataPath); } - DataPathUninitialize(Datapath); + if (IsRawDatapathOnly) { + // + // Raw-only mode: no platform (WinSock) datapath was initialized, + // so free directly without platform uninit. + // + CXPLAT_FREE(Datapath, QUIC_POOL_DATAPATH); + } else { + DataPathUninitialize(Datapath); + } } _IRQL_requires_max_(PASSIVE_LEVEL) @@ -125,7 +187,14 @@ CxPlatSocketCreateUdp( ) { QUIC_STATUS Status = QUIC_STATUS_SUCCESS; - BOOLEAN CreateRaw = Config->Flags & CXPLAT_SOCKET_FLAG_XDP; + BOOLEAN IsRawDatapathOnly = CxPlatDpRawIsRawDatapathOnly(Datapath->RawDataPath); + + // + // In raw-only mode the raw (XDP) datapath is the only data path. Implicitly + // enable XDP for all sockets and treat any raw socket failure as fatal + // (no OS fallback, no QTIP TCP port retry). + // + BOOLEAN CreateRaw = IsRawDatapathOnly || (Config->Flags & CXPLAT_SOCKET_FLAG_XDP); // // In a real production (XDP/QTIP+XDP) scenario, we never have to loop more than once @@ -151,6 +220,7 @@ CxPlatSocketCreateUdp( BOOLEAN CibirRequested = (Config->CibirIdLength > 0); (*NewSocket)->RawSocketAvailable = 0; + CXPLAT_DBG_ASSERT((IsRawDatapathOnly && CreateRaw && Datapath->RawDataPath) || !IsRawDatapathOnly); if (CreateRaw && Datapath->RawDataPath) { Status = RawSocketCreateUdp( @@ -162,6 +232,16 @@ CxPlatSocketCreateUdp( QuicTraceLogVerbose( RawSockCreateFail, "[sock] Failed to create raw socket, status:%d", Status); + + if (IsRawDatapathOnly) { + // + // Raw-only mode: no fallback allowed. + // + CxPlatSocketDelete(*NewSocket); + *NewSocket = NULL; + goto Error; + } + BOOLEAN IsServerSocket = !(*NewSocket)->HasFixedRemoteAddress; if (IsServerSocket && RequiresQtip) { // diff --git a/src/platform/platform_internal.h b/src/platform/platform_internal.h index f3e9e527d2..10cd324bd1 100644 --- a/src/platform/platform_internal.h +++ b/src/platform/platform_internal.h @@ -263,6 +263,7 @@ typedef struct CXPLAT_DATAPATH_PROC_CONTEXT { typedef struct CXPLAT_DATAPATH { CXPLAT_DATAPATH_COMMON; + // // The registration with WinSock Kernel. // @@ -473,6 +474,7 @@ typedef struct QUIC_CACHEALIGN CXPLAT_SOCKET_PROC { typedef struct CXPLAT_DATAPATH { CXPLAT_DATAPATH_COMMON; + // // Function pointer to AcceptEx. // @@ -991,6 +993,7 @@ typedef struct QUIC_CACHEALIGN CXPLAT_DATAPATH_PARTITION { typedef struct CXPLAT_DATAPATH { CXPLAT_DATAPATH_COMMON; + // // Synchronization mechanism for cleanup. // @@ -1203,6 +1206,7 @@ RawDataPathInitialize( _In_ uint32_t ClientRecvContextLength, _In_opt_ const CXPLAT_DATAPATH* ParentDataPath, _In_ CXPLAT_WORKER_POOL* WorkerPool, + _In_ CXPLAT_DATAPATH_INIT_CONFIG* InitConfig, _Outptr_result_maybenull_ CXPLAT_DATAPATH_RAW** DataPath ); @@ -1219,6 +1223,19 @@ RawDataPathUpdatePollingIdleTimeout( _In_ uint32_t PollingIdleTimeoutUs ); +// +// Inserts each interface's RX XSK sockets into the matching XSKMAP from the +// provided map configs. Uses an O(Interfaces * MapConfigCount) search to match +// each config to its interface by IfIndex. +// +_IRQL_requires_max_(PASSIVE_LEVEL) +QUIC_STATUS +CxPlatDpRawInsertXskByMapConfigs( + _In_ CXPLAT_DATAPATH_RAW* RawDataPath, + _In_reads_(MapConfigCount) const QUIC_XDP_MAP_CONFIG* MapConfigs, + _In_ uint32_t MapConfigCount + ); + _IRQL_requires_max_(DISPATCH_LEVEL) CXPLAT_DATAPATH_FEATURES RawDataPathGetSupportedFeatures( @@ -1231,6 +1248,16 @@ RawDataPathIsPaddingPreferred( _In_ CXPLAT_DATAPATH* Datapath ); +BOOLEAN +CxPlatDpRawIsRawDatapathOnly( + _In_opt_ const CXPLAT_DATAPATH_RAW* RawDataPath + ); + +void +CxPlatDpRawEnableRawDatapathOnly( + _In_ CXPLAT_DATAPATH_RAW* RawDataPath + ); + _IRQL_requires_max_(PASSIVE_LEVEL) QUIC_STATUS RawSocketUpdateQeo( diff --git a/src/platform/unittest/DataPathTest.cpp b/src/platform/unittest/DataPathTest.cpp index 134f8db040..cc59788799 100644 --- a/src/platform/unittest/DataPathTest.cpp +++ b/src/platform/unittest/DataPathTest.cpp @@ -1408,4 +1408,241 @@ TEST_F(DataPathTest, XdpRuleAddOomCleanup) } #endif // DEBUG +// +// XDP Map Mode Tests +// +// These tests exercise the XDP map mode initialization path in +// CxPlatDataPathInitialize (datapath_xplat.c). In map mode the platform +// (WinSock/epoll) datapath is bypassed and only the raw (XDP) datapath is +// used. Map mode is triggered when XdpMapConfigCount > 0 and +// XdpMapConfigs != NULL in the init config. +// + +TEST_F(DataPathTest, XdpMapMode_ZeroConfigUsesNormalPath) +{ + // + // XdpMapConfigCount == 0 should fall through to the normal (non-map) + // initialization path and succeed. + // + QUIC_GLOBAL_EXECUTION_CONFIG ExecConfig = { QUIC_GLOBAL_EXECUTION_CONFIG_FLAG_NONE, 0, 0, {0} }; + CXPLAT_WORKER_POOL* WorkerPool = + CxPlatWorkerPoolCreate(&ExecConfig, CXPLAT_WORKER_POOL_REF_TOOL); + ASSERT_NE(nullptr, WorkerPool); + + CXPLAT_DATAPATH_INIT_CONFIG InitConfig = {0}; + InitConfig.EnableDscpOnRecv = TRUE; + InitConfig.XdpMapConfigs = nullptr; + InitConfig.XdpMapConfigCount = 0; + + CXPLAT_DATAPATH* Datapath = nullptr; + QUIC_STATUS Status = + CxPlatDataPathInitialize( + 0, + &EmptyUdpCallbacks, + nullptr, + WorkerPool, + &InitConfig, + &Datapath); + VERIFY_QUIC_SUCCESS(Status); + ASSERT_NE(nullptr, Datapath); + + CxPlatDataPathUninitialize(Datapath); + CxPlatWorkerPoolDelete(WorkerPool, CXPLAT_WORKER_POOL_REF_TOOL); +} + + +TEST_F(DataPathTest, XdpMapMode_InitFailsWithoutRawDatapath) +{ + // + // On machines without XDP/DuoNic, raw datapath initialization fails. + // In map mode this is a hard failure (QUIC_STATUS_NOT_SUPPORTED) unlike + // the normal path where raw datapath failure is tolerated. Verify that + // the error is propagated cleanly and the output pointer is set to NULL. + // + if (UseDuoNic) { + GTEST_SKIP_NO_RETURN_("DuoNic is available; raw datapath will succeed"); + return; + } + + const uint32_t FakeIfIndex = 0xDEAD; + const QUIC_XDP_MAP_HANDLE FakeHandle = (QUIC_XDP_MAP_HANDLE)(intptr_t)-1; + QUIC_XDP_MAP_CONFIG MapConfig = { FakeIfIndex, FakeHandle }; + + QUIC_GLOBAL_EXECUTION_CONFIG ExecConfig = { QUIC_GLOBAL_EXECUTION_CONFIG_FLAG_NONE, 0, 0, {0} }; + CXPLAT_WORKER_POOL* WorkerPool = + CxPlatWorkerPoolCreate(&ExecConfig, CXPLAT_WORKER_POOL_REF_TOOL); + ASSERT_NE(nullptr, WorkerPool); + + CXPLAT_DATAPATH_INIT_CONFIG InitConfig = {0}; + InitConfig.EnableDscpOnRecv = TRUE; + InitConfig.XdpMapConfigs = &MapConfig; + InitConfig.XdpMapConfigCount = 1; + + CXPLAT_DATAPATH* Datapath = nullptr; + QUIC_STATUS Status = + CxPlatDataPathInitialize( + 0, + &EmptyUdpCallbacks, + nullptr, + WorkerPool, + &InitConfig, + &Datapath); + ASSERT_TRUE(QUIC_FAILED(Status)); + ASSERT_EQ(nullptr, Datapath); + + CxPlatWorkerPoolDelete(WorkerPool, CXPLAT_WORKER_POOL_REF_TOOL); +} + +TEST_F(DataPathTest, XdpMapMode_InitSucceedsWithNonMatchingIfIndex) +{ + // + // On DuoNic machines, raw datapath initialization succeeds. + // When the provided IfIndex doesn't match any real interface, + // CxPlatDpRawInsertXskByMapConfigs is a no-op and returns SUCCESS. + // The datapath should be created in XDP map mode. + // + if (!UseDuoNic) { + GTEST_SKIP_NO_RETURN_("Requires DuoNic/XDP for raw datapath init"); + return; + } + + const uint32_t FakeIfIndex = 0xDEAD; + const QUIC_XDP_MAP_HANDLE FakeHandle = (QUIC_XDP_MAP_HANDLE)(intptr_t)-1; + QUIC_XDP_MAP_CONFIG MapConfig = { FakeIfIndex, FakeHandle }; + + QUIC_GLOBAL_EXECUTION_CONFIG ExecConfig = { QUIC_GLOBAL_EXECUTION_CONFIG_FLAG_NONE, 0, 1, {0} }; + CXPLAT_WORKER_POOL* WorkerPool = + CxPlatWorkerPoolCreate(&ExecConfig, CXPLAT_WORKER_POOL_REF_TOOL); + ASSERT_NE(nullptr, WorkerPool); + + CXPLAT_DATAPATH_INIT_CONFIG InitConfig = {0}; + InitConfig.EnableDscpOnRecv = TRUE; + InitConfig.XdpMapConfigs = &MapConfig; + InitConfig.XdpMapConfigCount = 1; + + CXPLAT_DATAPATH* Datapath = nullptr; + QUIC_STATUS Status = + CxPlatDataPathInitialize( + 0, + &EmptyUdpCallbacks, + nullptr, + WorkerPool, + &InitConfig, + &Datapath); + VERIFY_QUIC_SUCCESS(Status); + ASSERT_NE(nullptr, Datapath); + + // + // Uninitialize should take the map-mode cleanup path (direct free, + // no platform uninit) without crashing. + // + CxPlatDataPathUninitialize(Datapath); + CxPlatWorkerPoolDelete(WorkerPool, CXPLAT_WORKER_POOL_REF_TOOL); +} + +TEST_F(DataPathTest, XdpMapMode_MultipleNonMatchingConfigs) +{ + // + // Multiple map configs with non-matching IfIndex values should all + // be silently skipped and initialization should succeed. + // + if (!UseDuoNic) { + GTEST_SKIP_NO_RETURN_("Requires DuoNic/XDP for raw datapath init"); + return; + } + + QUIC_XDP_MAP_CONFIG MapConfigs[3] = { + { 0xDEAD, (QUIC_XDP_MAP_HANDLE)(uintptr_t)0x1111 }, + { 0xBEEF, (QUIC_XDP_MAP_HANDLE)(uintptr_t)0x2222 }, + { 0xCAFE, (QUIC_XDP_MAP_HANDLE)(uintptr_t)0x3333 }, + }; + + QUIC_GLOBAL_EXECUTION_CONFIG ExecConfig = { QUIC_GLOBAL_EXECUTION_CONFIG_FLAG_NONE, 0, 1, {0} }; + CXPLAT_WORKER_POOL* WorkerPool = + CxPlatWorkerPoolCreate(&ExecConfig, CXPLAT_WORKER_POOL_REF_TOOL); + ASSERT_NE(nullptr, WorkerPool); + + CXPLAT_DATAPATH_INIT_CONFIG InitConfig = {0}; + InitConfig.EnableDscpOnRecv = TRUE; + InitConfig.XdpMapConfigs = MapConfigs; + InitConfig.XdpMapConfigCount = 3; + + CXPLAT_DATAPATH* Datapath = nullptr; + QUIC_STATUS Status = + CxPlatDataPathInitialize( + 0, + &EmptyUdpCallbacks, + nullptr, + WorkerPool, + &InitConfig, + &Datapath); + VERIFY_QUIC_SUCCESS(Status); + ASSERT_NE(nullptr, Datapath); + + CxPlatDataPathUninitialize(Datapath); + CxPlatWorkerPoolDelete(WorkerPool, CXPLAT_WORKER_POOL_REF_TOOL); +} + +TEST_F(DataPathTest, XdpMapMode_SocketSkipsRulePlumbing) +{ + // + // In XDP map mode, socket creation and deletion should skip XDP rule + // plumbing (CxPlatDpRawPlumbRulesOnSocket). Create a connected QTIP+XDP + // socket in map mode and verify that create and delete succeed without + // the rule plumbing step. + // + + if (!UseDuoNic) { + GTEST_SKIP_NO_RETURN_("Requires DuoNic/XDP for raw datapath init"); + return; + } + + const uint32_t FakeIfIndex = 0xDEAD; + const QUIC_XDP_MAP_HANDLE FakeHandle = (QUIC_XDP_MAP_HANDLE)(intptr_t)-1; + QUIC_XDP_MAP_CONFIG MapConfig = { FakeIfIndex, FakeHandle }; + + QUIC_GLOBAL_EXECUTION_CONFIG ExecConfig = { QUIC_GLOBAL_EXECUTION_CONFIG_FLAG_NONE, 0, 1, {0} }; + CXPLAT_WORKER_POOL* WorkerPool = + CxPlatWorkerPoolCreate(&ExecConfig, CXPLAT_WORKER_POOL_REF_TOOL); + ASSERT_NE(nullptr, WorkerPool); + + CXPLAT_DATAPATH_INIT_CONFIG InitConfig = {0}; + InitConfig.EnableDscpOnRecv = TRUE; + InitConfig.XdpMapConfigs = &MapConfig; + InitConfig.XdpMapConfigCount = 1; + + CXPLAT_DATAPATH* Datapath = nullptr; + QUIC_STATUS Status = + CxPlatDataPathInitialize( + 0, + &EmptyUdpCallbacks, + nullptr, + WorkerPool, + &InitConfig, + &Datapath); + VERIFY_QUIC_SUCCESS(Status); + ASSERT_NE(nullptr, Datapath); + + QuicAddr RemoteAddr = GetNewLocalIPv4(); + QuicAddr LocalAddr = GetNewLocalIPv4(); + + CXPLAT_UDP_CONFIG UdpConfig = {0}; + UdpConfig.RemoteAddress = &RemoteAddr.SockAddr; + UdpConfig.LocalAddress = &LocalAddr.SockAddr; + UdpConfig.Flags = CXPLAT_SOCKET_FLAG_XDP | CXPLAT_SOCKET_FLAG_QTIP; + + CXPLAT_SOCKET* Socket = nullptr; + Status = CxPlatSocketCreateUdp(Datapath, &UdpConfig, &Socket); + VERIFY_QUIC_SUCCESS(Status); + ASSERT_NE(nullptr, Socket); + + // + // Delete should also skip rule unplumbing in map mode. + // + CxPlatSocketDelete(Socket); + + CxPlatDataPathUninitialize(Datapath); + CxPlatWorkerPoolDelete(WorkerPool, CXPLAT_WORKER_POOL_REF_TOOL); +} + INSTANTIATE_TEST_SUITE_P(DataPathTest, DataPathTest, ::testing::Values(4, 6), testing::PrintToStringParamName()); diff --git a/src/test/MsQuicTests.h b/src/test/MsQuicTests.h index 7b361a923c..df04dba9b5 100644 --- a/src/test/MsQuicTests.h +++ b/src/test/MsQuicTests.h @@ -79,6 +79,13 @@ void QuicTestCloseConnBeforeStreamFlush(); void QuicTestGlobalParam(); #ifdef QUIC_API_ENABLE_PREVIEW_FEATURES void QuicTestXdpMapConfigParam(); +struct XdpMapModeArgs { + int Family; + uint16_t ServerPort; + uint16_t ClientPort; + bool UseCibir; +}; +void QuicTestXdpMapModeHandshake(const XdpMapModeArgs& Params); #endif void QuicTestCommonParam(); void QuicTestRegistrationParam(); diff --git a/src/test/bin/CMakeLists.txt b/src/test/bin/CMakeLists.txt index 3cfb210aff..7f1960d32d 100644 --- a/src/test/bin/CMakeLists.txt +++ b/src/test/bin/CMakeLists.txt @@ -6,10 +6,18 @@ set(SOURCES quic_gtest.h ) +if (WIN32) + list(APPEND SOURCES XdpMapModeHelpers.cpp XdpMapModeHelpers.h) +endif() + add_executable(msquictest ${SOURCES}) target_include_directories(msquictest PRIVATE ${PROJECT_SOURCE_DIR}/src/test) +if (WIN32) + target_include_directories(msquictest PRIVATE ${PROJECT_SOURCE_DIR}/submodules/xdp-for-windows/published/external) +endif() + set_property(TARGET msquictest PROPERTY FOLDER "${QUIC_FOLDER_PREFIX}tests") set_property(TARGET msquictest APPEND PROPERTY BUILD_RPATH "$ORIGIN") @@ -22,7 +30,7 @@ endif() target_link_libraries(msquictest inc gtest logging base_link) if (WIN32) - target_link_libraries(msquictest oldnames) + target_link_libraries(msquictest oldnames iphlpapi) endif() # At least /W3 must be used on all windows builds to pass compliance diff --git a/src/test/bin/XdpMapModeHelpers.cpp b/src/test/bin/XdpMapModeHelpers.cpp new file mode 100644 index 0000000000..276755b0c2 --- /dev/null +++ b/src/test/bin/XdpMapModeHelpers.cpp @@ -0,0 +1,343 @@ +/*++ + + Copyright (c) Microsoft Corporation. + Licensed under the MIT License. + +Abstract: + + Implementation of the XDP map mode rule configuration helpers. Keeping this + out of the test list file (quic_gtest.cpp) makes the per-test XDP rule setup + explicit at the start of each test rather than hidden inside a fixture. + +--*/ + +#include "quic_platform.h" +#include "MsQuicTests.h" +#include "msquichelper.h" +#undef min // gtest headers conflict with previous definitions of min/max. +#undef max +#include "gtest/gtest.h" +#include "XdpMapModeHelpers.h" + +#if defined(_WIN32) && defined(QUIC_API_ENABLE_PREVIEW_FEATURES) + +#include +#define XDP_API_VERSION 3 +#define XDP_INCLUDE_WINCOMMON +#include +#include + +extern const MsQuicApi* MsQuic; + +XdpMapModeState XdpMapState = {}; + +namespace { + +// +// CIBIR test constants matching the CIBIR id used by the map mode tests. +// Internal XDP rule format: just the id bytes, with offset computed as +// MsQuicLib.CidServerIdLength + 2. Default CidServerIdLength=0, so offset=2. +// +constexpr uint8_t CibirIdData[] = { 4, 3, 2, 1 }; +constexpr uint8_t CibirIdDataLength = sizeof(CibirIdData); +constexpr uint8_t CibirCidOffset = 2; // CidServerIdLength(0) + 2 + +} // namespace + +bool +DiscoverDuoNicInterfaces( + _Out_writes_(XDP_MAP_MODE_MAX_INTERFACES) uint32_t* IfIndices, + _Out_ uint32_t* Count + ) +{ + *Count = 0; + ULONG Flags = GAA_FLAG_INCLUDE_PREFIX | GAA_FLAG_SKIP_ANYCAST | + GAA_FLAG_SKIP_MULTICAST | GAA_FLAG_SKIP_DNS_SERVER; + ULONG BufSize = 0; + GetAdaptersAddresses(AF_INET, Flags, NULL, NULL, &BufSize); + if (BufSize == 0) return false; + + auto Adapters = (PIP_ADAPTER_ADDRESSES)malloc(BufSize); + if (!Adapters) return false; + + if (GetAdaptersAddresses(AF_INET, Flags, NULL, Adapters, &BufSize) != NO_ERROR) { + free(Adapters); + return false; + } + + for (auto Adapter = Adapters; Adapter && *Count < XDP_MAP_MODE_MAX_INTERFACES; Adapter = Adapter->Next) { + if (Adapter->IfType != IF_TYPE_ETHERNET_CSMACD || + Adapter->OperStatus != IfOperStatusUp) { + continue; + } + IN_ADDR DuoNicServer, DuoNicClient; + if (inet_pton(AF_INET, "192.168.1.11", &DuoNicServer) != 1 || + inet_pton(AF_INET, "192.168.1.12", &DuoNicClient) != 1) { + break; + } + for (auto Unicast = Adapter->FirstUnicastAddress; Unicast; Unicast = Unicast->Next) { + if (Unicast->Address.lpSockaddr->sa_family != AF_INET) continue; + auto* Sin = (SOCKADDR_IN*)Unicast->Address.lpSockaddr; + if (Sin->sin_addr.s_addr == DuoNicServer.s_addr || + Sin->sin_addr.s_addr == DuoNicClient.s_addr) { + IfIndices[*Count] = Adapter->IfIndex; + (*Count)++; + break; + } + } + } + free(Adapters); + return *Count > 0; +} + +void +XdpMapModeRuleScope::Setup(bool UseCibirParam, bool UseQtipParam) +{ + UseQtip = UseQtipParam; + + WSADATA WsaData; + ASSERT_EQ(WSAStartup(MAKEWORD(2, 2), &WsaData), 0); + WsaInitialized = true; + + // + // Reserve ports (server + client) with OS sockets that stay open. + // Always reserve both UDP and TCP to prevent port stealing. + // Retry if the OS-assigned UDP port collides with an existing TCP + // binding, since UDP and TCP port spaces are independent. + // + static const int MaxPortRetries = 1000; + for (int i = 0; i < PortCount; i++) { + bool PortReserved = false; + for (int Retry = 0; Retry < MaxPortRetries; Retry++) { + PortSocksUdp[i] = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + ASSERT_NE(PortSocksUdp[i], INVALID_SOCKET) + << "Failed to create UDP socket: WSAGetLastError=" << WSAGetLastError(); + struct sockaddr_in Addr = {}; + Addr.sin_family = AF_INET; + ASSERT_EQ(bind(PortSocksUdp[i], (struct sockaddr*)&Addr, sizeof(Addr)), 0); + int AddrLen = sizeof(Addr); + ASSERT_EQ(getsockname(PortSocksUdp[i], (struct sockaddr*)&Addr, &AddrLen), 0); + uint16_t Port = ntohs(Addr.sin_port); + ASSERT_NE(Port, (uint16_t)0); + + // + // Reserve the same port number on TCP. This prevents another + // process from binding a TCP socket to this port, which is + // required for QTIP (QUIC-over-TCP) tests. + // + PortSocksTcp[i] = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + ASSERT_NE(PortSocksTcp[i], INVALID_SOCKET) + << "Failed to create TCP socket: WSAGetLastError=" << WSAGetLastError(); + struct sockaddr_in TcpAddr = {}; + TcpAddr.sin_family = AF_INET; + TcpAddr.sin_port = htons(Port); + if (bind(PortSocksTcp[i], (struct sockaddr*)&TcpAddr, sizeof(TcpAddr)) == 0) { + if (i == 0) ServerPort = Port; else ClientPort = Port; + PortReserved = true; + break; + } + // + // TCP port collision. Close both sockets and retry with a + // new OS-assigned port. + // + printf("XDP Map Mode: port %u TCP collision (attempt %d/%d), retrying\n", + Port, Retry + 1, MaxPortRetries); + closesocket(PortSocksTcp[i]); + PortSocksTcp[i] = INVALID_SOCKET; + closesocket(PortSocksUdp[i]); + PortSocksUdp[i] = INVALID_SOCKET; + } + ASSERT_TRUE(PortReserved) + << "Failed to reserve a UDP+TCP port pair after " + << MaxPortRetries << " attempts"; + } + + printf("XDP Map Mode [%s]: ports Server=%u Client=%u CIBIR=%d QTIP=%d\n", + ::testing::UnitTest::GetInstance()->current_test_info()->name(), + ServerPort, ClientPort, UseCibirParam, UseQtipParam); + + // + // If QTIP is needed, enable it globally before creating XDP programs. + // + if (UseQtip) { + MsQuicSettings Settings; + uint32_t SettingsSize = sizeof(Settings); + MsQuic->GetParam(nullptr, QUIC_PARAM_GLOBAL_SETTINGS, &SettingsSize, &Settings); + PreviousQtipSetting = Settings.QTIPEnabled; + if (!PreviousQtipSetting) { + Settings.SetQtipEnabled(true); + ASSERT_TRUE(QUIC_SUCCEEDED(Settings.SetGlobal())); + } + } + + // + // Build XDP rules based on CIBIR/QTIP mode. + // + // Server (listener/wildcard) rules: + // - No CIBIR: XDP_MATCH_UDP_DST (+ XDP_MATCH_TCP_DST if QTIP) + // - CIBIR: QUIC_FLOW_SRC_CID + QUIC_FLOW_DST_CID + // (+ TCP_QUIC_FLOW_SRC_CID + TCP_QUIC_FLOW_DST_CID + TCP_CONTROL_DST if QTIP) + // + // Client (non-wildcard) rules: + // - Always: XDP_MATCH_UDP_DST (no QTIP) or XDP_MATCH_TCP_DST (QTIP) + // - CIBIR does NOT change client XDP matching. + // + static const XDP_HOOK_ID RxHook = { + XDP_HOOK_L2, + XDP_HOOK_RX, + XDP_HOOK_INSPECT, + }; + + for (uint32_t i = 0; i < XdpMapState.InterfaceCount; i++) { + for (uint32_t q = 0; q < XDP_MAP_MODE_MAX_QUEUES; q++) { + XDP_RULE Rules[8] = {}; + uint8_t RulesSize = 0; + + // + // Server port rules. + // + if (UseCibirParam) { + Rules[RulesSize].Match = XDP_MATCH_QUIC_FLOW_SRC_CID; + Rules[RulesSize].Pattern.QuicFlow.UdpPort = htons(ServerPort); + Rules[RulesSize].Pattern.QuicFlow.CidLength = CibirIdDataLength; + Rules[RulesSize].Pattern.QuicFlow.CidOffset = CibirCidOffset; + memcpy(Rules[RulesSize].Pattern.QuicFlow.CidData, CibirIdData, CibirIdDataLength); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + + Rules[RulesSize].Match = XDP_MATCH_QUIC_FLOW_DST_CID; + Rules[RulesSize].Pattern.QuicFlow.UdpPort = htons(ServerPort); + Rules[RulesSize].Pattern.QuicFlow.CidLength = CibirIdDataLength; + Rules[RulesSize].Pattern.QuicFlow.CidOffset = CibirCidOffset; + memcpy(Rules[RulesSize].Pattern.QuicFlow.CidData, CibirIdData, CibirIdDataLength); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + + if (UseQtip) { + Rules[RulesSize].Match = XDP_MATCH_TCP_QUIC_FLOW_SRC_CID; + Rules[RulesSize].Pattern.QuicFlow.UdpPort = htons(ServerPort); + Rules[RulesSize].Pattern.QuicFlow.CidLength = CibirIdDataLength; + Rules[RulesSize].Pattern.QuicFlow.CidOffset = CibirCidOffset; + memcpy(Rules[RulesSize].Pattern.QuicFlow.CidData, CibirIdData, CibirIdDataLength); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + + Rules[RulesSize].Match = XDP_MATCH_TCP_QUIC_FLOW_DST_CID; + Rules[RulesSize].Pattern.QuicFlow.UdpPort = htons(ServerPort); + Rules[RulesSize].Pattern.QuicFlow.CidLength = CibirIdDataLength; + Rules[RulesSize].Pattern.QuicFlow.CidOffset = CibirCidOffset; + memcpy(Rules[RulesSize].Pattern.QuicFlow.CidData, CibirIdData, CibirIdDataLength); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + + Rules[RulesSize].Match = XDP_MATCH_TCP_CONTROL_DST; + Rules[RulesSize].Pattern.Port = htons(ServerPort); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + } + } else { + Rules[RulesSize].Match = XDP_MATCH_UDP_DST; + Rules[RulesSize].Pattern.Port = htons(ServerPort); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + + if (UseQtip) { + Rules[RulesSize].Match = XDP_MATCH_TCP_DST; + Rules[RulesSize].Pattern.Port = htons(ServerPort); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + } + } + + // + // Client port rules. CIBIR does not change client XDP matching. + // QTIP clients use TCP-only; non-QTIP clients use UDP-only. + // + if (UseQtip) { + Rules[RulesSize].Match = XDP_MATCH_TCP_DST; + Rules[RulesSize].Pattern.Port = htons(ClientPort); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + } else { + Rules[RulesSize].Match = XDP_MATCH_UDP_DST; + Rules[RulesSize].Pattern.Port = htons(ClientPort); + Rules[RulesSize].Action = XDP_PROGRAM_ACTION_REDIRECT; + Rules[RulesSize].Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSKMAP_BY_QUEUEID; + Rules[RulesSize].Redirect.Target = XdpMapState.XskMaps[i]; + RulesSize++; + } + + HRESULT Hr = XdpCreateProgram( + XdpMapState.IfIndices[i], + &RxHook, + q, + XDP_CREATE_PROGRAM_FLAG_NONE, + Rules, + RulesSize, + &XdpPrograms[i][q]); + if (FAILED(Hr)) { + // + // No more queues on this interface. + // + QueueCounts[i] = q; + break; + } + } + printf("XDP Map Mode: IfIndex=%u created %u per-queue programs\n", + XdpMapState.IfIndices[i], QueueCounts[i]); + ASSERT_GT(QueueCounts[i], (uint32_t)0) + << "Failed to create any XDP programs for IfIndex=" + << XdpMapState.IfIndices[i]; + } +} + +XdpMapModeRuleScope::~XdpMapModeRuleScope() +{ + for (uint32_t i = 0; i < XdpMapState.InterfaceCount; i++) { + for (uint32_t q = 0; q < QueueCounts[i]; q++) { + if (XdpPrograms[i][q]) { + CloseHandle(XdpPrograms[i][q]); + XdpPrograms[i][q] = nullptr; + } + } + } + for (int i = 0; i < PortCount; i++) { + if (PortSocksUdp[i] != INVALID_SOCKET) { + closesocket(PortSocksUdp[i]); + PortSocksUdp[i] = INVALID_SOCKET; + } + if (PortSocksTcp[i] != INVALID_SOCKET) { + closesocket(PortSocksTcp[i]); + PortSocksTcp[i] = INVALID_SOCKET; + } + } + // + // Restore QTIP global setting. + // + if (UseQtip && !PreviousQtipSetting) { + MsQuicSettings Settings; + Settings.SetQtipEnabled(false); + Settings.SetGlobal(); + } + if (WsaInitialized) { + WSACleanup(); + } +} + +#endif // _WIN32 && QUIC_API_ENABLE_PREVIEW_FEATURES diff --git a/src/test/bin/XdpMapModeHelpers.h b/src/test/bin/XdpMapModeHelpers.h new file mode 100644 index 0000000000..bbacabaada --- /dev/null +++ b/src/test/bin/XdpMapModeHelpers.h @@ -0,0 +1,76 @@ +/*++ + + Copyright (c) Microsoft Corporation. + Licensed under the MIT License. + +Abstract: + + Helpers for configuring the XDP redirect rules used by the XDP map mode + tests. The rule setup is intentionally kept out of the test list file and is + invoked explicitly from the start of each test through an RAII scope, so the + important per-test setup stays visible in the test body instead of being + hidden in a fixture. + +--*/ + +#pragma once + +#if defined(_WIN32) && defined(QUIC_API_ENABLE_PREVIEW_FEATURES) + +#define XDP_MAP_MODE_MAX_INTERFACES 2 +#define XDP_MAP_MODE_MAX_QUEUES 64 + +struct XdpMapModeState { + uint32_t InterfaceCount; + uint32_t IfIndices[XDP_MAP_MODE_MAX_INTERFACES]; + HANDLE XskMaps[XDP_MAP_MODE_MAX_INTERFACES]; +}; + +extern XdpMapModeState XdpMapState; + +// +// Discover DuoNic interface indices by enumerating Ethernet adapters with +// known DuoNic IPv4 addresses (192.168.1.11 and 192.168.1.12). +// +bool +DiscoverDuoNicInterfaces( + _Out_writes_(XDP_MAP_MODE_MAX_INTERFACES) uint32_t* IfIndices, + _Out_ uint32_t* Count + ); + +// +// RAII helper that reserves the server/client ports and installs the per-queue +// XDP redirect programs for a single XDP map mode test. Construct it at the +// start of the test body and call Setup() via ASSERT_NO_FATAL_FAILURE(...) so +// the setup is visible; the destructor tears everything back down. +// +class XdpMapModeRuleScope { +public: + XdpMapModeRuleScope() = default; + ~XdpMapModeRuleScope(); + + XdpMapModeRuleScope(const XdpMapModeRuleScope&) = delete; + XdpMapModeRuleScope& operator=(const XdpMapModeRuleScope&) = delete; + + // + // Reserves the ports and creates the XDP programs for the given mode. + // + void Setup(bool UseCibir, bool UseQtip); + + uint16_t GetServerPort() const { return ServerPort; } + uint16_t GetClientPort() const { return ClientPort; } + +private: + static constexpr int PortCount = 2; // server + client + SOCKET PortSocksUdp[PortCount] = {INVALID_SOCKET, INVALID_SOCKET}; + SOCKET PortSocksTcp[PortCount] = {INVALID_SOCKET, INVALID_SOCKET}; + uint16_t ServerPort = 0; + uint16_t ClientPort = 0; + HANDLE XdpPrograms[XDP_MAP_MODE_MAX_INTERFACES][XDP_MAP_MODE_MAX_QUEUES] = {}; + uint32_t QueueCounts[XDP_MAP_MODE_MAX_INTERFACES] = {}; + bool WsaInitialized = false; + bool UseQtip = false; + bool PreviousQtipSetting = false; +}; + +#endif // _WIN32 && QUIC_API_ENABLE_PREVIEW_FEATURES diff --git a/src/test/bin/quic_gtest.cpp b/src/test/bin/quic_gtest.cpp index 3665957166..13f9e9cf80 100644 --- a/src/test/bin/quic_gtest.cpp +++ b/src/test/bin/quic_gtest.cpp @@ -13,6 +13,15 @@ #include #include +#include + +#if defined(_WIN32) && defined(QUIC_API_ENABLE_PREVIEW_FEATURES) +#define XDP_API_VERSION 3 +#define XDP_INCLUDE_WINCOMMON +#include +#include +#include "XdpMapModeHelpers.h" +#endif #ifdef QUIC_TEST_DATAPATH_HOOKS_ENABLED #pragma message("Test compiled with datapath hooks enabled") @@ -25,6 +34,7 @@ bool TestingKernelMode = false; bool PrivateTestLibrary = false; bool UseDuoNic = false; +bool UseXdpMapMode = false; CXPLAT_WORKER_POOL* WorkerPool; #if defined(QUIC_API_ENABLE_PREVIEW_FEATURES) bool UseQTIP = false; @@ -123,7 +133,40 @@ class QuicTestEnvironment : public ::testing::Environment { Settings.SetQtipEnabled(true); ASSERT_TRUE(QUIC_SUCCEEDED(Settings.SetGlobal())); } -#endif +#if defined(_WIN32) + if (UseXdpMapMode) { + // + // Discover DuoNic interfaces and create XSKMAPs. + // + ASSERT_TRUE( + DiscoverDuoNicInterfaces( + XdpMapState.IfIndices, + &XdpMapState.InterfaceCount)); + printf("XDP Map Mode: discovered %u DuoNic interface(s)\n", + XdpMapState.InterfaceCount); + + QUIC_XDP_MAP_CONFIG MapConfigs[XDP_MAP_MODE_MAX_INTERFACES]; + for (uint32_t i = 0; i < XdpMapState.InterfaceCount; i++) { + ASSERT_TRUE(SUCCEEDED( + XdpMapCreate(&XdpMapState.XskMaps[i], XDP_MAP_TYPE_XSKMAP))); + MapConfigs[i].InterfaceIndex = XdpMapState.IfIndices[i]; + MapConfigs[i].MapHandle = (QUIC_XDP_MAP_HANDLE)XdpMapState.XskMaps[i]; + printf(" IfIndex=%u, XskMap=%p\n", + XdpMapState.IfIndices[i], XdpMapState.XskMaps[i]); + } + + // + // Set the XDP map config before any registration is opened. + // This must happen before LazyInitComplete. + // + ASSERT_TRUE(QUIC_SUCCEEDED(MsQuic->SetParam( + nullptr, + QUIC_PARAM_GLOBAL_XDP_MAP_CONFIG, + XdpMapState.InterfaceCount * sizeof(QUIC_XDP_MAP_CONFIG), + MapConfigs))); + } +#endif // _WIN32 +#endif // QUIC_API_ENABLE_PREVIEW_FEATURES // // Enable DSCP on the receive path. This is needed to test DSCP Send path. // @@ -158,6 +201,16 @@ class QuicTestEnvironment : public ::testing::Environment { QuicTestUninitialize(); delete MsQuic; } +#if defined(_WIN32) && defined(QUIC_API_ENABLE_PREVIEW_FEATURES) + if (UseXdpMapMode) { + for (uint32_t i = 0; i < XdpMapState.InterfaceCount; i++) { + if (XdpMapState.XskMaps[i]) { + CloseHandle(XdpMapState.XskMaps[i]); + XdpMapState.XskMaps[i] = nullptr; + } + } + } +#endif CxPlatFreeSelfSignedCert(SelfSignedCertParams); CxPlatFreeSelfSignedCert(ClientCertParams); @@ -3101,6 +3154,68 @@ INSTANTIATE_TEST_SUITE_P( WithFamilyArgs, ::testing::ValuesIn(WithFamilyArgs::Generate())); +#if defined(_WIN32) && defined(QUIC_API_ENABLE_PREVIEW_FEATURES) +// +// XDP Map Mode tests. These only run when --xdpMapMode is passed. +// Each test reserves its own ports (keeping OS sockets open to prevent +// reuse), creates XDP programs for those ports, and tears them down. +// +// Test matrix: {IPv4, IPv6} x {no CIBIR, CIBIR} x {no QTIP, QTIP} +// + +struct XdpMapModeParams { + int Family; + bool UseCibir; + bool UseQtip; +}; + +std::ostream& operator << (std::ostream& o, const XdpMapModeParams& p) { + o << (p.Family == 4 ? "v4" : "v6"); + if (p.UseCibir) o << "/CIBIR"; + if (p.UseQtip) o << "/QTIP"; + if (!p.UseCibir && !p.UseQtip) o << "/Plain"; + return o; +} + +// +// Struct needed for GTEST parameterization. +// +struct XdpMapMode : public ::testing::TestWithParam { +}; + +TEST_P(XdpMapMode, Handshake) { + if (!UseXdpMapMode) { + GTEST_SKIP() << "XDP Map Mode not enabled (use --xdpMapMode)"; + } + auto Params = GetParam(); + XdpMapModeRuleScope Scope; + ASSERT_NO_FATAL_FAILURE(Scope.Setup(Params.UseCibir, Params.UseQtip)); + TestLogger Logger("QuicTestXdpMapModeHandshake"); + QuicTestXdpMapModeHandshake( + { Params.Family, Scope.GetServerPort(), Scope.GetClientPort(), Params.UseCibir }); +} + +static std::vector GenerateXdpMapModeParams() { + std::vector list; + for (int Family : { 4, 6 }) + for (bool UseCibir : { false, true }) + for (bool UseQtip : { false, true }) + list.push_back({ Family, UseCibir, UseQtip }); + return list; +} + +INSTANTIATE_TEST_SUITE_P( + XdpMapMode, + XdpMapMode, + ::testing::ValuesIn(GenerateXdpMapModeParams()), + [](const ::testing::TestParamInfo& info) { + std::string name = (info.param.Family == 4 ? "v4" : "v6"); + name += info.param.UseCibir ? "_CIBIR" : "_NoCIBIR"; + name += info.param.UseQtip ? "_QTIP" : "_NoQTIP"; + return name; + }); +#endif // _WIN32 && QUIC_API_ENABLE_PREVIEW_FEATURES + int main(int argc, char** argv) { #ifdef _WIN32 // @@ -3124,6 +3239,14 @@ int main(int argc, char** argv) { } } else if (strcmp("--duoNic", argv[i]) == 0) { UseDuoNic = true; + } else if (strcmp("--xdpMapMode", argv[i]) == 0) { +#if defined(_WIN32) && defined(QUIC_API_ENABLE_PREVIEW_FEATURES) + UseXdpMapMode = true; + UseDuoNic = true; // Map mode implies DuoNic +#else + printf("XDP Map Mode is only supported on Windows with preview features.\n"); + return -1; +#endif } else if (strcmp("--useQTIP", argv[i]) == 0) { #if defined(QUIC_API_ENABLE_PREVIEW_FEATURES) UseQTIP = true; diff --git a/src/test/lib/ApiTest.cpp b/src/test/lib/ApiTest.cpp index f34e8cc1a0..ca977ce8ee 100644 --- a/src/test/lib/ApiTest.cpp +++ b/src/test/lib/ApiTest.cpp @@ -3177,9 +3177,16 @@ void QuicTestXdpMapConfigParam() // // Setting after a registration is created should fail. + // Clear fake configs first so lazy init does not try to use them. // { TestScopeLogger LogScope1("SetParam after registration fails"); + TEST_QUIC_SUCCEEDED( + MsQuic->SetParam( + nullptr, + QUIC_PARAM_GLOBAL_XDP_MAP_CONFIG, + 0, + nullptr)); MsQuicRegistration Registration(true); TEST_TRUE(Registration.IsValid()); QUIC_XDP_MAP_CONFIG Config = { FakeIfIndex1, FakeHandle1 }; diff --git a/src/test/lib/HandshakeTest.cpp b/src/test/lib/HandshakeTest.cpp index 9da03a0fec..8f9ccaa469 100644 --- a/src/test/lib/HandshakeTest.cpp +++ b/src/test/lib/HandshakeTest.cpp @@ -4915,4 +4915,93 @@ QuicTestConnectionPoolCreate( } } +void +QuicTestXdpMapModeHandshake( + const XdpMapModeArgs& Params + ) +{ + QUIC_ADDRESS_FAMILY QuicAddrFamily = + (Params.Family == 4) ? QUIC_ADDRESS_FAMILY_INET : QUIC_ADDRESS_FAMILY_INET6; + + MsQuicRegistration Registration; + TEST_TRUE(Registration.IsValid()); + + MsQuicAlpn Alpn("MsQuicTest"); + + MsQuicSettings Settings; + Settings.SetPeerBidiStreamCount(1); + Settings.SetIdleTimeoutMs(10000); + + MsQuicConfiguration ServerConfiguration(Registration, Alpn, Settings, ServerSelfSignedCredConfig); + TEST_TRUE(ServerConfiguration.IsValid()); + + MsQuicCredentialConfig ClientCredConfig; + MsQuicConfiguration ClientConfiguration(Registration, Alpn, Settings, ClientCredConfig); + TEST_TRUE(ClientConfiguration.IsValid()); + + // + // CIBIR ID in API format: {offset, id_byte0, ...} + // + const uint8_t CibirId[] = { 0 /* offset */, 4, 3, 2, 1 }; + const uint8_t CibirIdLength = sizeof(CibirId); + + TestListener Listener(Registration, ListenerAcceptConnection, ServerConfiguration); + TEST_TRUE(Listener.IsValid()); + + if (Params.UseCibir) { + TEST_QUIC_SUCCEEDED(Listener.SetCibirId(CibirId, CibirIdLength)); + } + + QuicAddr ServerLocalAddr(QuicAddrFamily); + QuicAddrSetPort(&ServerLocalAddr.SockAddr, Params.ServerPort); + TEST_QUIC_SUCCEEDED(Listener.Start(Alpn, &ServerLocalAddr.SockAddr)); + TEST_QUIC_SUCCEEDED(Listener.GetLocalAddr(ServerLocalAddr)); + + UniquePtr Server; + ServerAcceptContext ServerAcceptCtx((TestConnection**)&Server); + Listener.Context = &ServerAcceptCtx; + + TestConnection Client(Registration); + TEST_TRUE(Client.IsValid()); + + if (Params.UseCibir) { + TEST_QUIC_SUCCEEDED(Client.SetShareUdpBinding(true)); + TEST_QUIC_SUCCEEDED( + MsQuic->SetParam( + Client.GetConnection(), + QUIC_PARAM_CONN_CIBIR_ID, + CibirIdLength, + CibirId)); + } + + QuicAddr ClientLocalAddr(QuicAddrFamily); + QuicAddrSetToDuoNicClient(&ClientLocalAddr.SockAddr); + QuicAddrSetPort(&ClientLocalAddr.SockAddr, Params.ClientPort); + TEST_QUIC_SUCCEEDED(Client.SetLocalAddr(ClientLocalAddr)); + + QuicAddr RemoteAddr{QuicAddrGetFamily(&ServerLocalAddr.SockAddr), ServerLocalAddr.GetPort()}; + QuicAddrSetToDuoNic(&RemoteAddr.SockAddr); + TEST_QUIC_SUCCEEDED(Client.SetRemoteAddr(RemoteAddr)); + + TEST_QUIC_SUCCEEDED( + Client.Start( + ClientConfiguration, + QuicAddrFamily, + QUIC_LOCALHOST_FOR_AF(QuicAddrFamily), + ServerLocalAddr.GetPort())); + + if (!Client.WaitForConnectionComplete()) { + return; + } + TEST_TRUE(Client.GetIsConnected()); + + TEST_NOT_EQUAL(nullptr, Server); + if (!Server->WaitForConnectionComplete()) { + return; + } + TEST_TRUE(Server->GetIsConnected()); + + Client.Shutdown(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0); +} + #endif // QUIC_API_ENABLE_PREVIEW_FEATURES diff --git a/src/test/lib/TestHelpers.h b/src/test/lib/TestHelpers.h index 0c30532627..b99b3c9595 100644 --- a/src/test/lib/TestHelpers.h +++ b/src/test/lib/TestHelpers.h @@ -16,6 +16,7 @@ #endif extern bool UseDuoNic; +extern bool UseXdpMapMode; // // Connect to the duonic address (if using duonic) or localhost (if not). diff --git a/src/test/lib/TestListener.cpp b/src/test/lib/TestListener.cpp index cc254437bd..bdaa12b0b0 100644 --- a/src/test/lib/TestListener.cpp +++ b/src/test/lib/TestListener.cpp @@ -100,6 +100,22 @@ TestListener::GetStatistics( &stats); } +#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES +QUIC_STATUS +TestListener::SetCibirId( + _In_reads_(Length) const uint8_t* CibirId, + _In_ uint8_t Length + ) +{ + return + MsQuic->SetParam( + QuicListener, + QUIC_PARAM_LISTENER_CIBIR_ID, + Length, + CibirId); +} +#endif // QUIC_API_ENABLE_PREVIEW_FEATURES + QUIC_STATUS TestListener::HandleListenerEvent( _Inout_ QUIC_LISTENER_EVENT* Event diff --git a/src/test/lib/TestListener.h b/src/test/lib/TestListener.h index 6842612a1a..61b7145391 100644 --- a/src/test/lib/TestListener.h +++ b/src/test/lib/TestListener.h @@ -106,6 +106,10 @@ class TestListener QUIC_STATUS GetLocalAddr(_Out_ QuicAddr &localAddr); QUIC_STATUS GetStatistics(_Out_ QUIC_LISTENER_STATISTICS &stats); +#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES + QUIC_STATUS SetCibirId(_In_reads_(Length) const uint8_t* CibirId, _In_ uint8_t Length); +#endif + bool GetHasRandomLoss() const { return HasRandomLoss; } void SetHasRandomLoss(bool Value) { HasRandomLoss = Value; } }; diff --git a/src/tools/recvfuzz/recvfuzz.cpp b/src/tools/recvfuzz/recvfuzz.cpp index 03b1cf1577..f5acc57dc8 100644 --- a/src/tools/recvfuzz/recvfuzz.cpp +++ b/src/tools/recvfuzz/recvfuzz.cpp @@ -1660,7 +1660,7 @@ void SetupAndFuzz() { CxPlatSystemLoad(); CxPlatInitialize(); - CXPLAT_DATAPATH* Datapath; + CXPLAT_DATAPATH* Datapath = NULL; const CXPLAT_UDP_DATAPATH_CALLBACKS DatapathCallbacks = { UdpRecvCallback, UdpUnreachCallback,