diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af5cdb81..0ca3c663 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,6 +423,59 @@ jobs: shell: PowerShell run: tools/check-drivers.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose + xskfwdkm_test: + name: XskFwdKm Test + needs: build + strategy: + fail-fast: false + matrix: + windows: [2019, 2022, Prerelease] + configuration: [Release, Debug] + platform: [x64, arm64] + exclude: + - windows: 2019 + platform: arm64 + - windows: 2022 + platform: arm64 + runs-on: + - self-hosted + - "1ES.Pool=xdp-ci-functional${{ matrix.platform != 'x64' && format('-{0}', matrix.platform) || '' }}-gh" + - "1ES.ImageOverride=WS${{ matrix.windows }}${{ matrix.platform != 'x64' && format('-{0}', matrix.platform) || '' }}-Functional" + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + sparse-checkout: tools + - name: Check Drivers + shell: PowerShell + run: tools/check-drivers.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose + - name: Prepare Machine + shell: PowerShell + run: tools/prepare-machine.ps1 -Platform ${{ matrix.platform }} -ForFunctionalTest -RequireNoReboot -Verbose + - name: Download Artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 + with: + name: bin_${{ matrix.configuration }}_${{ matrix.platform }} + path: artifacts/bin + - name: Run XskFwdKm + shell: PowerShell + run: tools/xskfwdkm.ps1 -Verbose -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} + - name: Convert Logs + if: ${{ always() }} + timeout-minutes: 15 + shell: PowerShell + run: tools/log.ps1 -Convert -Name xskfwdkm -Verbose -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} + - name: Upload Logs + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + if: ${{ always() }} + with: + name: logs_xskfwdkm_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }} + path: artifacts/logs + - name: Check Drivers + if: ${{ always() }} + shell: PowerShell + run: tools/check-drivers.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose + create_artifacts: name: Create Release Artifacts needs: build @@ -506,7 +559,7 @@ jobs: Complete: name: Complete if: always() - needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, perf_tests, downlevel_functional_tests, create_artifacts, etw] + needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, perf_tests, xskfwdkm_test, downlevel_functional_tests, create_artifacts, etw] runs-on: ubuntu-latest permissions: {} # No need for any permissions. steps: diff --git a/published/external/xdp/details/afxdp.h b/published/external/xdp/details/afxdp.h index e82844da..78d093ab 100644 --- a/published/external/xdp/details/afxdp.h +++ b/published/external/xdp/details/afxdp.h @@ -28,7 +28,7 @@ _XskCreateVersion( _XdpInitializeEaVersion(XDP_OBJECT_TYPE_XSK, ApiVersion, EaBuffer, sizeof(EaBuffer)); Res = _XdpOpen(Socket, FILE_CREATE, EaBuffer, sizeof(EaBuffer)); - if (FAILED(Res)) { + if (XDP_FAILED(Res)) { return Res; } @@ -36,7 +36,7 @@ _XskCreateVersion( // Performance optimization: skip setting the file handle upon each IO completion. // Res = _XdpSetFileCompletionModes(*Socket, FILE_SKIP_SET_EVENT_ON_HANDLE); - if (FAILED(Res)) { + if (XDP_FAILED(Res)) { _XdpCloseHandle(*Socket); } @@ -198,7 +198,7 @@ XskGetSockopt( &BytesReturned, NULL, FALSE); - if (FAILED(Res)) { + if (XDP_FAILED(Res)) { return Res; } @@ -263,7 +263,7 @@ XskNotifySocket( &BytesReturned, NULL, FALSE); - if (FAILED(Res)) { + if (XDP_FAILED(Res)) { return Res; } @@ -311,8 +311,8 @@ XskGetNotifyAsyncResult( if (!NT_SUCCESS(Iosb->Status)) { XDP_STATUS Status; Status = _XdpConvertNtStatusToXdpStatus(Iosb->Status); - XDPAPI_ASSERT(FAILED(Status)); - _Analysis_assume_(FAILED(Status)); + XDPAPI_ASSERT(XDP_FAILED(Status)); + _Analysis_assume_(XDP_FAILED(Status)); return Status; } diff --git a/published/external/xdp/details/ioctlfn.h b/published/external/xdp/details/ioctlfn.h index c2ecae8c..79f9ee71 100644 --- a/published/external/xdp/details/ioctlfn.h +++ b/published/external/xdp/details/ioctlfn.h @@ -163,13 +163,19 @@ _XdpOpen( UNICODE_STRING DeviceName; OBJECT_ATTRIBUTES ObjectAttributes; IO_STATUS_BLOCK IoStatusBlock; +#ifdef _KERNEL_MODE +#define _XDP_OPEN_KERNEL_OBJ_FLAGS OBJ_KERNEL_HANDLE +#else +#define _XDP_OPEN_KERNEL_OBJ_FLAGS 0 +#endif // // Open a handle to the XDP device. // RtlInitUnicodeString(&DeviceName, XDP_DEVICE_NAME); InitializeObjectAttributes( - &ObjectAttributes, &DeviceName, OBJ_CASE_INSENSITIVE, NULL, NULL); + &ObjectAttributes, &DeviceName, OBJ_CASE_INSENSITIVE | _XDP_OPEN_KERNEL_OBJ_FLAGS, + NULL, NULL); return _XdpConvertNtStatusToXdpStatus( @@ -243,7 +249,7 @@ _XdpIoctl( if (Event == &LocalEvent && XdpStatus == XDP_STATUS_PENDING) { XdpStatus = _XdpWaitInfinite(Event); - if (FAILED(XdpStatus)) { + if (XDP_FAILED(XdpStatus)) { goto Exit; } diff --git a/published/external/xdp/wincommon.h b/published/external/xdp/wincommon.h index 7ba3272f..a8b98c83 100644 --- a/published/external/xdp/wincommon.h +++ b/published/external/xdp/wincommon.h @@ -23,7 +23,10 @@ extern "C" { #ifdef _KERNEL_MODE -#include +#include +#include +#include +#include #else diff --git a/samples/xskfwd/xskfwd.c b/samples/xskfwd/xskfwd.c index 3a4a1f08..53563233 100644 --- a/samples/xskfwd/xskfwd.c +++ b/samples/xskfwd/xskfwd.c @@ -5,9 +5,16 @@ #include #include +#include +#ifdef _KERNEL_MODE +#define LOGERR(...) +#else #include #include +#define LOGERR(...) \ + fprintf(stderr, "ERR: "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n") +#endif const CHAR *UsageText = "xskfwd.exe " @@ -24,9 +31,6 @@ const XDP_HOOK_ID XdpInspectRxL2 = { .SubLayer = XDP_HOOK_INSPECT, }; -#define LOGERR(...) \ - fprintf(stderr, "ERR: "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n") - static VOID TranslateRxToTx( @@ -49,17 +53,15 @@ TranslateRxToTx( } } -INT -__cdecl -main( - INT argc, - CHAR **argv +XDP_STATUS +XskFwd( + _In_ UINT32 IfIndex, + _In_ volatile BOOLEAN *Stop ) { HRESULT Result; - HANDLE Socket; - HANDLE Program; - UINT32 IfIndex; + HANDLE Socket = NULL; + HANDLE Program = NULL; XDP_RULE Rule = {0}; UCHAR Frame[1514]; XSK_UMEM_REG UmemReg = {0}; @@ -72,20 +74,13 @@ main( XSK_RING TxCompRing; UINT32 RingIndex; - if (argc < 2) { - fprintf(stderr, UsageText); - return EXIT_FAILURE; - } - - IfIndex = atoi(argv[1]); - // // Create an AF_XDP socket. The newly created socket is not connected. // Result = XskCreate(&Socket); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XskCreate failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } // @@ -99,9 +94,9 @@ main( UmemReg.Address = Frame; Result = XskSetSockopt(Socket, XSK_SOCKOPT_UMEM_REG, &UmemReg, sizeof(UmemReg)); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XSK_UMEM_REG failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } // @@ -109,9 +104,9 @@ main( // queue, and indicate the intent to perform RX and TX actions. // Result = XskBind(Socket, IfIndex, 0, XSK_BIND_FLAG_RX | XSK_BIND_FLAG_TX); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XskBind failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } // @@ -122,27 +117,27 @@ main( // Result = XskSetSockopt(Socket, XSK_SOCKOPT_RX_RING_SIZE, &RingSize, sizeof(RingSize)); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XSK_SOCKOPT_RX_RING_SIZE failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } Result = XskSetSockopt(Socket, XSK_SOCKOPT_RX_FILL_RING_SIZE, &RingSize, sizeof(RingSize)); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XSK_SOCKOPT_RX_FILL_RING_SIZE failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } Result = XskSetSockopt(Socket, XSK_SOCKOPT_TX_RING_SIZE, &RingSize, sizeof(RingSize)); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XSK_SOCKOPT_TX_RING_SIZE failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } Result = XskSetSockopt(Socket, XSK_SOCKOPT_TX_COMPLETION_RING_SIZE, &RingSize, sizeof(RingSize)); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XSK_SOCKOPT_TX_COMPLETION_RING_SIZE failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } // @@ -150,9 +145,9 @@ main( // available and RX and TX can occur. // Result = XskActivate(Socket, XSK_ACTIVATE_FLAG_NONE); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XskActivate failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } // @@ -160,9 +155,9 @@ main( // OptionLength = sizeof(RingInfo); Result = XskGetSockopt(Socket, XSK_SOCKOPT_RING_INFO, &RingInfo, &OptionLength); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XSK_SOCKOPT_RING_INFO failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } // @@ -204,9 +199,9 @@ main( Rule.Redirect.Target = Socket; Result = XdpCreateProgram(IfIndex, &XdpInspectRxL2, 0, 0, &Rule, 1, &Program); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XdpCreateProgram failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } // @@ -215,7 +210,7 @@ main( // be optimized further by consuming, reserving, and submitting batches of // frames across each XskRing* function. // - while (TRUE) { + do { if (XskRingConsumerReserve(&RxRing, 1, &RingIndex) == 1) { XSK_BUFFER_DESCRIPTOR *RxBuffer; XSK_BUFFER_DESCRIPTOR *TxBuffer; @@ -261,9 +256,9 @@ main( // optimized further using the XskRingProducerNeedPoke helper. // Result = XskNotifySocket(Socket, XSK_NOTIFY_FLAG_POKE_TX, 0, &NotifyResult); - if (FAILED(Result)) { + if (XDP_FAILED(Result)) { LOGERR("XskNotifySocket failed: %x", Result); - return EXIT_FAILURE; + goto Exit; } } @@ -299,17 +294,192 @@ main( XskRingConsumerRelease(&TxCompRing, 1); XskRingProducerSubmit(&RxFillRing, 1); } - } + } while (!ReadBooleanNoFence(Stop)); + + Result = XDP_STATUS_SUCCESS; + +Exit: // // Close the XDP program. Traffic will no longer be intercepted by XDP. // - CloseHandle(Program); + if (Program != NULL) { + CxPlatCloseHandle(Program); + } // // Close the AF_XDP socket. All socket resources will be cleaned up by XDP. // - CloseHandle(Socket); + if (Socket != NULL) { + CxPlatCloseHandle(Socket); + } + + return Result; +} + +#ifdef _KERNEL_MODE + +DRIVER_INITIALIZE DriverEntry; +static DRIVER_UNLOAD DriverUnload; +static KSTART_ROUTINE XskFwdWorker; - return EXIT_SUCCESS; +static UINT32 IfIndex; +static BOOLEAN Stop; +static CXPLAT_THREAD WorkerThread; + +static +_Use_decl_annotations_ +VOID +XskFwdWorker( + VOID *Context + ) +{ + UNREFERENCED_PARAMETER(Context); + + XskFwd(IfIndex, &Stop); } + +static +_Use_decl_annotations_ +VOID +DriverUnload( + DRIVER_OBJECT *DriverObject + ) +{ + UNREFERENCED_PARAMETER(DriverObject); + + if (WorkerThread != NULL) { + WriteBooleanNoFence(&Stop, TRUE); + CxPlatThreadWaitForever(&WorkerThread); + CxPlatThreadDelete(&WorkerThread); + } +} + +_Use_decl_annotations_ +NTSTATUS +DriverEntry( + DRIVER_OBJECT *DriverObject, + UNICODE_STRING *RegistryPath + ) +{ + NTSTATUS Status; + CXPLAT_THREAD_CONFIG ThreadConfig = {0}; + HANDLE KeyHandle; + UNICODE_STRING UnicodeName; + OBJECT_ATTRIBUTES ObjectAttributes = {0}; + UCHAR InformationBuffer[512] = {0}; + KEY_VALUE_FULL_INFORMATION *Information = (KEY_VALUE_FULL_INFORMATION *) InformationBuffer; + ULONG ResultLength; + BOOLEAN RunInline = FALSE; + +#pragma prefast(suppress : __WARNING_BANNED_MEM_ALLOCATION_UNSAFE, "Non executable pool is enabled via -DPOOL_NX_OPTIN_AUTO=1.") + ExInitializeDriverRuntime(0); + DriverObject->DriverUnload = DriverUnload; + + InitializeObjectAttributes( + &ObjectAttributes, + RegistryPath, + OBJ_CASE_INSENSITIVE | OBJ_KERNEL_HANDLE, + NULL, + NULL); + Status = ZwOpenKey(&KeyHandle, KEY_READ, &ObjectAttributes); + if (!NT_SUCCESS(Status)) { + goto Exit; + } + + // + // Read the configured IfIndex from HKLM\SYSTEM\CurrentControlSet\Services\xskfwd\IfIndex + // + RtlInitUnicodeString(&UnicodeName, L"IfIndex"); + Status = + ZwQueryValueKey( + KeyHandle, + &UnicodeName, + KeyValueFullInformation, + Information, + sizeof(InformationBuffer), + &ResultLength); + if (NT_SUCCESS(Status)) { + if (Information->Type != REG_DWORD) { + Status = STATUS_INVALID_PARAMETER_MIX; + } else { + IfIndex = *((DWORD UNALIGNED *)((CHAR *)Information + Information->DataOffset)); + } + } + + // + // Read the configured IfIndex from HKLM\SYSTEM\CurrentControlSet\Services\xskfwd\RunInline + // + RtlInitUnicodeString(&UnicodeName, L"RunInline"); + Status = + ZwQueryValueKey( + KeyHandle, + &UnicodeName, + KeyValueFullInformation, + Information, + sizeof(InformationBuffer), + &ResultLength); + if (NT_SUCCESS(Status)) { + if (Information->Type != REG_DWORD) { + Status = STATUS_INVALID_PARAMETER_MIX; + } else { + RunInline = !!*((DWORD UNALIGNED *)((CHAR *)Information + Information->DataOffset)); + } + } + + ZwClose(KeyHandle); + + if (RunInline) { + BOOLEAN AlwaysStop = TRUE; + + // + // Run a single iteration and return the result. + // + Status = XskFwd(IfIndex, &AlwaysStop); + if (!NT_SUCCESS(Status)) { + goto Exit; + } + } else { + // + // Start a system worker thread to run until the driver is unloaded. + // + ThreadConfig.Callback = XskFwdWorker; + Status = CxPlatThreadCreate(&ThreadConfig, &WorkerThread); + if (!NT_SUCCESS(Status)) { + goto Exit; + } + } + + +Exit: + + if (!NT_SUCCESS(Status)) { + DriverUnload(DriverObject); + } + + return Status; +} + +#else // _KERNEL_MODE + +INT +__cdecl +main( + INT argc, + CHAR **argv + ) +{ + UINT32 IfIndex; + BOOLEAN Stop = FALSE; // Run until the process is terminated. + + if (argc < 2) { + LOGERR(UsageText); + return EXIT_FAILURE; + } + + IfIndex = atoi(argv[1]); + + return XskFwd(IfIndex, &Stop); +} + +#endif // _KERNEL_MODE diff --git a/samples/xskfwd/xskfwd.vcxproj b/samples/xskfwd/xskfwd.vcxproj index b6314528..3bc86197 100644 --- a/samples/xskfwd/xskfwd.vcxproj +++ b/samples/xskfwd/xskfwd.vcxproj @@ -10,6 +10,12 @@ + + + $(SolutionDir)submodules\cxplat\inc; + %(AdditionalIncludeDirectories) + + ntdll.lib;onecore.lib;%(AdditionalDependencies) diff --git a/samples/xskfwd/xskfwdkm.vcxproj b/samples/xskfwd/xskfwdkm.vcxproj new file mode 100644 index 00000000..90689729 --- /dev/null +++ b/samples/xskfwd/xskfwdkm.vcxproj @@ -0,0 +1,21 @@ + + + + + + + {f722da97-3252-42f1-92d3-215c99935607} + xskfwdkm + sys + + + + + + $(SolutionDir)submodules\cxplat\inc; + %(AdditionalIncludeDirectories) + + + + + diff --git a/src/xdp/xsk.c b/src/xdp/xsk.c index 763f1c8f..05eeef7c 100644 --- a/src/xdp/xsk.c +++ b/src/xdp/xsk.c @@ -1810,35 +1810,49 @@ XskFreeRing( ) { ASSERT( - (Ring->Size != 0 && Ring->Mdl != NULL && Ring->Shared != NULL) || - (Ring->Size == 0 && Ring->Mdl == NULL && Ring->Shared == NULL)); + (Ring->Size != 0 && Ring->Shared != NULL) || + (Ring->Size == 0 && Ring->Shared == NULL && Ring->Mdl == NULL && Ring->UserVa == NULL)); - if (Ring->UserVa != NULL) { - VOID *CurrentProcess = PsGetCurrentProcess(); - KAPC_STATE ApcState; + if (Ring->Mdl != NULL) { + if (Ring->UserVa != NULL) { + VOID *CurrentProcess = PsGetCurrentProcess(); + KAPC_STATE ApcState; - ASSERT(Ring->OwningProcess != NULL); + ASSERT(Ring->OwningProcess != NULL); + ASSERT(Ring->Mdl != NULL); - if (CurrentProcess != Ring->OwningProcess) { - KeStackAttachProcess(Ring->OwningProcess, &ApcState); - } + if (CurrentProcess != Ring->OwningProcess) { + KeStackAttachProcess(Ring->OwningProcess, &ApcState); + } - ASSERT(Ring->Mdl); - MmUnmapLockedPages(Ring->UserVa, Ring->Mdl); + MmUnmapLockedPages(Ring->UserVa, Ring->Mdl); - if (CurrentProcess != Ring->OwningProcess) { + if (CurrentProcess != Ring->OwningProcess) { #pragma prefast(suppress:6001, "ApcState is correctly initialized in KeStackAttachProcess above.") - KeUnstackDetachProcess(&ApcState); + KeUnstackDetachProcess(&ApcState); + } + + Ring->UserVa = NULL; } + IoFreeMdl(Ring->Mdl); + Ring->Mdl = NULL; + } + if (Ring->UserVa != NULL) { + // + // If the UserVa was set without an MDL, then it must be a direct kernel + // mapping. + // + ASSERT(Ring->UserVa == Ring->Shared); + Ring->UserVa = NULL; + } + if (Ring->OwningProcess != NULL) { ObDereferenceObject(Ring->OwningProcess); Ring->OwningProcess = NULL; } - if (Ring->Mdl != NULL) { - IoFreeMdl(Ring->Mdl); - } if (Ring->Shared != NULL) { ExFreePoolWithTag(Ring->Shared, POOLTAG_RING); + Ring->Shared = NULL; } } @@ -2868,7 +2882,6 @@ XskFillRingInfo( _Out_ XSK_RING_INFO *Info ) { - ASSERT(Ring->Mdl != NULL); ASSERT(Ring->Shared != NULL); ASSERT(Ring->Size != 0); ASSERT(Ring->UserVa != NULL); @@ -3175,35 +3188,41 @@ XskSockoptSetRingSize( goto Exit; } - Mdl = - IoAllocateMdl( - Shared, - AllocationSize, - FALSE, // SecondaryBuffer - FALSE, // ChargeQuota - NULL); // Irp - if (Mdl == NULL) { - Status = STATUS_INSUFFICIENT_RESOURCES; - goto Exit; - } - MmBuildMdlForNonPagedPool(Mdl); + ASSERT(ALIGN_DOWN_POINTER_BY(Shared, PAGE_SIZE) == Shared); - __try { - UserVa = - MmMapLockedPagesSpecifyCache( - Mdl, - RequestorMode, - MmCached, - NULL, // RequestedAddress - FALSE,// BugCheckOnFailure - NormalPagePriority | MdlMappingNoExecute); - if (UserVa == NULL) { + if (RequestorMode == KernelMode) { + UserVa = Shared; + } else { + Mdl = + IoAllocateMdl( + Shared, + AllocationSize, + FALSE, // SecondaryBuffer + FALSE, // ChargeQuota + NULL); // Irp + if (Mdl == NULL) { Status = STATUS_INSUFFICIENT_RESOURCES; goto Exit; } - } __except (EXCEPTION_EXECUTE_HANDLER) { - Status = GetExceptionCode(); - goto Exit; + MmBuildMdlForNonPagedPool(Mdl); + + __try { + UserVa = + MmMapLockedPagesSpecifyCache( + Mdl, + RequestorMode, + MmCached, + NULL, // RequestedAddress + FALSE,// BugCheckOnFailure + NormalPagePriority | MdlMappingNoExecute); + if (UserVa == NULL) { + Status = STATUS_INSUFFICIENT_RESOURCES; + goto Exit; + } + } __except (EXCEPTION_EXECUTE_HANDLER) { + Status = GetExceptionCode(); + goto Exit; + } } KeAcquireSpinLock(&Xsk->Lock, &OldIrql); @@ -3234,8 +3253,8 @@ XskSockoptSetRingSize( } ASSERT( - (Ring->Size != 0 && Ring->Mdl != NULL && Ring->Shared != NULL) || - (Ring->Size == 0 && Ring->Mdl == NULL && Ring->Shared == NULL)); + (Ring->Size != 0 && Ring->Shared != NULL) || + (Ring->Size == 0 && Ring->Shared == NULL && Ring->Mdl == NULL && Ring->UserVa == NULL)); if (Ring->Size != 0) { Status = STATUS_INVALID_DEVICE_STATE; @@ -3267,10 +3286,11 @@ XskSockoptSetRingSize( if (IsLockHeld) { KeReleaseSpinLock(&Xsk->Lock, OldIrql); } - if (UserVa != NULL) { - MmUnmapLockedPages(UserVa, Mdl); - } if (Mdl != NULL) { + if (UserVa != NULL) { + ASSERT(RequestorMode != KernelMode); + MmUnmapLockedPages(UserVa, Mdl); + } IoFreeMdl(Mdl); } if (Shared != NULL) { diff --git a/test/build_headers/ddk/c/ddkheaders.c b/test/build_headers/ddk/c/ddkheaders.c index 652bd5ea..6556f0ad 100644 --- a/test/build_headers/ddk/c/ddkheaders.c +++ b/test/build_headers/ddk/c/ddkheaders.c @@ -3,5 +3,4 @@ // Licensed under the MIT License. // -#include #include diff --git a/test/build_headers/ddk/cpp/ddkheaders.cpp b/test/build_headers/ddk/cpp/ddkheaders.cpp index 652bd5ea..6556f0ad 100644 --- a/test/build_headers/ddk/cpp/ddkheaders.cpp +++ b/test/build_headers/ddk/cpp/ddkheaders.cpp @@ -3,5 +3,4 @@ // Licensed under the MIT License. // -#include #include diff --git a/test/fakendis/inc/fndisnpi.h b/test/fakendis/inc/fndisnpi.h index 67ba0a60..f6c0db8c 100644 --- a/test/fakendis/inc/fndisnpi.h +++ b/test/fakendis/inc/fndisnpi.h @@ -25,7 +25,7 @@ FNdisClientOpen( RtlZeroMemory(FndisClient, sizeof(*FndisClient)); RtlInitUnicodeString(&Name, FNDIS_DEVICE_NAME); - InitializeObjectAttributes(&Oa, &Name, OBJ_CASE_INSENSITIVE, NULL, NULL); + InitializeObjectAttributes(&Oa, &Name, OBJ_CASE_INSENSITIVE | OBJ_KERNEL_HANDLE, NULL, NULL); Status = ZwCreateFile( diff --git a/test/xdpmp/precomp.h b/test/xdpmp/precomp.h index 8c445df6..49d52de8 100644 --- a/test/xdpmp/precomp.h +++ b/test/xdpmp/precomp.h @@ -5,14 +5,13 @@ #pragma once -#include +#include #include -#include -#include -#include #include #include #include +#include +#include #include #include diff --git a/test/xdpmp/xdpmp.vcxproj b/test/xdpmp/xdpmp.vcxproj index dc1e55c2..ed9de209 100644 --- a/test/xdpmp/xdpmp.vcxproj +++ b/test/xdpmp/xdpmp.vcxproj @@ -10,7 +10,6 @@ - diff --git a/tools/check-drivers.ps1 b/tools/check-drivers.ps1 index 8b20b4cc..fdf8df91 100644 --- a/tools/check-drivers.ps1 +++ b/tools/check-drivers.ps1 @@ -53,6 +53,7 @@ function Check-And-Remove-Driver($Driver, $Component) { } # Check for any XDP drivers. +Check-And-Remove-Driver "xskfwdkm.sys" "xskfwdkm" Check-And-Remove-Driver "fnmp.sys" "fnmp" Check-And-Remove-Driver "fnlwf.sys" "fnlwf" Check-And-Remove-Driver "xdpmp.sys" "xdpmp" diff --git a/tools/setup.ps1 b/tools/setup.ps1 index 9afa82b9..092dfff0 100644 --- a/tools/setup.ps1 +++ b/tools/setup.ps1 @@ -29,11 +29,11 @@ param ( [string]$Platform = "x64", [Parameter(Mandatory = $false)] - [ValidateSet("", "fndis", "xdp", "xdpmp", "fnmp", "fnlwf", "fnsock", "ebpf")] + [ValidateSet("", "fndis", "xdp", "xdpmp", "fnmp", "fnlwf", "fnsock", "ebpf", "xskfwdkm")] [string]$Install = "", [Parameter(Mandatory = $false)] - [ValidateSet("", "fndis", "xdp", "xdpmp", "fnmp", "fnlwf", "fnsock", "ebpf")] + [ValidateSet("", "fndis", "xdp", "xdpmp", "fnmp", "fnlwf", "fnsock", "ebpf", "xskfwdkm")] [string]$Uninstall = "", [Parameter(Mandatory = $false)] @@ -78,6 +78,7 @@ $XdpMpCert = "$ArtifactsDir\test\xdpmp.cer" $XdpMpComponentId = "ms_xdpmp" $XdpMpDeviceId = "xdpmp0" $XdpMpServiceName = "XDPMP" +$XskFwdKmSys = "$ArtifactsDir\test\xskfwdkm.sys" # Ensure the output path exists. New-Item -ItemType Directory -Force -Path $LogsDir | Out-Null @@ -620,6 +621,33 @@ function Uninstall-Ebpf { Refresh-Path } +# Installs the xskfwdkm driver. +function Install-XskFwdKm { + if (!(Test-Path $XskFwdKmSys)) { + Write-Error "$XskFwdKmSys does not exist!" + } + + Write-Verbose "sc.exe create xskfwdkm type= kernel start= demand binpath= $XskFwdKmSys" + sc.exe create xskfwdkm type= kernel start= demand binpath= $XskFwdKmSys | Write-Verbose + if ($LastExitCode) { + Write-Error "sc.exe exit code: $LastExitCode" + } + + Start-Service-With-Retry xskfwdkm + + Write-Verbose "xskfwdkm.sys install complete!" +} + +# Uninstalls the xskfwdkm driver. +function Uninstall-XskFwdKm { + Write-Verbose "Stop-Service xskfwdkm" + try { Stop-Service xskfwdkm -NoWait } catch { } + + Cleanup-Service xskfwdkm + + Write-Verbose "xskfwdkm.sys uninstall complete!" +} + try { if ($Install -eq "fndis") { Install-FakeNdis @@ -642,6 +670,9 @@ try { if ($Install -eq "fnsock") { Install-FnSock } + if ($Install -eq "xskfwdkm") { + Install-XskFwdKm + } if ($Uninstall -eq "fndis") { Uninstall-FakeNdis @@ -664,6 +695,9 @@ try { if ($Uninstall -eq "fnsock") { Uninstall-FnSock } + if ($Uninstall -eq "xskfwdkm") { + Uninstall-XskFwdKm + } } catch { Write-Error $_ -ErrorAction $OriginalErrorActionPreference } diff --git a/tools/xskfwdkm.ps1 b/tools/xskfwdkm.ps1 new file mode 100644 index 00000000..05441016 --- /dev/null +++ b/tools/xskfwdkm.ps1 @@ -0,0 +1,67 @@ +<# + +.SYNOPSIS +This script runs the XskFwdKm sample driver. + +.PARAMETER Config + Specifies the build configuration to use. + +.PARAMETER Platform + The CPU architecture to use. + +#> + +param ( + [Parameter(Mandatory = $false)] + [ValidateSet("Debug", "Release")] + [string]$Config = "Debug", + + [Parameter(Mandatory = $false)] + [ValidateSet("x64", "arm64")] + [string]$Platform = "x64" +) + +Set-StrictMode -Version 'Latest' +$ErrorActionPreference = 'Stop' + +# Important paths. +$RootDir = Split-Path $PSScriptRoot -Parent +. $RootDir\tools\common.ps1 + +try { + & "$RootDir\tools\log.ps1" -Start -Name xskfwdkm -Profile XdpFunctional.Verbose -Config $Config -Platform $Platform + + Write-Verbose "installing xdp..." + & "$RootDir\tools\setup.ps1" -Install xdp -Config $Config -Platform $Platform + Write-Verbose "installed xdp." + + Write-Verbose "installing fndis..." + & "$RootDir\tools\setup.ps1" -Install fndis -Config $Config -Platform $Platform + Write-Verbose "installed fndis." + + Write-Verbose "installing xdpmp..." + & "$RootDir\tools\setup.ps1" -Install xdpmp -Config $Config -Platform $Platform -XdpmpPollProvider FNDIS + Write-Verbose "installed xdpmp." + + Write-Verbose "installing xskfwdkm..." + & "$RootDir\tools\setup.ps1" -Install xskfwdkm -Config $Config -Platform $Platform + Write-Verbose "installed xskfwdkm." + + Stop-Service xskfwdkm + + $IfIndex = (Get-NetAdapter XDPMP).ifIndex + Write-Verbose "reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xskfwdkm /v IfIndex /d $IfIndex /t REG_DWORD /f" + reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xskfwdkm /v IfIndex /d $IfIndex /t REG_DWORD /f | Write-Verbose + + Write-Verbose "reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xskfwdkm /v RunInline /d 1 /t REG_DWORD /f" + reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xskfwdkm /v RunInline /d 1 /t REG_DWORD /f | Write-Verbose + + # If XskFwdKm fails to run an iteration inline, it will fail to start. + Start-Service xskfwdkm +} finally { + & "$RootDir\tools\setup.ps1" -Uninstall xskfwdkm -Config $Config -Platform $Platform -ErrorAction 'Continue' + & "$RootDir\tools\setup.ps1" -Uninstall xdpmp -Config $Config -Platform $Platform -ErrorAction 'Continue' + & "$RootDir\tools\setup.ps1" -Uninstall fndis -Config $Config -Platform $Platform -ErrorAction 'Continue' + & "$RootDir\tools\setup.ps1" -Uninstall xdp -Config $Config -Platform $Platform -ErrorAction 'Continue' + & "$RootDir\tools\log.ps1" -Stop -Name xskfwdkm -Config $Config -Platform $Platform -ErrorAction 'Continue' +} diff --git a/xdp.sln b/xdp.sln index 4fae4dca..1a697776 100644 --- a/xdp.sln +++ b/xdp.sln @@ -65,6 +65,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "onebranch", "src\onebranch\ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xdpruntime", "src\xdpruntime\xdpruntime.vcxproj", "{8250C528-6A2C-44E7-AA76-1E6BEC9127A3}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xskfwdkm", "samples\xskfwd\xskfwdkm.vcxproj", "{F722DA97-3252-42F1-92D3-215C99935607}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64 = Debug|ARM64 @@ -385,6 +387,14 @@ Global {8250C528-6A2C-44E7-AA76-1E6BEC9127A3}.Release|ARM64.Build.0 = Release|ARM64 {8250C528-6A2C-44E7-AA76-1E6BEC9127A3}.Release|x64.ActiveCfg = Release|x64 {8250C528-6A2C-44E7-AA76-1E6BEC9127A3}.Release|x64.Build.0 = Release|x64 + {F722DA97-3252-42F1-92D3-215C99935607}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {F722DA97-3252-42F1-92D3-215C99935607}.Debug|ARM64.Build.0 = Debug|ARM64 + {F722DA97-3252-42F1-92D3-215C99935607}.Debug|x64.ActiveCfg = Debug|x64 + {F722DA97-3252-42F1-92D3-215C99935607}.Debug|x64.Build.0 = Debug|x64 + {F722DA97-3252-42F1-92D3-215C99935607}.Release|ARM64.ActiveCfg = Release|ARM64 + {F722DA97-3252-42F1-92D3-215C99935607}.Release|ARM64.Build.0 = Release|ARM64 + {F722DA97-3252-42F1-92D3-215C99935607}.Release|x64.ActiveCfg = Release|x64 + {F722DA97-3252-42F1-92D3-215C99935607}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE