offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp - external/github.com/llvm/llvm-project - Git at Google

 //===------- Offload API tests - olLaunchKernel --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//

 #include "../common/Fixtures.hpp"
 #include <OffloadAPI.h>
 #include <gtest/gtest.h>

 #define KERNEL_TEST(NAME, KERNEL)                                              \
   struct olLaunchKernel##NAME##Test : LaunchSingleKernelTestBase {             \
     void SetUp() override { SetUpKernel(#KERNEL); }                            \
   };                                                                           \
   OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernel##NAME##Test);

 KERNEL_TEST(Foo, foo)
 KERNEL_TEST(NoArgs, noargs)
 KERNEL_TEST(MultiArgs, multiargs)
 KERNEL_TEST(Composite, composite)
 KERNEL_TEST(Byte, byte)
 KERNEL_TEST(LocalMem, localmem)
 KERNEL_TEST(LocalMemReduction, localmem_reduction)
 KERNEL_TEST(LocalMemStatic, localmem_static)
 KERNEL_TEST(SingleCounterSyncEvent, single_counter)
 KERNEL_TEST(GlobalCtor, global_ctor)
 KERNEL_TEST(GlobalDtor, global_dtor)
 KERNEL_TEST(GridSize, gridsize)

 struct LaunchMultipleKernelTestBase : LaunchKernelTestBase {
   void SetUpKernels(const char *program, std::vector<const char *> kernels) {
     RETURN_ON_FATAL_FAILURE(SetUpProgram(program));

     Kernels.resize(kernels.size());
     size_t I = 0;
     for (auto K : kernels)
       ASSERT_SUCCESS(
           olGetSymbol(Program, K, OL_SYMBOL_KIND_KERNEL, &Kernels[I++]));
   }

   std::vector<ol_symbol_handle_t> Kernels;
 };

 struct ArgsSingleCounter {
   int32_t InitLoop;
   int32_t Addend;
   uint32_t *InitVal;
   uint32_t *Out;
 };

 #define KERNEL_MULTI_TEST(NAME, PROGRAM, ...)                                  \
   struct olLaunchKernel##NAME##Test : LaunchMultipleKernelTestBase {           \
     void SetUp() override { SetUpKernels(#PROGRAM, {__VA_ARGS__}); }           \
   };                                                                           \
   OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernel##NAME##Test);

 KERNEL_MULTI_TEST(Global, global, "write", "read")

 TEST_P(olLaunchKernelFooTest, Success) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

   void *ArgPtrs[] = {&Mem};
   size_t ArgSizes[] = {sizeof(Mem)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < 64; i++) {
     ASSERT_EQ(Data[i], i);
   }

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 TEST_P(olLaunchKernelFooTest, SuccessThreaded) {
   SKIP_KNOWN_FAILURE(LevelZero{"thread-safety issues"});

   threadify([&](size_t) {
     void *DevAlloc, *HstAlloc;
     size_t Size = LaunchArgs.GroupSize.x * sizeof(uint32_t);
     ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &DevAlloc));
     ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_HOST, Size, &HstAlloc));

     void *ArgPtrs[] = {&DevAlloc};
     size_t ArgSizes[] = {sizeof(DevAlloc)};

     ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                   std::size(ArgPtrs), ArgPtrs, ArgSizes));

     ASSERT_SUCCESS(olMemcpy(Queue, HstAlloc, Host, DevAlloc, Device, Size));

     ASSERT_SUCCESS(olSyncQueue(Queue));

     uint32_t *Data = static_cast<uint32_t *>(HstAlloc);
     for (uint32_t i = 0; i < LaunchArgs.GroupSize.x; i++) {
       ASSERT_EQ(Data[i], i);
     }

     ASSERT_SUCCESS(olMemFree(DevAlloc));
     ASSERT_SUCCESS(olMemFree(HstAlloc));
   });
 }

 TEST_P(olLaunchKernelNoArgsTest, Success) {
   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr, 0,
                                 nullptr, nullptr));

   ASSERT_SUCCESS(olSyncQueue(Queue));
 }

 TEST_P(olLaunchKernelMultiArgsTest, Success) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(int), &Mem));

   char A = 3;
   int *B = (int *)Mem;
   short C = 5;

   void *ArgPtrs[] = {&A, &B, &C};
   size_t ArgSizes[] = {sizeof(A), sizeof(B), sizeof(C)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue));

   int *Data = (int *)Mem;
   for (uint32_t i = 0; i < LaunchArgs.GroupSize.x; i++)
     ASSERT_EQ(Data[i], A + C + static_cast<int>(i));

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 struct Foo {
   uint32_t a;
   uint32_t b;
 };

 TEST_P(olLaunchKernelCompositeTest, Success) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

   uint8_t N = 1;
   Foo F{2, 3};
   uint32_t *Out = (uint32_t *)Mem;

   void *ArgPtrs[] = {&N, &F, &Out};
   size_t ArgSizes[] = {sizeof(N), sizeof(F), sizeof(Out)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < LaunchArgs.GroupSize.x; i++)
     ASSERT_EQ(Data[i], N + F.a + F.b + i);

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 TEST_P(olLaunchKernelFooTest, SuccessSynchronous) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

   void *ArgPtrs[] = {&Mem};
   size_t ArgSizes[] = {sizeof(Mem)};

   ASSERT_SUCCESS(olLaunchKernel(nullptr, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < 64; i++) {
     ASSERT_EQ(Data[i], i);
   }

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 TEST_P(olLaunchKernelByteTest, Success) {
   unsigned char C = 42;

   void *ArgPtrs[] = {&C};
   size_t ArgSizes[] = {sizeof(C)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue));
 }

 TEST_P(olLaunchKernelLocalMemTest, Success) {
   SKIP_KNOWN_FAILURE(LevelZero{"unsupported DynSharedMemory"});

   LaunchArgs.NumGroups.x = 4;
   LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);

   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x *
                                 sizeof(uint32_t),
                             &Mem));

   void *ArgPtrs[] = {&Mem};
   size_t ArgSizes[] = {sizeof(Mem)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x; i++)
     ASSERT_EQ(Data[i], (i % 64) * 2);

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 TEST_P(olLaunchKernelLocalMemReductionTest, Success) {
   SKIP_KNOWN_FAILURE(LevelZero{"unsupported DynSharedMemory"});

   LaunchArgs.NumGroups.x = 4;
   LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);

   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));

   void *ArgPtrs[] = {&Mem};
   size_t ArgSizes[] = {sizeof(Mem)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
     ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 TEST_P(olLaunchKernelLocalMemStaticTest, Success) {
   SKIP_KNOWN_FAILURE(LevelZero{"unsupported DynSharedMemory"});

   LaunchArgs.NumGroups.x = 4;
   LaunchArgs.DynSharedMemory = 0;

   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));

   void *ArgPtrs[] = {&Mem};
   size_t ArgSizes[] = {sizeof(Mem)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
     ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 // The test intends to verify the correctness of the current implementation of
 // the event synchronisation.
 TEST_P(olLaunchKernelSingleCounterSyncEventTest, SuccessSyncEvent) {
   void *InitValuePassed;
   void *ResNum;

   size_t Size = sizeof(uint32_t);

   ASSERT_SUCCESS(
       olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &InitValuePassed));
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &ResNum));

   uint32_t HostInitVal = 0;
   ASSERT_SUCCESS(
       olMemcpy(Queue, InitValuePassed, Device, &HostInitVal, Host, Size));
   ASSERT_SUCCESS(olMemcpy(Queue, ResNum, Device, &HostInitVal, Host, Size));
   ASSERT_SUCCESS(olSyncQueue(Queue));

   // The execution time of the provided kernel should be high enough to ensure
   // that the read value of the final result is not correct without explicit
   // synchronization: explicit waiting for the event following the submitted
   // operation. LoopRange is arbitrarily set with the goal of prolonging the
   // kernel execution through a high number of loop iterations. In each
   // iteration, NumberToAdd is added to the final sum, which is stored in
   // ResNum.
   int32_t LoopRange = 1000000;
   int32_t NumberToAdd = 2;

   ArgsSingleCounter Args{LoopRange, NumberToAdd, (uint32_t *)InitValuePassed,
                          (uint32_t *)ResNum};

   void *ArgPtrs[] = {&Args.InitLoop, &Args.Addend, &Args.InitVal, &Args.Out};
   size_t ArgSizes[] = {sizeof(Args.InitLoop), sizeof(Args.Addend),
                        sizeof(Args.InitVal), sizeof(Args.Out)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   uint32_t FinalResVal = 0;
   ASSERT_SUCCESS(olMemcpy(Queue, &FinalResVal, Host, ResNum, Device, Size));

   ol_event_handle_t Event = nullptr;
   ASSERT_SUCCESS(olCreateEvent(Queue, OL_EVENT_FLAGS_NONE, &Event));
   ASSERT_SUCCESS(olSyncEvent(Event));

   ASSERT_EQ(FinalResVal, NumberToAdd * LoopRange);

   ASSERT_SUCCESS(olMemFree(InitValuePassed));
   ASSERT_SUCCESS(olMemFree(ResNum));
 }

 // The test checks the correctness of the synchronization between queues using
 // events. Enqueueing the kernel on the queue `Q1` should produce `Result1`,
 // which is used as an initial value for the queue `Q2`. Therefore, before
 // executing any future work, `Q2` should wait for the event `Event1`, which is
 // created after submitting work on `Q1`. The required synchronization is
 // ensured by `olWaitEvents(Q2, &Event1, 1)`. If `Q2` uses the value passed as
 // `Result1` before the work on `Q1` has completed, the result of the kernel
 // enqueued on `Q2` would be incorrect.
 TEST_P(olLaunchKernelSingleCounterSyncEventTest, SuccessTwoQueues) {
   void *InitValuePassed;
   void *ResNum1;
   void *ResNum2;

   size_t Size = sizeof(uint32_t);

   ASSERT_SUCCESS(
       olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &InitValuePassed));
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &ResNum1));
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &ResNum2));

   uint32_t HostInitVal = 0;
   ASSERT_SUCCESS(
       olMemcpy(Queue, InitValuePassed, Device, &HostInitVal, Host, Size));
   ASSERT_SUCCESS(olMemcpy(Queue, ResNum1, Device, &HostInitVal, Host, Size));
   ASSERT_SUCCESS(olMemcpy(Queue, ResNum2, Device, &HostInitVal, Host, Size));
   ASSERT_SUCCESS(olSyncQueue(Queue));

   ol_queue_handle_t Queue2 = nullptr;
   ASSERT_SUCCESS(olCreateQueue(Device, &Queue2));

   // For the explanation of the reasoning behind particular values assigned to
   // parameters, see the comment in the Success test from the same test suite
   int32_t LoopRange = 1000000;
   int32_t NumberToAdd = 2;

   ArgsSingleCounter Args{LoopRange, NumberToAdd, (uint32_t *)InitValuePassed,
                          (uint32_t *)ResNum1};
   void *ArgPtrs[] = {&Args.InitLoop, &Args.Addend, &Args.InitVal, &Args.Out};
   size_t ArgSizes[] = {sizeof(Args.InitLoop), sizeof(Args.Addend),
                        sizeof(Args.InitVal), sizeof(Args.Out)};
   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));

   ol_event_handle_t Event = nullptr;
   ASSERT_SUCCESS(olCreateEvent(Queue, OL_EVENT_FLAGS_NONE, &Event));
   ASSERT_SUCCESS(olWaitEvents(Queue2, &Event, 1));
   ArgsSingleCounter Args2{LoopRange, NumberToAdd, (uint32_t *)ResNum1,
                           (uint32_t *)ResNum2};
   void *ArgPtrs2[] = {&Args2.InitLoop, &Args2.Addend, &Args2.InitVal,
                       &Args2.Out};

   // At the beginning of the kernel, ResNum from the first queue is saved
   // locally as the initial value. If operations enqueued before Event have not
   // been completed by the time the kernel is executed using the second queue,
   // the FinalResVal would be incorrect.
   ASSERT_SUCCESS(olLaunchKernel(Queue2, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs2), ArgPtrs2, ArgSizes));

   ASSERT_SUCCESS(olSyncQueue(Queue2));
   uint32_t FinalResVal = 0;
   ASSERT_SUCCESS(olMemcpy(Queue2, &FinalResVal, Host, ResNum2, Device, Size));
   ASSERT_SUCCESS(olSyncQueue(Queue2));

   ASSERT_EQ(FinalResVal, 2 * NumberToAdd * LoopRange);

   ASSERT_SUCCESS(olMemFree(InitValuePassed));
   ASSERT_SUCCESS(olMemFree(ResNum1));
   ASSERT_SUCCESS(olMemFree(ResNum2));
 }

 TEST_P(olLaunchKernelGlobalTest, Success) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

   void *ArgPtrs[] = {&Mem};
   size_t ArgSizes[] = {sizeof(Mem)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernels[0], &LaunchArgs, nullptr,
                                 0, nullptr, nullptr));
   ASSERT_SUCCESS(olSyncQueue(Queue));
   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernels[1], &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));
   ASSERT_SUCCESS(olSyncQueue(Queue));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < 64; i++) {
     ASSERT_EQ(Data[i], i * 2);
   }

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 TEST_P(olLaunchKernelGlobalTest, InvalidNotAKernel) {
   ol_symbol_handle_t Global = nullptr;
   ASSERT_SUCCESS(
       olGetSymbol(Program, "global", OL_SYMBOL_KIND_GLOBAL_VARIABLE, &Global));
   ASSERT_ERROR(OL_ERRC_SYMBOL_KIND,
                olLaunchKernel(Queue, Device, Global, &LaunchArgs, nullptr, 0,
                               nullptr, nullptr));
 }

 TEST_P(olLaunchKernelGlobalCtorTest, Success) {
   SKIP_KNOWN_FAILURE(LevelZero{"unsupported feature"});

   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

   void *ArgPtrs[] = {&Mem};
   size_t ArgSizes[] = {sizeof(Mem)};

   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                 std::size(ArgPtrs), ArgPtrs, ArgSizes));
   ASSERT_SUCCESS(olSyncQueue(Queue));

   uint32_t *Data = (uint32_t *)Mem;
   for (uint32_t i = 0; i < 64; i++) {
     ASSERT_EQ(Data[i], i + 100);
   }

   ASSERT_SUCCESS(olMemFree(Mem));
 }

 TEST_P(olLaunchKernelGlobalDtorTest, Success) {
   // TODO: We can't inspect the result of a destructor yet, once we
   // find/implement a way, update this test. For now we just check that nothing
   // crashes
   ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr, 0,
                                 nullptr, nullptr));
   ASSERT_SUCCESS(olSyncQueue(Queue));
 }

 TEST_P(olLaunchKernelGridSizeTest, Success) {
   void *Mem;
   ASSERT_SUCCESS(
       olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, 6 * sizeof(uint32_t), &Mem));

   uint32_t *NumBlocks = static_cast<uint32_t *>(Mem);
   uint32_t *NumThreads = static_cast<uint32_t *>(Mem) + 3;

   void *ArgPtrs[] = {&NumBlocks, &NumThreads};
   size_t ArgSizes[] = {sizeof(NumBlocks), sizeof(NumThreads)};

   const uint32_t BaseBlocks[3] = {64, 16, 8};
   const uint32_t BaseThreads[3] = {32, 4, 2};

   for (uint32_t Dim = 1; Dim <= 3; ++Dim) {
     NumBlocks[0] = NumBlocks[1] = NumBlocks[2] = 0;
     NumThreads[0] = NumThreads[1] = NumThreads[2] = 0;

     LaunchArgs.Dimensions = 3;
     LaunchArgs.NumGroups = {BaseBlocks[0], Dim >= 2 ? BaseBlocks[1] : 1,
                             Dim >= 3 ? BaseBlocks[2] : 1};
     LaunchArgs.GroupSize = {BaseThreads[0], Dim >= 2 ? BaseThreads[1] : 1,
                             Dim >= 3 ? BaseThreads[2] : 1};

     ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
                                   std::size(ArgPtrs), ArgPtrs, ArgSizes));

     ASSERT_SUCCESS(olSyncQueue(Queue));

     ASSERT_EQ(NumBlocks[0], LaunchArgs.NumGroups.x);
     ASSERT_EQ(NumBlocks[1], LaunchArgs.NumGroups.y);
     ASSERT_EQ(NumBlocks[2], LaunchArgs.NumGroups.z);
     ASSERT_EQ(NumThreads[0], LaunchArgs.GroupSize.x);
     ASSERT_EQ(NumThreads[1], LaunchArgs.GroupSize.y);
     ASSERT_EQ(NumThreads[2], LaunchArgs.GroupSize.z);
   }

   ASSERT_SUCCESS(olMemFree(Mem));
 }
	//===------- Offload API tests - olLaunchKernel --------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#include "../common/Fixtures.hpp"
	#include <OffloadAPI.h>
	#include <gtest/gtest.h>

	#define KERNEL_TEST(NAME, KERNEL) \
	struct olLaunchKernel##NAME##Test : LaunchSingleKernelTestBase { \
	void SetUp() override { SetUpKernel(#KERNEL); } \
	}; \
	OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernel##NAME##Test);

	KERNEL_TEST(Foo, foo)
	KERNEL_TEST(NoArgs, noargs)
	KERNEL_TEST(MultiArgs, multiargs)
	KERNEL_TEST(Composite, composite)
	KERNEL_TEST(Byte, byte)
	KERNEL_TEST(LocalMem, localmem)
	KERNEL_TEST(LocalMemReduction, localmem_reduction)
	KERNEL_TEST(LocalMemStatic, localmem_static)
	KERNEL_TEST(SingleCounterSyncEvent, single_counter)
	KERNEL_TEST(GlobalCtor, global_ctor)
	KERNEL_TEST(GlobalDtor, global_dtor)
	KERNEL_TEST(GridSize, gridsize)

	struct LaunchMultipleKernelTestBase : LaunchKernelTestBase {
	void SetUpKernels(const char program, std::vector<const char > kernels) {
	RETURN_ON_FATAL_FAILURE(SetUpProgram(program));

	Kernels.resize(kernels.size());
	size_t I = 0;
	for (auto K : kernels)
	ASSERT_SUCCESS(
	olGetSymbol(Program, K, OL_SYMBOL_KIND_KERNEL, &Kernels[I++]));
	}

	std::vector<ol_symbol_handle_t> Kernels;
	};

	struct ArgsSingleCounter {
	int32_t InitLoop;
	int32_t Addend;
	uint32_t *InitVal;
	uint32_t *Out;
	};

	#define KERNEL_MULTI_TEST(NAME, PROGRAM, ...) \
	struct olLaunchKernel##NAME##Test : LaunchMultipleKernelTestBase { \
	void SetUp() override { SetUpKernels(#PROGRAM, {__VA_ARGS__}); } \
	}; \
	OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernel##NAME##Test);

	KERNEL_MULTI_TEST(Global, global, "write", "read")

	TEST_P(olLaunchKernelFooTest, Success) {
	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

	void *ArgPtrs[] = {&Mem};
	size_t ArgSizes[] = {sizeof(Mem)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < 64; i++) {
	ASSERT_EQ(Data[i], i);
	}

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	TEST_P(olLaunchKernelFooTest, SuccessThreaded) {
	SKIP_KNOWN_FAILURE(LevelZero{"thread-safety issues"});

	threadify([&](size_t) {
	void DevAlloc, HstAlloc;
	size_t Size = LaunchArgs.GroupSize.x * sizeof(uint32_t);
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &DevAlloc));
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_HOST, Size, &HstAlloc));

	void *ArgPtrs[] = {&DevAlloc};
	size_t ArgSizes[] = {sizeof(DevAlloc)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olMemcpy(Queue, HstAlloc, Host, DevAlloc, Device, Size));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = static_cast<uint32_t >(HstAlloc);
	for (uint32_t i = 0; i < LaunchArgs.GroupSize.x; i++) {
	ASSERT_EQ(Data[i], i);
	}

	ASSERT_SUCCESS(olMemFree(DevAlloc));
	ASSERT_SUCCESS(olMemFree(HstAlloc));
	});
	}

	TEST_P(olLaunchKernelNoArgsTest, Success) {
	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr, 0,
	nullptr, nullptr));

	ASSERT_SUCCESS(olSyncQueue(Queue));
	}

	TEST_P(olLaunchKernelMultiArgsTest, Success) {
	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.GroupSize.x * sizeof(int), &Mem));

	char A = 3;
	int B = (int )Mem;
	short C = 5;

	void *ArgPtrs[] = {&A, &B, &C};
	size_t ArgSizes[] = {sizeof(A), sizeof(B), sizeof(C)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	int Data = (int )Mem;
	for (uint32_t i = 0; i < LaunchArgs.GroupSize.x; i++)
	ASSERT_EQ(Data[i], A + C + static_cast<int>(i));

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	struct Foo {
	uint32_t a;
	uint32_t b;
	};

	TEST_P(olLaunchKernelCompositeTest, Success) {
	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

	uint8_t N = 1;
	Foo F{2, 3};
	uint32_t Out = (uint32_t )Mem;

	void *ArgPtrs[] = {&N, &F, &Out};
	size_t ArgSizes[] = {sizeof(N), sizeof(F), sizeof(Out)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < LaunchArgs.GroupSize.x; i++)
	ASSERT_EQ(Data[i], N + F.a + F.b + i);

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	TEST_P(olLaunchKernelFooTest, SuccessSynchronous) {
	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

	void *ArgPtrs[] = {&Mem};
	size_t ArgSizes[] = {sizeof(Mem)};

	ASSERT_SUCCESS(olLaunchKernel(nullptr, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < 64; i++) {
	ASSERT_EQ(Data[i], i);
	}

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	TEST_P(olLaunchKernelByteTest, Success) {
	unsigned char C = 42;

	void *ArgPtrs[] = {&C};
	size_t ArgSizes[] = {sizeof(C)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));
	}

	TEST_P(olLaunchKernelLocalMemTest, Success) {
	SKIP_KNOWN_FAILURE(LevelZero{"unsupported DynSharedMemory"});

	LaunchArgs.NumGroups.x = 4;
	LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);

	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x *
	sizeof(uint32_t),
	&Mem));

	void *ArgPtrs[] = {&Mem};
	size_t ArgSizes[] = {sizeof(Mem)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x; i++)
	ASSERT_EQ(Data[i], (i % 64) * 2);

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	TEST_P(olLaunchKernelLocalMemReductionTest, Success) {
	SKIP_KNOWN_FAILURE(LevelZero{"unsupported DynSharedMemory"});

	LaunchArgs.NumGroups.x = 4;
	LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);

	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));

	void *ArgPtrs[] = {&Mem};
	size_t ArgSizes[] = {sizeof(Mem)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
	ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	TEST_P(olLaunchKernelLocalMemStaticTest, Success) {
	SKIP_KNOWN_FAILURE(LevelZero{"unsupported DynSharedMemory"});

	LaunchArgs.NumGroups.x = 4;
	LaunchArgs.DynSharedMemory = 0;

	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));

	void *ArgPtrs[] = {&Mem};
	size_t ArgSizes[] = {sizeof(Mem)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
	ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	// The test intends to verify the correctness of the current implementation of
	// the event synchronisation.
	TEST_P(olLaunchKernelSingleCounterSyncEventTest, SuccessSyncEvent) {
	void *InitValuePassed;
	void *ResNum;

	size_t Size = sizeof(uint32_t);

	ASSERT_SUCCESS(
	olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &InitValuePassed));
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &ResNum));

	uint32_t HostInitVal = 0;
	ASSERT_SUCCESS(
	olMemcpy(Queue, InitValuePassed, Device, &HostInitVal, Host, Size));
	ASSERT_SUCCESS(olMemcpy(Queue, ResNum, Device, &HostInitVal, Host, Size));
	ASSERT_SUCCESS(olSyncQueue(Queue));

	// The execution time of the provided kernel should be high enough to ensure
	// that the read value of the final result is not correct without explicit
	// synchronization: explicit waiting for the event following the submitted
	// operation. LoopRange is arbitrarily set with the goal of prolonging the
	// kernel execution through a high number of loop iterations. In each
	// iteration, NumberToAdd is added to the final sum, which is stored in
	// ResNum.
	int32_t LoopRange = 1000000;
	int32_t NumberToAdd = 2;

	ArgsSingleCounter Args{LoopRange, NumberToAdd, (uint32_t *)InitValuePassed,
	(uint32_t *)ResNum};

	void *ArgPtrs[] = {&Args.InitLoop, &Args.Addend, &Args.InitVal, &Args.Out};
	size_t ArgSizes[] = {sizeof(Args.InitLoop), sizeof(Args.Addend),
	sizeof(Args.InitVal), sizeof(Args.Out)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	uint32_t FinalResVal = 0;
	ASSERT_SUCCESS(olMemcpy(Queue, &FinalResVal, Host, ResNum, Device, Size));

	ol_event_handle_t Event = nullptr;
	ASSERT_SUCCESS(olCreateEvent(Queue, OL_EVENT_FLAGS_NONE, &Event));
	ASSERT_SUCCESS(olSyncEvent(Event));

	ASSERT_EQ(FinalResVal, NumberToAdd * LoopRange);

	ASSERT_SUCCESS(olMemFree(InitValuePassed));
	ASSERT_SUCCESS(olMemFree(ResNum));
	}

	// The test checks the correctness of the synchronization between queues using
	// events. Enqueueing the kernel on the queue `Q1` should produce `Result1`,
	// which is used as an initial value for the queue `Q2`. Therefore, before
	// executing any future work, `Q2` should wait for the event `Event1`, which is
	// created after submitting work on `Q1`. The required synchronization is
	// ensured by `olWaitEvents(Q2, &Event1, 1)`. If `Q2` uses the value passed as
	// `Result1` before the work on `Q1` has completed, the result of the kernel
	// enqueued on `Q2` would be incorrect.
	TEST_P(olLaunchKernelSingleCounterSyncEventTest, SuccessTwoQueues) {
	void *InitValuePassed;
	void *ResNum1;
	void *ResNum2;

	size_t Size = sizeof(uint32_t);

	ASSERT_SUCCESS(
	olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &InitValuePassed));
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &ResNum1));
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_DEVICE, Size, &ResNum2));

	uint32_t HostInitVal = 0;
	ASSERT_SUCCESS(
	olMemcpy(Queue, InitValuePassed, Device, &HostInitVal, Host, Size));
	ASSERT_SUCCESS(olMemcpy(Queue, ResNum1, Device, &HostInitVal, Host, Size));
	ASSERT_SUCCESS(olMemcpy(Queue, ResNum2, Device, &HostInitVal, Host, Size));
	ASSERT_SUCCESS(olSyncQueue(Queue));

	ol_queue_handle_t Queue2 = nullptr;
	ASSERT_SUCCESS(olCreateQueue(Device, &Queue2));

	// For the explanation of the reasoning behind particular values assigned to
	// parameters, see the comment in the Success test from the same test suite
	int32_t LoopRange = 1000000;
	int32_t NumberToAdd = 2;

	ArgsSingleCounter Args{LoopRange, NumberToAdd, (uint32_t *)InitValuePassed,
	(uint32_t *)ResNum1};
	void *ArgPtrs[] = {&Args.InitLoop, &Args.Addend, &Args.InitVal, &Args.Out};
	size_t ArgSizes[] = {sizeof(Args.InitLoop), sizeof(Args.Addend),
	sizeof(Args.InitVal), sizeof(Args.Out)};
	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ol_event_handle_t Event = nullptr;
	ASSERT_SUCCESS(olCreateEvent(Queue, OL_EVENT_FLAGS_NONE, &Event));
	ASSERT_SUCCESS(olWaitEvents(Queue2, &Event, 1));
	ArgsSingleCounter Args2{LoopRange, NumberToAdd, (uint32_t *)ResNum1,
	(uint32_t *)ResNum2};
	void *ArgPtrs2[] = {&Args2.InitLoop, &Args2.Addend, &Args2.InitVal,
	&Args2.Out};

	// At the beginning of the kernel, ResNum from the first queue is saved
	// locally as the initial value. If operations enqueued before Event have not
	// been completed by the time the kernel is executed using the second queue,
	// the FinalResVal would be incorrect.
	ASSERT_SUCCESS(olLaunchKernel(Queue2, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs2), ArgPtrs2, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue2));
	uint32_t FinalResVal = 0;
	ASSERT_SUCCESS(olMemcpy(Queue2, &FinalResVal, Host, ResNum2, Device, Size));
	ASSERT_SUCCESS(olSyncQueue(Queue2));

	ASSERT_EQ(FinalResVal, 2 * NumberToAdd * LoopRange);

	ASSERT_SUCCESS(olMemFree(InitValuePassed));
	ASSERT_SUCCESS(olMemFree(ResNum1));
	ASSERT_SUCCESS(olMemFree(ResNum2));
	}

	TEST_P(olLaunchKernelGlobalTest, Success) {
	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

	void *ArgPtrs[] = {&Mem};
	size_t ArgSizes[] = {sizeof(Mem)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernels[0], &LaunchArgs, nullptr,
	0, nullptr, nullptr));
	ASSERT_SUCCESS(olSyncQueue(Queue));
	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernels[1], &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));
	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < 64; i++) {
	ASSERT_EQ(Data[i], i * 2);
	}

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	TEST_P(olLaunchKernelGlobalTest, InvalidNotAKernel) {
	ol_symbol_handle_t Global = nullptr;
	ASSERT_SUCCESS(
	olGetSymbol(Program, "global", OL_SYMBOL_KIND_GLOBAL_VARIABLE, &Global));
	ASSERT_ERROR(OL_ERRC_SYMBOL_KIND,
	olLaunchKernel(Queue, Device, Global, &LaunchArgs, nullptr, 0,
	nullptr, nullptr));
	}

	TEST_P(olLaunchKernelGlobalCtorTest, Success) {
	SKIP_KNOWN_FAILURE(LevelZero{"unsupported feature"});

	void *Mem;
	ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
	LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));

	void *ArgPtrs[] = {&Mem};
	size_t ArgSizes[] = {sizeof(Mem)};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));
	ASSERT_SUCCESS(olSyncQueue(Queue));

	uint32_t Data = (uint32_t )Mem;
	for (uint32_t i = 0; i < 64; i++) {
	ASSERT_EQ(Data[i], i + 100);
	}

	ASSERT_SUCCESS(olMemFree(Mem));
	}

	TEST_P(olLaunchKernelGlobalDtorTest, Success) {
	// TODO: We can't inspect the result of a destructor yet, once we
	// find/implement a way, update this test. For now we just check that nothing
	// crashes
	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr, 0,
	nullptr, nullptr));
	ASSERT_SUCCESS(olSyncQueue(Queue));
	}

	TEST_P(olLaunchKernelGridSizeTest, Success) {
	void *Mem;
	ASSERT_SUCCESS(
	olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, 6 * sizeof(uint32_t), &Mem));

	uint32_t NumBlocks = static_cast<uint32_t >(Mem);
	uint32_t NumThreads = static_cast<uint32_t >(Mem) + 3;

	void *ArgPtrs[] = {&NumBlocks, &NumThreads};
	size_t ArgSizes[] = {sizeof(NumBlocks), sizeof(NumThreads)};

	const uint32_t BaseBlocks[3] = {64, 16, 8};
	const uint32_t BaseThreads[3] = {32, 4, 2};

	for (uint32_t Dim = 1; Dim <= 3; ++Dim) {
	NumBlocks[0] = NumBlocks[1] = NumBlocks[2] = 0;
	NumThreads[0] = NumThreads[1] = NumThreads[2] = 0;

	LaunchArgs.Dimensions = 3;
	LaunchArgs.NumGroups = {BaseBlocks[0], Dim >= 2 ? BaseBlocks[1] : 1,
	Dim >= 3 ? BaseBlocks[2] : 1};
	LaunchArgs.GroupSize = {BaseThreads[0], Dim >= 2 ? BaseThreads[1] : 1,
	Dim >= 3 ? BaseThreads[2] : 1};

	ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &LaunchArgs, nullptr,
	std::size(ArgPtrs), ArgPtrs, ArgSizes));

	ASSERT_SUCCESS(olSyncQueue(Queue));

	ASSERT_EQ(NumBlocks[0], LaunchArgs.NumGroups.x);
	ASSERT_EQ(NumBlocks[1], LaunchArgs.NumGroups.y);
	ASSERT_EQ(NumBlocks[2], LaunchArgs.NumGroups.z);
	ASSERT_EQ(NumThreads[0], LaunchArgs.GroupSize.x);
	ASSERT_EQ(NumThreads[1], LaunchArgs.GroupSize.y);
	ASSERT_EQ(NumThreads[2], LaunchArgs.GroupSize.z);
	}

	ASSERT_SUCCESS(olMemFree(Mem));
	}