blob: bcb404ca61a7bbd9bf0c2576d163d3d3b117e5ea [file]
// Copyright 2026 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <stddef.h>
#include <stdint.h>
#include <iostream>
#include <random>
#include <string>
#include "hwy/aligned_allocator.h"
#include "hwy/contrib/thread_pool/index_range.h"
#include "hwy/contrib/thread_pool/thread_pool.h"
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "hwy/examples/stream_triad.cc"
#include "hwy/foreach_target.h" // IWYU pragma: keep
#include "hwy/highway.h"
#include "hwy/timer.h"
/*
Highway SIMD Tutorial: Stream triad
This example demonstrates how to combine multithreading and
simd vectorization.
1) https://www.cs.virginia.edu/stream/
2) https://github.com/RRZE-HPC/likwid
*/
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {
namespace hn = hwy::HWY_NAMESPACE;
using DF = hn::ScalableTag<float>;
const DF df;
using VF = hn::Vec<DF>;
void StreamTriadScalar(float* HWY_RESTRICT a, const float* HWY_RESTRICT b,
const float* HWY_RESTRICT c, const float q,
const size_t count) {
for (size_t i = 0; i < count; ++i) {
a[i] = b[i] + q * c[i];
}
}
void StreamTriadSIMD(float* HWY_RESTRICT a, const float* HWY_RESTRICT b,
const float* HWY_RESTRICT c, const float q,
const size_t count) {
const size_t NF = hn::Lanes(df);
size_t i = 0;
const VF qv = hn::Set(df, q);
// Unroll by 4 to mask latency
HWY_DASSERT(count >= 4 * NF);
for (; i <= count - 4 * NF; i += 4 * NF) {
hn::Store(MulAdd(hn::Load(df, c + i), qv, hn::Load(df, b + i)), df, a + i);
hn::Store(MulAdd(hn::Load(df, c + i + NF), qv, hn::Load(df, b + i + NF)),
df, a + i + NF);
hn::Store(
MulAdd(hn::Load(df, c + i + 2 * NF), qv, hn::Load(df, b + i + 2 * NF)),
df, a + i + 2 * NF);
hn::Store(
MulAdd(hn::Load(df, c + i + 3 * NF), qv, hn::Load(df, b + i + 3 * NF)),
df, a + i + 3 * NF);
}
for (; i <= count - NF; i += NF) {
hn::Store(MulAdd(hn::Load(df, c + i), qv, hn::Load(df, b + i)), df, a + i);
}
// Use LoadN for remainder, sets 0 for values beyond remainder
size_t remainder = count - i;
HWY_DASSERT(remainder < NF);
if (remainder > 0) {
hn::StoreN(MulAdd(hn::LoadN(df, c + i, remainder), qv,
hn::LoadN(df, b + i, remainder)),
df, a + i, remainder);
}
}
IndexRangePartition StaticSIMDPartition(size_t num_items, size_t workers,
size_t NF) {
size_t size = hwy::RoundUpTo(hwy::DivCeil(num_items, workers), NF);
return IndexRangePartition(IndexRange(0, num_items),
HWY_MIN(size, num_items));
}
void StreamTriadSIMDThreads(float* HWY_RESTRICT a, const float* HWY_RESTRICT b,
const float* HWY_RESTRICT c, const float q,
const size_t count) {
const size_t NF = hn::Lanes(df);
ThreadPool pool(ThreadPool::MaxThreads());
size_t workers = pool.NumWorkers();
IndexRangePartition partition = StaticSIMDPartition(count, workers, NF);
pool.Run(0, workers, [=](uint64_t task, HWY_MAYBE_UNUSED size_t thread) {
const VF qv = hn::Set(df, q);
// Use work chunks that are multiples of the number of lane
const IndexRange range = partition.Range(static_cast<size_t>(task));
const size_t low = range.begin();
const size_t high = range.end();
size_t i = low;
// Unroll to hide latency
for (; i <= high - 4 * NF; i += 4 * NF) {
hn::Store(MulAdd(hn::Load(df, c + i), qv, hn::Load(df, b + i)), df,
a + i);
hn::Store(MulAdd(hn::Load(df, c + i + NF), qv, hn::Load(df, b + i + NF)),
df, a + i + NF);
hn::Store(MulAdd(hn::Load(df, c + i + 2 * NF), qv,
hn::Load(df, b + i + 2 * NF)),
df, a + i + 2 * NF);
hn::Store(MulAdd(hn::Load(df, c + i + 3 * NF), qv,
hn::Load(df, b + i + 3 * NF)),
df, a + i + 3 * NF);
}
for (; i <= high - NF; i += NF) {
hn::Store(MulAdd(hn::Load(df, c + i), qv, hn::Load(df, b + i)), df,
a + i);
}
// Use LoadN for remainder, sets 0 for values beyond remainder
size_t remainder = high - i;
HWY_DASSERT(remainder < NF);
if (remainder > 0) {
hn::StoreN(MulAdd(hn::LoadN(df, c + i, remainder), qv,
hn::LoadN(df, b + i, remainder)),
df, a + i, remainder);
}
});
}
IndexRangePartition StaticScalarPartition(size_t num_items, size_t workers) {
size_t size = hwy::DivCeil(num_items, workers);
return IndexRangePartition(IndexRange(0, num_items),
HWY_MIN(size, num_items));
}
void StreamTriadScalarThreads(float* HWY_RESTRICT a,
const float* HWY_RESTRICT b,
const float* HWY_RESTRICT c, const float q,
const size_t count) {
ThreadPool pool(ThreadPool::MaxThreads());
size_t workers = pool.NumWorkers();
IndexRangePartition partition = StaticScalarPartition(count, workers);
pool.Run(0, workers, [=](uint64_t task, HWY_MAYBE_UNUSED size_t thread) {
// Divide up work between the tasks
const IndexRange range = partition.Range(static_cast<size_t>(task));
size_t low = range.begin();
size_t high = range.end();
for (size_t i = low; i < high; ++i) {
a[i] = b[i] + q * c[i];
}
});
}
bool TriadValidate(const float* HWY_RESTRICT ref,
const float* HWY_RESTRICT check, const size_t count,
const float threshold) {
const size_t NF = hn::Lanes(df);
size_t i = 0;
VF sumdiff = hn::Zero(df);
bool ret = false;
if (count >= NF) {
for (; i <= count - NF; i += NF) {
sumdiff = hn::Add(
sumdiff, hn::AbsDiff(hn::Load(df, ref + i), hn::Load(df, check + i)));
}
}
// Use LoadN for remainder, sets 0 for values beyond remainder
size_t remainder = count - i;
HWY_DASSERT(remainder < NF);
if (remainder > 0) {
sumdiff =
hn::Add(sumdiff, hn::AbsDiff(hn::LoadN(df, ref + i, remainder),
hn::LoadN(df, check + i, remainder)));
}
const float total_diff = hn::ReduceSum(df, sumdiff);
if (total_diff <= threshold * static_cast<float>(count)) {
ret = true;
}
return ret;
}
} // namespace
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace hwy {
HWY_EXPORT(StreamTriadScalar);
HWY_EXPORT(StreamTriadScalarThreads);
HWY_EXPORT(StreamTriadSIMD);
HWY_EXPORT(StreamTriadSIMDThreads);
HWY_EXPORT(TriadValidate);
void PrintResults(const bool check_validated, const bool validated,
const std::string test_type, const double t_0,
const double t_1, const size_t count) {
const double dt = 1000.0 * (t_1 - t_0);
const double bandwidth =
3.0 * sizeof(float) * static_cast<double>(count) / (1000.0 * dt);
if (check_validated) {
if (validated) {
std::cout << test_type << " validated" << std::endl;
} else {
std::cout << test_type << " validation failed" << std::endl;
}
}
std::cout << test_type << " execution time: " << dt << " ms" << std::endl;
std::cout << test_type << " bandwidth: " << bandwidth << " MB/s" << std::endl;
}
int Run() {
const size_t count = 20000025;
const float threshold = 0.00001f;
AlignedVector<float> a_scalar(count);
AlignedVector<float> a_simd(count);
AlignedVector<float> a_scalar_threads(count);
AlignedVector<float> a_simd_threads(count);
AlignedVector<float> b(count);
AlignedVector<float> c(count);
std::mt19937 gen(42);
std::uniform_real_distribution<float> dis(-10.5f, 10.5f);
for (size_t i = 0; i < count; ++i) {
b[i] = dis(gen);
c[i] = dis(gen);
}
int ret = 1;
const float q = 3.142f;
// Record start time
const double t_scalar_0 = hwy::platform::Now();
HWY_DYNAMIC_DISPATCH(StreamTriadScalar)(a_scalar.data(), b.data(), c.data(),
q, count);
// Record end time and print execution time and dot product
const double t_scalar_1 = hwy::platform::Now();
PrintResults(false, false, "Scalar", t_scalar_0, t_scalar_1, count);
// Record start time
const double t_simd_0 = hwy::platform::Now();
HWY_DYNAMIC_DISPATCH(StreamTriadSIMD)(a_simd.data(), b.data(), c.data(), q,
count);
// Record stop timer and print time
const double t_simd_1 = hwy::platform::Now();
const bool simd_validate = HWY_DYNAMIC_DISPATCH(TriadValidate)(
a_scalar.data(), a_simd.data(), count, threshold);
PrintResults(true, simd_validate, "SIMD", t_simd_0, t_simd_1, count);
// Record start time
const double t_scalar_threads_0 = hwy::platform::Now();
HWY_DYNAMIC_DISPATCH(StreamTriadScalar)(a_scalar_threads.data(), b.data(),
c.data(), q, count);
// Record end time and print execution time and dot product
const double t_scalar_threads_1 = hwy::platform::Now();
const bool scalar_threads_validate = HWY_DYNAMIC_DISPATCH(TriadValidate)(
a_scalar.data(), a_scalar_threads.data(), count, threshold);
PrintResults(true, scalar_threads_validate, "Threaded scalar",
t_scalar_threads_0, t_scalar_threads_1, count);
// Record start time
const double t_simd_threads_0 = hwy::platform::Now();
HWY_DYNAMIC_DISPATCH(StreamTriadSIMDThreads)(a_simd_threads.data(), b.data(),
c.data(), q, count);
// Record stop timer and print time
const double t_simd_threads_1 = hwy::platform::Now();
const bool simd_threads_validate = HWY_DYNAMIC_DISPATCH(TriadValidate)(
a_scalar.data(), a_simd_threads.data(), count, threshold);
PrintResults(true, simd_threads_validate, "Threaded SIMD", t_simd_threads_0,
t_simd_threads_1, count);
if (simd_validate && scalar_threads_validate && simd_threads_validate) {
ret = 0;
}
return ret;
}
} // namespace hwy
int main() { return hwy::Run(); }
#endif // HWY_ONCE