blob: 8027882cffe7de0352af5044eff14203bf8bf2e7 [file]
// Copyright 2026 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hwy/io/io_batch.h"
#include <stddef.h>
#include <stdint.h>
#include <algorithm>
#include <type_traits>
#include <utility>
#include <vector>
#include "hwy/base.h"
#include "hwy/io/file.h"
#include "hwy/io/io_span.h"
#include "hwy/io/io_test_util.h"
#include "hwy/io/memory_file.h"
#include "hwy/status.h"
#include "hwy/tests/hwy_gtest.h"
#include "hwy/tests/test_util-inl.h"
namespace hwy {
namespace {
TEST(IOBatchTest, TestIOBatchMemoryFile) {
IOWorkerBuf io_buf;
std::vector<uint8_t> src(256);
for (size_t i = 0; i < src.size(); ++i) {
src[i] = static_cast<uint8_t>(i);
}
MemoryFile ro_file(src);
// Contiguous spans coalesce into a single span (so > 256 contiguous additions
// succeed) and empty spans are ignored without consuming a slot.
uint8_t contiguous_dst[40] = {};
IOBatch contiguous_batch(/*offset=*/10, /*user_tag=*/3);
HWY_ASSERT(contiguous_batch.IsEmpty());
HWY_ASSERT(contiguous_batch.Add(Span<uint8_t>()));
HWY_ASSERT(contiguous_batch.IsEmpty());
for (size_t i = 0; i < sizeof(contiguous_dst); ++i) {
HWY_ASSERT(contiguous_batch.Add(Span<uint8_t>(contiguous_dst + i, 1)));
}
HWY_ASSERT(!contiguous_batch.IsEmpty());
HWY_ASSERT_EQ(uint64_t{10}, contiguous_batch.Offset());
HWY_ASSERT_EQ(uint64_t{40}, contiguous_batch.TotalBytes());
HWY_ASSERT_EQ(uint64_t{3}, contiguous_batch.UserTag());
HWY_ASSERT_EQ(uint64_t{50}, contiguous_batch.NextOffset());
HWY_ASSERT(contiguous_batch.Read(ro_file, io_buf) == Status(40));
for (size_t i = 0; i < sizeof(contiguous_dst); ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(10 + i), contiguous_dst[i]);
}
// Discontiguous spans: slow path. Reuse `contiguous_batch` via `Reset`.
constexpr size_t kLen1 = 16;
constexpr size_t kGap = 8;
constexpr size_t kLen2 = 24;
constexpr uint8_t kSentinel = 0xEE;
std::vector<uint8_t> dst(kLen1 + kGap + kLen2, kSentinel);
contiguous_batch.Reset(/*offset=*/32);
HWY_ASSERT(contiguous_batch.IsEmpty());
HWY_ASSERT_EQ(uint64_t{0}, contiguous_batch.UserTag());
HWY_ASSERT(contiguous_batch.Add(Span<uint8_t>(dst.data(), kLen1)));
HWY_ASSERT(
contiguous_batch.Add(Span<uint8_t>(dst.data() + kLen1 + kGap, kLen2)));
HWY_ASSERT(contiguous_batch.Read(ro_file, io_buf) == Status(kLen1 + kLen2));
for (size_t i = 0; i < kLen1; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + i), dst[i]);
}
for (size_t i = 0; i < kGap; ++i) {
HWY_ASSERT_EQ(kSentinel, dst[kLen1 + i]); // Gap untouched.
}
for (size_t i = 0; i < kLen2; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + kLen1 + i), dst[kLen1 + kGap + i]);
}
// Reading past the end of the file fails.
uint8_t past_end[16];
IOBatch past_batch(/*offset=*/250);
HWY_ASSERT(past_batch.Add(Span<uint8_t>(past_end, sizeof(past_end))));
HWY_ASSERT(past_batch.Read(ro_file, io_buf) == Status(Status::kTruncated, 6));
// Discontiguous spans past EOF: the payload is the total bytes read, and the
// available prefix is scattered into the spans; the rest is untouched.
std::vector<uint8_t> partial(10 + kGap + 10, kSentinel);
IOBatch partial_batch(/*offset=*/240);
HWY_ASSERT(partial_batch.Add(Span<uint8_t>(partial.data(), 10)));
HWY_ASSERT(partial_batch.Add(Span<uint8_t>(partial.data() + 10 + kGap, 10)));
HWY_ASSERT(partial_batch.Read(ro_file, io_buf) ==
Status(Status::kTruncated, 16));
for (size_t i = 0; i < 10; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(240 + i), partial[i]);
}
for (size_t i = 0; i < kGap; ++i) {
HWY_ASSERT_EQ(kSentinel, partial[10 + i]);
}
for (size_t i = 0; i < 6; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(250 + i), partial[10 + kGap + i]);
}
for (size_t i = 6; i < 10; ++i) {
HWY_ASSERT_EQ(kSentinel, partial[10 + kGap + i]);
}
// Per-worker `IOWorkerBuf` in `ReadSpans` / `IOBatch::Read`: starts empty,
// stays empty on single-span fast path, and retains high-water-mark capacity
// across multi-span fallback reads.
static_assert(std::is_constructible<IOSpans, IOSpan(&)[4]>::value,
"C array must construct IOSpans");
static_assert(std::is_constructible<IOSpans, Span<IOSpan>>::value,
"Span<IOSpan> must construct IOSpans");
static_assert(!std::is_constructible<IOSpans, std::vector<IOSpan>&&>::value,
"rvalue container must be rejected");
static_assert(
!std::is_constructible<IOSpans, Span<const IOSpan>, uint64_t>::value,
"unchecked 2-arg IOSpans constructor must be private");
static_assert(!std::is_constructible<IOSpans, Span<IOSpan>, size_t>::value,
"unchecked 2-arg IOSpans constructor must be private");
IOWorkerBuf worker_buf;
HWY_ASSERT_EQ(size_t{0}, worker_buf.Capacity());
HWY_ASSERT(worker_buf.Ensure(0) == nullptr);
HWY_ASSERT_EQ(size_t{0}, worker_buf.Capacity());
IOBatch single_batch(/*offset=*/0);
HWY_ASSERT(single_batch.Add(Span<uint8_t>(contiguous_dst, 16)));
HWY_ASSERT(single_batch.Read(ro_file, worker_buf) == Status(16));
HWY_ASSERT_EQ(size_t{0}, worker_buf.Capacity());
std::fill(dst.begin(), dst.end(), kSentinel);
HWY_ASSERT(contiguous_batch.Read(ro_file, worker_buf) ==
Status(kLen1 + kLen2));
HWY_ASSERT_EQ(kLen1 + kLen2, worker_buf.Capacity());
for (size_t i = 0; i < kLen1; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + i), dst[i]);
}
for (size_t i = 0; i < kLen2; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + kLen1 + i), dst[kLen1 + kGap + i]);
}
// Growing to a larger multi-span batch reallocates once to the new high-water
// mark; subsequent smaller batches reuse it without shrinking.
std::vector<uint8_t> larger_dst(30 + kGap + 30, kSentinel);
IOBatch larger_batch(/*offset=*/0);
HWY_ASSERT(larger_batch.Add(Span<uint8_t>(larger_dst.data(), 30)));
HWY_ASSERT(
larger_batch.Add(Span<uint8_t>(larger_dst.data() + 30 + kGap, 30)));
HWY_ASSERT(larger_batch.Read(ro_file, worker_buf) == Status(60));
HWY_ASSERT_EQ(size_t{60}, worker_buf.Capacity());
HWY_ASSERT_ARRAY_EQ(src.data(), larger_dst.data(), 30);
HWY_ASSERT_ARRAY_EQ(src.data() + 30, larger_dst.data() + 30 + kGap, 30);
// Direct File::ReadSpans call with empty IOSpan{nullptr, 0} preceding and
// separating non-empty spans must skip the empty spans and scatter the rest.
// Also exercises IOSpans construction from C arrays and Span<IOSpan>, and
// custom File backends calling IOWorkerBuf::Ensure directly.
std::fill(dst.begin(), dst.end(), kSentinel);
IOSpan mixed_spans[4] = {
{nullptr, 0},
{dst.data(), kLen1},
{nullptr, 0},
{dst.data() + kLen1 + kGap, kLen2},
};
const IOSpans mixed_from_mutable((Span<IOSpan>(mixed_spans)));
HWY_ASSERT_EQ(kLen1 + kLen2, mixed_from_mutable.TotalBytes());
HWY_ASSERT(ro_file.ReadSpans(/*offset=*/32, IOSpans(mixed_spans), io_buf) ==
Status(kLen1 + kLen2));
for (size_t i = 0; i < kLen1; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + i), dst[i]);
}
for (size_t i = 0; i < kGap; ++i) {
HWY_ASSERT_EQ(kSentinel, dst[kLen1 + i]);
}
for (size_t i = 0; i < kLen2; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + kLen1 + i), dst[kLen1 + kGap + i]);
}
struct CustomScratchFile : public MemoryFile {
using MemoryFile::MemoryFile;
Status ReadSpans(uint64_t offset, const IOSpans& spans,
IOWorkerBuf& buf) const override {
const Status total = spans.ValidateAt(offset);
if (!total.ok()) return total;
if (total.size() == 0) return MemoryFile::ReadSpans(offset, spans, buf);
const size_t want = static_cast<size_t>(total.size());
uint8_t* scratch = buf.Ensure(want);
HWY_ASSERT(scratch != nullptr);
const Status s = Read(offset, Span<uint8_t>(scratch, want));
if (!s.ok()) return s;
size_t pos = 0;
for (const IOSpan& span : spans) {
if (span.bytes != 0) CopyBytes(scratch + pos, span.mem, span.bytes);
pos += span.bytes;
}
return s;
}
};
std::fill(dst.begin(), dst.end(), kSentinel);
CustomScratchFile custom_ro(src);
IOWorkerBuf custom_buf;
HWY_ASSERT(custom_ro.ReadSpans(/*offset=*/32, IOSpans(mixed_spans),
custom_buf) == Status(kLen1 + kLen2));
HWY_ASSERT_EQ(kLen1 + kLen2, custom_buf.Capacity());
HWY_ASSERT(custom_buf.Ensure(0) != nullptr);
for (size_t i = 0; i < kLen1; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + i), dst[i]);
}
for (size_t i = 0; i < kLen2; ++i) {
HWY_ASSERT_EQ(static_cast<uint8_t>(32 + kLen1 + i), dst[kLen1 + kGap + i]);
}
// Gather write via IOBatch::Write and AddForWrite.
std::vector<uint8_t> out_buf(128, 0);
MemoryFile rw_file(out_buf, FileMode::kWriteTruncate);
IOBatch write_batch(/*offset=*/8);
HWY_ASSERT(write_batch.AddForWrite(Span<const uint8_t>(src.data() + 10, 12)));
HWY_ASSERT(write_batch.AddForWrite(Span<const uint8_t>(src.data() + 40, 20)));
// Reading into a write-only batch fails with kWrongMode.
HWY_ASSERT(write_batch.Read(ro_file, io_buf).Is(IOCode::kWrongMode));
HWY_ASSERT(write_batch.Read(ro_file, worker_buf).Is(IOCode::kWrongMode));
HWY_ASSERT(write_batch.Write(ro_file).Is(IOCode::kWrongMode));
HWY_ASSERT(write_batch.Write(rw_file) == Status(32));
HWY_ASSERT(rw_file.Size() == Status(40));
HWY_ASSERT_ARRAY_EQ(src.data() + 10, out_buf.data() + 8, 12);
HWY_ASSERT_ARRAY_EQ(src.data() + 40, out_buf.data() + 20, 20);
// IOSpans overflow validation (> Status::kMaxPayload) fails with kArgument.
if (ScalarFits<size_t>(Status::kMaxPayload)) {
const size_t kMaxPay = static_cast<size_t>(Status::kMaxPayload);
const std::vector<IOSpan> bad_vec = {{nullptr, kMaxPay}, {nullptr, 1}};
const IOSpans bad_spans(bad_vec);
HWY_ASSERT(!bad_spans.IsValid());
HWY_ASSERT(ro_file.ReadSpans(0, bad_spans, io_buf).Is(Status::kArgument));
HWY_ASSERT(rw_file.WriteSpans(0, bad_spans).Is(Status::kArgument));
}
// AddStrided: packed fast path (stride_bytes == bytes_per_row) coalesces into
// a single span; padded path (stride_bytes > bytes_per_row) scatters/gathers
// row by row leaving padding untouched.
IOBatch strided_batch(/*offset=*/0);
HWY_ASSERT_EQ(size_t{0}, strided_batch.AddStrided(dst.data(), /*rows=*/0,
/*bytes_per_row=*/4,
/*stride_bytes=*/8));
HWY_ASSERT_EQ(size_t{3}, strided_batch.AddStrided(dst.data(), /*rows=*/3,
/*bytes_per_row=*/0,
/*stride_bytes=*/8));
HWY_ASSERT(strided_batch.IsEmpty());
// Empty read/write batches after 0-byte Add/AddStrided succeed with Status(0)
// rather than aborting.
HWY_ASSERT(strided_batch.Read(ro_file, io_buf) == Status(0));
HWY_ASSERT(strided_batch.Write(rw_file) == Status(0));
HWY_ASSERT(strided_batch.Write(ro_file).Is(IOCode::kWrongMode));
std::vector<uint8_t> packed(4 * 6, 0);
HWY_ASSERT_EQ(size_t{4}, strided_batch.AddStrided(packed.data(), /*rows=*/4,
/*bytes_per_row=*/6,
/*stride_bytes=*/6));
HWY_ASSERT_EQ(size_t{1}, strided_batch.Spans().size());
HWY_ASSERT(strided_batch.Read(ro_file, io_buf) == Status(24));
HWY_ASSERT_ARRAY_EQ(src.data(), packed.data(), 24);
constexpr size_t kRows = 5;
constexpr size_t kRowBytes = 7;
constexpr size_t kStride = 16;
std::vector<uint8_t> padded(kRows * kStride, kSentinel);
strided_batch.Reset(/*offset=*/20);
HWY_ASSERT_EQ(kRows, strided_batch.AddStrided(padded.data(), kRows, kRowBytes,
kStride));
HWY_ASSERT_EQ(kRows, strided_batch.Spans().size());
HWY_ASSERT(strided_batch.Read(ro_file, worker_buf) ==
Status(kRows * kRowBytes));
// High-water-mark capacity (60) is retained across smaller (35-byte) reads.
HWY_ASSERT_EQ(size_t{60}, worker_buf.Capacity());
IOWorkerBuf moved_buf = std::move(worker_buf);
HWY_ASSERT_EQ(size_t{60}, moved_buf.Capacity());
// NOLINTNEXTLINE(bugprone-use-after-move)
HWY_ASSERT_EQ(size_t{0}, worker_buf.Capacity());
moved_buf.Release();
HWY_ASSERT_EQ(size_t{0}, moved_buf.Capacity());
for (size_t r = 0; r < kRows; ++r) {
HWY_ASSERT_ARRAY_EQ(src.data() + 20 + r * kRowBytes,
padded.data() + r * kStride, kRowBytes);
for (size_t p = kRowBytes; p < kStride; ++p) {
HWY_ASSERT_EQ(kSentinel, padded[r * kStride + p]);
}
}
// Gather write from padded rows into a packed output buffer.
std::vector<uint8_t> gathered(kRows * kRowBytes, 0);
MemoryFile gather_file(gathered, FileMode::kWriteTruncate);
IOBatch strided_write(/*offset=*/0);
// Zero-byte or zero-row write additions keep the batch empty (`Read`/`Write`
// return `Status(0)`) without marking the batch write-only until bytes are
// actually added (matching `AddForWrite(empty)`).
HWY_ASSERT(strided_write.AddForWrite(Span<const uint8_t>()));
HWY_ASSERT_EQ(size_t{0}, strided_write.AddStridedForWrite(
padded.data(), /*rows=*/0, kRowBytes, kStride));
HWY_ASSERT_EQ(kRows,
strided_write.AddStridedForWrite(padded.data(), kRows,
/*bytes_per_row=*/0, kStride));
HWY_ASSERT(strided_write.IsEmpty());
HWY_ASSERT(strided_write.Read(ro_file, io_buf) == Status(0));
HWY_ASSERT(strided_write.Write(gather_file) == Status(0));
HWY_ASSERT_EQ(kRows, strided_write.AddStridedForWrite(padded.data(), kRows,
kRowBytes, kStride));
HWY_ASSERT(strided_write.Read(ro_file, io_buf).Is(IOCode::kWrongMode));
HWY_ASSERT(strided_write.Write(gather_file) == Status(kRows * kRowBytes));
HWY_ASSERT_ARRAY_EQ(src.data() + 20, gathered.data(), kRows * kRowBytes);
}
TEST(IOBatchTest, TestIOBatchFirstSpanAlwaysAccepted) {
IOWorkerBuf io_buf;
IOBatch batch(/*offset=*/0);
// Larger than the batch limit, but must still be accepted as the first span.
// Not read, so the pointer is never dereferenced.
const size_t kHuge = static_cast<size_t>(
HWY_MIN(uint64_t{0x80000000}, static_cast<uint64_t>(~size_t{0})));
HWY_ASSERT(batch.Add(Span<uint8_t>(static_cast<uint8_t*>(nullptr), kHuge)));
// Subsequent spans exceeding the limit are rejected.
uint8_t one;
HWY_ASSERT(!batch.Add(Span<uint8_t>(&one, 1)));
// More than IOV_MAX discontiguous spans are accepted by Add and chunked by
// ReadSpans.
#ifdef IOV_MAX
const size_t kMaxSpans = IOV_MAX;
#else
const size_t kMaxSpans = 1024;
#endif
std::vector<uint8_t> src(kMaxSpans + 5);
for (size_t i = 0; i < src.size(); ++i) src[i] = static_cast<uint8_t>(i);
std::vector<uint8_t> discontiguous((kMaxSpans + 5) * 2, 0);
IOBatch max_spans_batch(/*offset=*/0);
for (size_t i = 0; i < kMaxSpans + 5; ++i) {
HWY_ASSERT(max_spans_batch.Add(Span<uint8_t>(&discontiguous[i * 2], 1)));
}
MemoryFile src_file(src);
HWY_ASSERT(max_spans_batch.Read(src_file, io_buf) == Status(kMaxSpans + 5));
for (size_t i = 0; i < kMaxSpans + 5; ++i) {
HWY_ASSERT_EQ(src[i], discontiguous[i * 2]);
}
// Subsequent spans whose offset + total_bytes would exceed int64_t (off_t)
// are rejected by Add, and a single span exceeding off_t is rejected by Read.
const uint64_t kMaxOff = static_cast<uint64_t>(LimitsMax<int64_t>());
IOBatch overflow_batch(kMaxOff - 1);
HWY_ASSERT(overflow_batch.Add(Span<uint8_t>(&one, 1)));
HWY_ASSERT(!overflow_batch.Add(Span<uint8_t>(&one, 1)));
// A failed AddForWrite does not mark the batch read-only.
HWY_ASSERT(!overflow_batch.AddForWrite(Span<const uint8_t>(&one, 1)));
MemoryFile empty_file((Span<const uint8_t>()));
HWY_ASSERT(overflow_batch.Read(empty_file, io_buf) ==
Status(Status::kTruncated, 0));
overflow_batch.Reset(kMaxOff);
HWY_ASSERT(overflow_batch.Add(Span<uint8_t>(&one, 1)));
HWY_ASSERT(overflow_batch.Read(empty_file, io_buf).Is(Status::kArgument));
}
} // namespace
} // namespace hwy
HWY_TEST_MAIN();