Store primitive GC arrays as raw byte buffers (#9051)
GCData previously represented all allocations using a vector of
Literals. Storing numeric array elements this way introduces unnecessary
memory overhead and prevents efficient byte-level operations.
Represent primitive numeric GC arrays using a raw byte buffer in GCData
while preserving Literals storage for reference arrays and structs.
diff --git a/src/literal.h b/src/literal.h
index adb8e2f..335e117 100644
--- a/src/literal.h
+++ b/src/literal.h
@@ -19,6 +19,7 @@
#include <array>
#include <iostream>
+#include <variant>
#include "support/bits.h"
#include "support/hash.h"
@@ -212,7 +213,7 @@
}
}
- static Literal makeFromMemory(void* p, Type type);
+ static Literal makeFromMemory(const void* p, Type type);
static Literal makeSignedMin(Type type) {
switch (type.getBasic()) {
@@ -312,6 +313,12 @@
Name getFunc() const;
std::shared_ptr<FuncData> getFuncData() const;
std::shared_ptr<GCData> getGCData() const;
+ size_t getNumElements() const;
+ Literal getElement(size_t index, bool signed_ = false) const;
+ void setElement(size_t index, Literal value);
+ bool isRawBytes() const;
+ const std::vector<uint8_t>& getRawBytes() const;
+ std::vector<uint8_t>& getRawBytes();
std::shared_ptr<ExnData> getExnData() const;
std::shared_ptr<ContData> getContData() const;
@@ -784,28 +791,65 @@
// A GC Struct, Array, or String is a set of values with a type saying how it
// should be interpreted.
struct GCData {
- // The element or field values.
- Literals values;
+ // The element or field values. Primitive numeric arrays use raw byte buffers
+ // (std::vector<uint8_t>), while reference arrays, structs, strings, and other
+ // reference allocations use Literals.
+ std::variant<std::vector<uint8_t>, Literals> storage;
// The descriptor, if it exists, or null.
Literal desc;
GCData(Literals&& values,
const Literal& desc = Literal::makeNull(HeapType::none))
- : values(std::move(values)), desc(desc) {}
+ : storage(std::move(values)), desc(desc) {}
+
+ GCData(std::vector<uint8_t>&& data,
+ const Literal& desc = Literal::makeNull(HeapType::none))
+ : storage(std::move(data)), desc(desc) {}
+
+ bool isRawBytes() const {
+ return std::holds_alternative<std::vector<uint8_t>>(storage);
+ }
+
+ const std::vector<uint8_t>& getRawBytes() const {
+ return std::get<std::vector<uint8_t>>(storage);
+ }
+
+ std::vector<uint8_t>& getRawBytes() {
+ return std::get<std::vector<uint8_t>>(storage);
+ }
+
+ const Literals& getLiterals() const { return std::get<Literals>(storage); }
+
+ Literals& getLiterals() { return std::get<Literals>(storage); }
};
+inline bool Literal::isRawBytes() const {
+ assert(isData());
+ return gcData->isRawBytes();
+}
+
+inline const std::vector<uint8_t>& Literal::getRawBytes() const {
+ assert(isData());
+ return gcData->getRawBytes();
+}
+
+inline std::vector<uint8_t>& Literal::getRawBytes() {
+ assert(isData());
+ return gcData->getRawBytes();
+}
+
inline bool Literal::hasExternPayload() const {
if (isNull()) {
return false;
}
assert(type.getHeapType().isMaybeShared(HeapType::ext));
- return gcData->values[0].type == Type::i32;
+ return gcData->getLiterals()[0].type == Type::i32;
}
inline int32_t Literal::getExternPayload() const {
assert(hasExternPayload());
- return gcData->values[0].geti32();
+ return gcData->getLiterals()[0].geti32();
}
} // namespace wasm
@@ -869,7 +913,7 @@
return digest;
}
if (a.type.isString()) {
- auto& values = a.getGCData()->values;
+ auto& values = a.getGCData()->getLiterals();
wasm::rehash(digest, values.size());
for (auto c : values) {
wasm::rehash(digest, c.getInteger());
diff --git a/src/passes/Precompute.cpp b/src/passes/Precompute.cpp
index 8f550bd..a1357e6 100644
--- a/src/passes/Precompute.cpp
+++ b/src/passes/Precompute.cpp
@@ -1103,7 +1103,7 @@
// string.
bool isValidUTF16Literal(const Literal& value) {
bool expectLowSurrogate = false;
- for (auto& v : value.getGCData()->values) {
+ for (auto& v : value.getGCData()->getLiterals()) {
auto c = v.getInteger();
if (c >= 0xDC00 && c <= 0xDFFF) {
if (expectLowSurrogate) {
diff --git a/src/tools/wasm-ctor-eval.cpp b/src/tools/wasm-ctor-eval.cpp
index f5b05c9..9025832 100644
--- a/src/tools/wasm-ctor-eval.cpp
+++ b/src/tools/wasm-ctor-eval.cpp
@@ -912,7 +912,6 @@
} else {
// This is the first usage of this data. Generate a struct.new /
// array.new for it.
- auto& values = data->values;
std::vector<Expression*> args;
// The initial values for this allocation may themselves be GC
@@ -934,8 +933,8 @@
definingGlobals[data] = DefiningGlobalInfo{definingGlobalName, type};
}
- for (auto& value : values) {
- auto* serialized = getSerialization(value);
+ for (size_t i = 0; i < value.getNumElements(); i++) {
+ auto* serialized = getSerialization(value.getElement(i));
if (!serialized) {
return nullptr;
}
diff --git a/src/wasm-builder.h b/src/wasm-builder.h
index d060dd6..56db519 100644
--- a/src/wasm-builder.h
+++ b/src/wasm-builder.h
@@ -1511,7 +1511,7 @@
// The string is already WTF-16, but we need to convert from `Literals` to
// actual string.
std::stringstream wtf16;
- for (auto c : value.getGCData()->values) {
+ for (auto c : value.getGCData()->getLiterals()) {
auto u = c.getInteger();
assert(u < 0x10000);
wtf16 << uint8_t(u & 0xFF);
diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h
index 56fccae..fbf63a9 100644
--- a/src/wasm-interpreter.h
+++ b/src/wasm-interpreter.h
@@ -370,10 +370,11 @@
// this function in LSan.
//
// This consumes the input |data| entirely.
- Literal makeGCData(Literals&& data,
+ template<typename T>
+ Literal makeGCData(T&& data,
Type type,
Literal desc = Literal::makeNull(HeapType::none)) {
- auto allocation = std::make_shared<GCData>(std::move(data), desc);
+ auto allocation = std::make_shared<GCData>(std::forward<T>(data), desc);
#if __has_feature(leak_sanitizer) || __has_feature(address_sanitizer)
// GC data with cycles will leak, since shared_ptrs do not handle cycles.
// Binaryen is generally not used in long-running programs so we just ignore
@@ -384,6 +385,23 @@
return Literal(allocation, type.getHeapType());
}
+ Literal makeGCArray(Type type,
+ size_t num,
+ Literal desc = Literal::makeNull(HeapType::none)) {
+ auto heapType = type.getHeapType();
+ const auto& element = heapType.getArray().element;
+ if (element.type.isRef()) {
+ Literals data(num);
+ auto zero = Literal::makeZero(element.type);
+ for (size_t i = 0; i < num; ++i) {
+ data[i] = zero;
+ }
+ return makeGCData(std::move(data), type, desc);
+ }
+ size_t elemBytes = element.getByteSize();
+ return makeGCData(std::vector<uint8_t>(num * elemBytes, 0), type, desc);
+ }
+
// Same as makeGCData but for ExnData.
Literal makeExnData(Tag* tag, const Literals& payload) {
auto allocation = std::make_shared<ExnData>(tag, payload);
@@ -393,20 +411,6 @@
return Literal(allocation);
}
- template<typename T>
- void writeBytes(T value, int numBytes, size_t index, Literals& values) {
- if constexpr (std::is_same_v<T, std::array<uint8_t, 16>>) {
- for (int i = 0; i < numBytes; ++i) {
- values[index + i] = Literal(static_cast<int32_t>(value[i]));
- }
- } else {
- for (int i = 0; i < numBytes; ++i) {
- values[index + i] =
- Literal(static_cast<int32_t>((value >> (i * 8)) & 0xff));
- }
- }
- }
-
static Literal applyRMW(AtomicRMWOp op, Literal lhs, Literal rhs) {
switch (op) {
case RMWAdd:
@@ -2276,7 +2280,8 @@
trap("null ref");
}
auto field = curr->ref->type.getHeapType().getStruct().fields[curr->index];
- return extendForPacking(data->values[curr->index], field, curr->signed_);
+ return extendForPacking(
+ data->getLiterals()[curr->index], field, curr->signed_);
}
Flow visitStructSet(StructSet* curr) {
VISIT(ref, curr->ref)
@@ -2286,7 +2291,7 @@
trap("null ref");
}
auto field = curr->ref->type.getHeapType().getStruct().fields[curr->index];
- data->values[curr->index] =
+ data->getLiterals()[curr->index] =
truncateForPacking(value.getSingleValue(), field);
return Flow();
}
@@ -2298,7 +2303,7 @@
if (!data) {
trap("null ref");
}
- auto& field = data->values[curr->index];
+ auto& field = data->getLiterals()[curr->index];
auto oldVal = field;
field = applyRMW(curr->op, oldVal, value.getSingleValue());
return oldVal;
@@ -2312,7 +2317,7 @@
if (!data) {
trap("null ref");
}
- auto& field = data->values[curr->index];
+ auto& field = data->getLiterals()[curr->index];
auto oldVal = field;
if (field == expected.getSingleValue()) {
field = replacement.getSingleValue();
@@ -2337,7 +2342,7 @@
if (!waitqueue.getSingleValue().getGCData()) {
trap("null ref");
}
- auto& field = data->values[curr->index];
+ auto& field = data->getLiterals()[curr->index];
if (field != expected.getSingleValue()) {
return Literal(int32_t{1}); // not equal
}
@@ -2354,7 +2359,7 @@
}
Flow visitWaitqueueNew(WaitqueueNew* curr) {
- return self()->makeGCData({},
+ return self()->makeGCData(Literals{},
Type(HeapTypes::sharedWaitqueue, NonNullable));
}
@@ -2394,26 +2399,18 @@
assert(init.breaking());
return init;
}
- auto heapType = curr->type.getHeapType();
- const auto& element = heapType.getArray().element;
Index num = size.getSingleValue().geti32();
if (num >= DataLimit) {
hostLimit("allocation failure");
}
- Literals data(num);
- if (curr->isWithDefault()) {
- auto zero = Literal::makeZero(element.type);
+ auto ret = makeGCArray(curr->type, num);
+ if (!curr->isWithDefault()) {
+ Literal val = init.getSingleValue();
for (Index i = 0; i < num; i++) {
- data[i] = zero;
- }
- } else {
- auto field = curr->type.getHeapType().getArray().element;
- auto value = truncateForPacking(init.getSingleValue(), field);
- for (Index i = 0; i < num; i++) {
- data[i] = value;
+ ret.setElement(i, val);
}
}
- return makeGCData(std::move(data), curr->type);
+ return ret;
}
Flow visitArrayNewData(ArrayNewData* curr) { WASM_UNREACHABLE("unimp"); }
Flow visitArrayNewElem(ArrayNewElem* curr) { WASM_UNREACHABLE("unimp"); }
@@ -2430,91 +2427,105 @@
}
WASM_UNREACHABLE("unreachable but no unreachable child");
}
- auto heapType = curr->type.getHeapType();
- auto field = heapType.getArray().element;
- Literals data(num);
+ auto ret = makeGCArray(curr->type, num);
for (Index i = 0; i < num; i++) {
VISIT(value, curr->values[i])
- data[i] = truncateForPacking(value.getSingleValue(), field);
+ ret.setElement(i, value.getSingleValue());
}
- return makeGCData(std::move(data), curr->type);
+ return ret;
}
Flow visitArrayGet(ArrayGet* curr) {
VISIT(ref, curr->ref)
VISIT(index, curr->index)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
Index i = index.getSingleValue().geti32();
- if (i >= data->values.size()) {
+ if (i >= refVal.getNumElements()) {
trap("array oob");
}
- auto field = curr->ref->type.getHeapType().getArray().element;
- return extendForPacking(data->values[i], field, curr->signed_);
+ return refVal.getElement(i, curr->signed_);
}
Flow visitArraySet(ArraySet* curr) {
VISIT(ref, curr->ref)
VISIT(index, curr->index)
VISIT(value, curr->value)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
Index i = index.getSingleValue().geti32();
- if (i >= data->values.size()) {
+ if (i >= refVal.getNumElements()) {
trap("array oob");
}
- auto field = curr->ref->type.getHeapType().getArray().element;
- data->values[i] = truncateForPacking(value.getSingleValue(), field);
+ refVal.setElement(i, value.getSingleValue());
return Flow();
}
Flow visitArrayLoad(ArrayLoad* curr) {
VISIT(ref, curr->ref)
VISIT(index, curr->index)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
Index i = index.getSingleValue().geti32();
- size_t size = data->values.size();
+ size_t size = refVal.getRawBytes().size();
if (i >= size || curr->bytes > (size - i)) {
trap("array oob");
}
- uint64_t val = 0;
- for (unsigned b = 0; b < curr->bytes; ++b) {
- val |= static_cast<uint64_t>(data->values[i + b].geti32()) << (b * 8);
- }
+ const uint8_t* p = &refVal.getRawBytes()[i];
switch (curr->type.getBasic()) {
case Type::i32: {
- int32_t sval = static_cast<int32_t>(val);
- if (curr->signed_) {
- if (curr->bytes == 1) {
- sval = static_cast<int32_t>(static_cast<int8_t>(sval));
- } else if (curr->bytes == 2) {
- sval = static_cast<int32_t>(static_cast<int16_t>(sval));
- }
+ switch (curr->bytes) {
+ case 1:
+ return curr->signed_ ? Literal((int32_t)Bits::readLE<int8_t>(p))
+ : Literal((int32_t)Bits::readLE<uint8_t>(p));
+ case 2:
+ return curr->signed_ ? Literal((int32_t)Bits::readLE<int16_t>(p))
+ : Literal((int32_t)Bits::readLE<uint16_t>(p));
+ case 4:
+ return Literal(Bits::readLE<int32_t>(p));
+ default:
+ WASM_UNREACHABLE("invalid size");
}
- return Literal(sval);
}
case Type::i64: {
- int64_t sval = static_cast<int64_t>(val);
- if (curr->signed_) {
- if (curr->bytes == 1) {
- sval = static_cast<int64_t>(static_cast<int8_t>(sval));
- } else if (curr->bytes == 2) {
- sval = static_cast<int64_t>(static_cast<int16_t>(sval));
- } else if (curr->bytes == 4) {
- sval = static_cast<int64_t>(static_cast<int32_t>(sval));
- }
+ switch (curr->bytes) {
+ case 1:
+ return curr->signed_ ? Literal((int64_t)Bits::readLE<int8_t>(p))
+ : Literal((int64_t)Bits::readLE<uint8_t>(p));
+ case 2:
+ return curr->signed_ ? Literal((int64_t)Bits::readLE<int16_t>(p))
+ : Literal((int64_t)Bits::readLE<uint16_t>(p));
+ case 4:
+ return curr->signed_ ? Literal((int64_t)Bits::readLE<int32_t>(p))
+ : Literal((int64_t)Bits::readLE<uint32_t>(p));
+ case 8:
+ return Literal(Bits::readLE<int64_t>(p));
+ default:
+ WASM_UNREACHABLE("invalid size");
}
- return Literal(sval);
}
case Type::f32: {
- return Literal(bit_cast<float>(static_cast<int32_t>(val)));
+ switch (curr->bytes) {
+ case 2:
+ return Literal(bit_cast<int32_t>(fp16_ieee_to_fp32_value(
+ Bits::readLE<uint16_t>(p))))
+ .castToF32();
+ case 4:
+ return Literal(Bits::readLE<uint32_t>(p)).castToF32();
+ default:
+ WASM_UNREACHABLE("invalid size");
+ }
}
case Type::f64: {
- return Literal(bit_cast<double>(static_cast<int64_t>(val)));
+ assert(curr->bytes == 8);
+ return Literal(Bits::readLE<uint64_t>(p)).castToF64();
+ }
+ case Type::v128: {
+ assert(curr->bytes == 16);
+ return Literal(p);
}
default:
WASM_UNREACHABLE("invalid type");
@@ -2525,55 +2536,36 @@
VISIT(ref, curr->ref)
VISIT(index, curr->index)
VISIT(value, curr->value)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
Index i = index.getSingleValue().geti32();
- size_t size = data->values.size();
+ size_t size = refVal.getRawBytes().size();
// Use subtraction to avoid overflow.
if (i >= size || curr->bytes > (size - i)) {
trap("array oob");
}
- switch (curr->value->type.getBasic()) {
- case Type::i32:
- writeBytes(
- value.getSingleValue().geti32(), curr->bytes, i, data->values);
- break;
- case Type::i64:
- writeBytes(
- value.getSingleValue().geti64(), curr->bytes, i, data->values);
- break;
- case Type::f32:
- writeBytes(value.getSingleValue().reinterpreti32(),
- curr->bytes,
- i,
- data->values);
- break;
- case Type::f64:
- writeBytes(value.getSingleValue().reinterpreti64(),
- curr->bytes,
- i,
- data->values);
- break;
- case Type::v128:
- writeBytes(
- value.getSingleValue().getv128(), curr->bytes, i, data->values);
- break;
- case Type::none:
- case Type::unreachable:
- WASM_UNREACHABLE("unimp basic type");
+ uint8_t* p = &refVal.getRawBytes()[i];
+ auto val = value.getSingleValue();
+ if (curr->value->type == Type::f32 && curr->bytes == 2) {
+ float f32 = bit_cast<float>(val.reinterpreti32());
+ Bits::writeLE<uint16_t>(fp16_ieee_from_fp32_value(f32), p);
+ } else {
+ uint8_t buf[16];
+ val.getBits(buf);
+ memcpy(p, buf, curr->bytes);
}
return Flow();
}
Flow visitArrayLen(ArrayLen* curr) {
VISIT(ref, curr->ref)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
- return Literal(int32_t(data->values.size()));
+ return Literal(int32_t(refVal.getNumElements()));
}
Flow visitArrayCopy(ArrayCopy* curr) {
VISIT(destRef, curr->destRef)
@@ -2581,30 +2573,42 @@
VISIT(srcRef, curr->srcRef)
VISIT(srcIndex, curr->srcIndex)
VISIT(length, curr->length)
- auto destData = destRef.getSingleValue().getGCData();
- if (!destData) {
+ auto destVal = destRef.getSingleValue();
+ if (destVal.isNull()) {
trap("null ref");
}
- auto srcData = srcRef.getSingleValue().getGCData();
- if (!srcData) {
+ auto srcVal = srcRef.getSingleValue();
+ if (srcVal.isNull()) {
trap("null ref");
}
- size_t destVal = destIndex.getSingleValue().getUnsigned();
- size_t srcVal = srcIndex.getSingleValue().getUnsigned();
+ size_t destIdx = destIndex.getSingleValue().getUnsigned();
+ size_t srcIdx = srcIndex.getSingleValue().getUnsigned();
size_t lengthVal = length.getSingleValue().getUnsigned();
- if (destVal + lengthVal > destData->values.size()) {
+ if (destIdx + lengthVal > destVal.getNumElements()) {
trap("oob");
}
- if (srcVal + lengthVal > srcData->values.size()) {
+ if (srcIdx + lengthVal > srcVal.getNumElements()) {
trap("oob");
}
+ // Fast path: bulk copy raw byte buffers directly with memmove rather than
+ // extracting and setting elements one-by-one via Literals.
+ if (destVal.isRawBytes() && srcVal.isRawBytes()) {
+ if (lengthVal > 0) {
+ auto elemSize =
+ destVal.type.getHeapType().getArray().element.getByteSize();
+ memmove(&destVal.getRawBytes()[destIdx * elemSize],
+ &srcVal.getRawBytes()[srcIdx * elemSize],
+ lengthVal * elemSize);
+ }
+ return Flow();
+ }
std::vector<Literal> copied;
copied.resize(lengthVal);
for (size_t i = 0; i < lengthVal; i++) {
- copied[i] = srcData->values[srcVal + i];
+ copied[i] = srcVal.getElement(srcIdx + i);
}
for (size_t i = 0; i < lengthVal; i++) {
- destData->values[destVal + i] = copied[i];
+ destVal.setElement(destIdx + i, copied[i]);
}
return Flow();
}
@@ -2613,24 +2617,21 @@
VISIT(index, curr->index)
VISIT(value, curr->value)
VISIT(size, curr->size)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
size_t indexVal = index.getSingleValue().getUnsigned();
Literal fillVal = value.getSingleValue();
size_t sizeVal = size.getSingleValue().getUnsigned();
- auto field = curr->ref->type.getHeapType().getArray().element;
- fillVal = truncateForPacking(fillVal, field);
-
- size_t arraySize = data->values.size();
+ size_t arraySize = refVal.getNumElements();
if (indexVal > arraySize || sizeVal > arraySize ||
indexVal + sizeVal > arraySize || indexVal + sizeVal < indexVal) {
trap("out of bounds array access in array.fill");
}
for (size_t i = 0; i < sizeVal; ++i) {
- data->values[indexVal + i] = fillVal;
+ refVal.setElement(indexVal + i, fillVal);
}
return {};
}
@@ -2640,17 +2641,17 @@
VISIT(ref, curr->ref)
VISIT(index, curr->index)
VISIT(value, curr->value)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
size_t indexVal = index.getSingleValue().getUnsigned();
- if (indexVal >= data->values.size()) {
+ if (indexVal >= refVal.getNumElements()) {
trap("array oob");
}
- auto& field = data->values[indexVal];
- auto oldVal = field;
- field = applyRMW(curr->op, oldVal, value.getSingleValue());
+ auto oldVal = refVal.getElement(indexVal);
+ refVal.setElement(indexVal,
+ applyRMW(curr->op, oldVal, value.getSingleValue()));
return oldVal;
}
@@ -2659,18 +2660,17 @@
VISIT(index, curr->index)
VISIT(expected, curr->expected)
VISIT(replacement, curr->replacement)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
size_t indexVal = index.getSingleValue().getUnsigned();
- if (indexVal >= data->values.size()) {
+ if (indexVal >= refVal.getNumElements()) {
trap("array oob");
}
- auto& field = data->values[indexVal];
- auto oldVal = field;
- if (field == expected.getSingleValue()) {
- field = replacement.getSingleValue();
+ auto oldVal = refVal.getElement(indexVal);
+ if (oldVal == expected.getSingleValue()) {
+ refVal.setElement(indexVal, replacement.getSingleValue());
}
return oldVal;
}
@@ -2696,22 +2696,21 @@
case StringNewWTF16Array: {
VISIT(start, curr->start)
VISIT(end, curr->end)
- auto ptrData = ptr.getSingleValue().getGCData();
- if (!ptrData) {
+ auto ptrVal = ptr.getSingleValue();
+ if (ptrVal.isNull()) {
trap("null ref");
}
- const auto& ptrDataValues = ptrData->values;
+ size_t arrayLen = ptrVal.getNumElements();
size_t startVal = start.getSingleValue().getUnsigned();
size_t endVal = end.getSingleValue().getUnsigned();
- if (startVal > ptrDataValues.size() || endVal > ptrDataValues.size() ||
- endVal < startVal) {
+ if (startVal > arrayLen || endVal > arrayLen || endVal < startVal) {
trap("array oob");
}
Literals contents;
if (endVal > startVal) {
contents.reserve(endVal - startVal);
for (size_t i = startVal; i < endVal; i++) {
- contents.push_back(ptrDataValues[i]);
+ contents.push_back(ptrVal.getElement(i));
}
}
return makeGCData(std::move(contents), curr->type);
@@ -2748,7 +2747,7 @@
trap("null ref");
}
- return Literal(int32_t(data->values.size()));
+ return Literal(int32_t(data->getLiterals().size()));
}
Flow visitStringConcat(StringConcat* curr) {
VISIT(flow, curr->left)
@@ -2761,17 +2760,19 @@
trap("null ref");
}
- auto totalSize = leftData->values.size() + rightData->values.size();
+ auto totalSize =
+ leftData->getLiterals().size() + rightData->getLiterals().size();
if (totalSize >= DataLimit) {
hostLimit("allocation failure");
}
Literals contents;
- contents.reserve(leftData->values.size() + rightData->values.size());
- for (Literal& l : leftData->values) {
+ contents.reserve(leftData->getLiterals().size() +
+ rightData->getLiterals().size());
+ for (Literal& l : leftData->getLiterals()) {
contents.push_back(l);
}
- for (Literal& l : rightData->values) {
+ for (Literal& l : rightData->getLiterals()) {
contents.push_back(l);
}
@@ -2788,24 +2789,24 @@
VISIT(start, curr->start)
auto strData = str.getSingleValue().getGCData();
- auto arrayData = array.getSingleValue().getGCData();
- if (!strData || !arrayData) {
+ auto arrayVal = array.getSingleValue();
+ if (!strData || arrayVal.isNull()) {
trap("null ref");
}
auto startVal = start.getSingleValue().getUnsigned();
- auto& strValues = strData->values;
- auto& arrayValues = arrayData->values;
+ auto& strValues = strData->getLiterals();
+ size_t arrayLen = arrayVal.getNumElements();
size_t end;
if (std::ckd_add<size_t>(&end, startVal, strValues.size()) ||
- end > arrayValues.size()) {
+ end > arrayLen) {
trap("oob");
}
for (Index i = 0; i < strValues.size(); i++) {
- arrayValues[startVal + i] = strValues[i];
+ arrayVal.setElement(startVal + i, strValues[i]);
}
- return Literal(int32_t(strData->values.size()));
+ return Literal(int32_t(strValues.size()));
}
Flow visitStringEq(StringEq* curr) {
VISIT(flow, curr->left)
@@ -2818,17 +2819,17 @@
switch (curr->op) {
case StringEqEqual: {
// They are equal if both are null, or both are non-null and equal.
- result =
- (!leftData && !rightData) ||
- (leftData && rightData && leftData->values == rightData->values);
+ result = (!leftData && !rightData) ||
+ (leftData && rightData &&
+ leftData->getLiterals() == rightData->getLiterals());
break;
}
case StringEqCompare: {
if (!leftData || !rightData) {
trap("null ref");
}
- auto& leftValues = leftData->values;
- auto& rightValues = rightData->values;
+ auto& leftValues = leftData->getLiterals();
+ auto& rightValues = rightData->getLiterals();
Index i = 0;
while (1) {
if (i == leftValues.size() && i == rightValues.size()) {
@@ -2878,7 +2879,7 @@
if (!data) {
trap("null ref");
}
- auto& values = data->values;
+ auto& values = data->getLiterals();
Index i = pos.getSingleValue().geti32();
if (i >= values.size()) {
trap("string oob");
@@ -2895,7 +2896,7 @@
if (!refData) {
trap("null ref");
}
- auto& refValues = refData->values;
+ auto& refValues = refData->getLiterals();
auto startVal = start.getSingleValue().getUnsigned();
auto endVal = end.getSingleValue().getUnsigned();
endVal = std::min<size_t>(endVal, refValues.size());
@@ -2955,22 +2956,6 @@
}
return value;
}
-
- Literal makeFromMemory(void* p, Field field) {
- switch (field.packedType) {
- case Field::NotPacked:
- return Literal::makeFromMemory(p, field.type);
- case Field::i8: {
- return truncateForPacking(Literal(int32_t(Bits::readLE<int8_t>(p))),
- field);
- }
- case Field::i16: {
- return truncateForPacking(Literal(int32_t(Bits::readLE<int16_t>(p))),
- field);
- }
- }
- WASM_UNREACHABLE("unexpected type");
- }
};
// Execute a suspected constant expression (precompute and C-API).
@@ -4742,7 +4727,6 @@
auto heapType = curr->type.getHeapType();
const auto& element = heapType.getArray().element;
- Literals contents;
const auto& seg = *wasm.getDataSegment(curr->segment);
auto elemBytes = element.getByteSize();
@@ -4757,10 +4741,9 @@
if (droppedDataSegments.contains(curr->segment) && end > 0) {
trap("dropped segment access in array.new_data");
}
- contents.reserve(size);
- for (Index i = offset; i < end; i += elemBytes) {
- auto addr = (void*)&seg.data[i];
- contents.push_back(this->makeFromMemory(addr, element));
+ std::vector<uint8_t> contents(size * elemBytes);
+ if (size > 0) {
+ memcpy(contents.data(), &seg.data[offset], size * elemBytes);
}
#pragma GCC diagnostic pop
@@ -4796,15 +4779,15 @@
VISIT(index, curr->index)
VISIT(offset, curr->offset)
VISIT(size, curr->size)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
size_t indexVal = index.getSingleValue().getUnsigned();
size_t offsetVal = offset.getSingleValue().getUnsigned();
size_t sizeVal = size.getSingleValue().getUnsigned();
- size_t arraySize = data->values.size();
+ size_t arraySize = refVal.getNumElements();
if ((uint64_t)indexVal + sizeVal > arraySize) {
trap("out of bounds array access in array.init");
}
@@ -4822,9 +4805,11 @@
droppedDataSegments.contains(curr->segment)) {
trap("out of bounds segment access in array.init_data");
}
- for (size_t i = 0; i < sizeVal; i++) {
- void* addr = (void*)&seg->data[offsetVal + i * elemSize];
- data->values[indexVal + i] = this->makeFromMemory(addr, elem);
+ assert(refVal.isRawBytes());
+ if (sizeVal > 0) {
+ memcpy(&refVal.getRawBytes()[indexVal * elemSize],
+ &seg->data[offsetVal],
+ sizeVal * elemSize);
}
return {};
}
@@ -4833,15 +4818,15 @@
VISIT(index, curr->index)
VISIT(offset, curr->offset)
VISIT(size, curr->size)
- auto data = ref.getSingleValue().getGCData();
- if (!data) {
+ auto refVal = ref.getSingleValue();
+ if (refVal.isNull()) {
trap("null ref");
}
size_t indexVal = index.getSingleValue().getUnsigned();
size_t offsetVal = offset.getSingleValue().getUnsigned();
size_t sizeVal = size.getSingleValue().getUnsigned();
- size_t arraySize = data->values.size();
+ size_t arraySize = refVal.getNumElements();
if ((uint64_t)indexVal + sizeVal > arraySize) {
trap("out of bounds array access in array.init");
}
@@ -4861,7 +4846,8 @@
// of references in the table! ArrayNew suffers the same problem.
// Fixing it will require changing how we represent segments, at least
// in the interpreter.
- data->values[indexVal + i] = self()->visit(seg->data[i]).getSingleValue();
+ refVal.setElement(indexVal + i,
+ self()->visit(seg->data[i]).getSingleValue());
}
return {};
}
diff --git a/src/wasm/literal.cpp b/src/wasm/literal.cpp
index a037fef..272cc5a 100644
--- a/src/wasm/literal.cpp
+++ b/src/wasm/literal.cpp
@@ -314,7 +314,7 @@
return makeFromInt32(-1, type);
}
-Literal Literal::makeFromMemory(void* p, Type type) {
+Literal Literal::makeFromMemory(const void* p, Type type) {
assert(type.isNumber());
switch (type.getBasic()) {
case Type::i32: {
@@ -499,7 +499,7 @@
return *funcData == *other.funcData;
}
if (type.isString()) {
- return gcData->values == other.gcData->values;
+ return gcData->getLiterals() == other.gcData->getLiterals();
}
if (type.isData()) {
return gcData == other.gcData;
@@ -775,8 +775,8 @@
break;
case HeapType::any: {
auto data = literal.getGCData();
- assert(data->values.size() == 1);
- o << "internalized " << literal.getGCData()->values[0];
+ assert(data->getLiterals().size() == 1);
+ o << "internalized " << data->getLiterals()[0];
break;
}
case HeapType::ext: {
@@ -806,7 +806,7 @@
o << "string(";
// Convert WTF-16 literals to WTF-16 string.
std::stringstream wtf16;
- for (auto c : data->values) {
+ for (auto c : data->getLiterals()) {
auto u = c.getInteger();
assert(u < 0x10000);
wtf16 << uint8_t(u & 0xFF);
@@ -844,9 +844,15 @@
assert(literal.isData());
auto data = literal.getGCData();
assert(data);
- o << "[ref " << literal.type.getHeapType() << ' ' << data->values;
+ o << "[ref " << literal.type.getHeapType() << ' ';
+ for (size_t i = 0; i < literal.getNumElements(); i++) {
+ if (i > 0) {
+ o << ' ';
+ }
+ o << literal.getElement(i);
+ }
if (!data->desc.isNull()) {
- if (!data->values.empty()) {
+ if (literal.getNumElements() > 0) {
o << ", ";
}
o << "desc=" << data->desc;
@@ -3102,8 +3108,8 @@
}
if (heapType.isMaybeShared(HeapType::any)) {
// This is an internalized externref or string; just unwrap it.
- assert(gcData->values.size() == 1);
- return gcData->values[0];
+ assert(gcData->getLiterals().size() == 1);
+ return gcData->getLiterals()[0];
}
// This is an internal reference. Wrap it.
auto ext = HeapTypes::ext.getBasic(heapType.getShared());
@@ -3124,8 +3130,8 @@
return Literal(std::make_shared<GCData>(Literals{*this}), any);
}
// This is an externalized internal reference; just unwrap it.
- assert(gcData->values.size() == 1);
- return gcData->values[0];
+ assert(gcData->getLiterals().size() == 1);
+ return gcData->getLiterals()[0];
}
Literal Literal::unwrap() const {
@@ -3148,7 +3154,7 @@
assert(type.isRef());
if (auto desc = type.getHeapType().getDescriptorType();
desc && JSUtils::hasPossibleJSPrototypeField(*desc)) {
- auto proto = gcData->desc.getGCData()->values[0].unwrap();
+ auto proto = gcData->desc.getGCData()->getLiterals()[0].unwrap();
// Strings and numbers are not valid prototypes, so they appear as null.
// Externref nulls are also converted to nullref.
auto protoType = proto.type.getHeapType();
@@ -3161,4 +3167,79 @@
return Literal::makeNull(HeapType::none);
}
+size_t Literal::getNumElements() const {
+ assert(isData());
+ if (gcData->isRawBytes()) {
+ auto field = type.getHeapType().getArray().element;
+ return gcData->getRawBytes().size() / field.getByteSize();
+ }
+ return gcData->getLiterals().size();
+}
+
+namespace {
+
+void writeField(void* dest, const Field& field, Literal value) {
+ if (field.isPacked()) {
+ assert(field.type == Type::i32);
+ int32_t c = value.geti32();
+ if (field.packedType == Field::i8) {
+ Bits::writeLE<int8_t>(static_cast<int8_t>(c), dest);
+ } else if (field.packedType == Field::i16) {
+ Bits::writeLE<int16_t>(static_cast<int16_t>(c), dest);
+ } else {
+ WASM_UNREACHABLE("invalid packed type");
+ }
+ return;
+ }
+
+ uint8_t buf[16];
+ assert(field.getByteSize() <= sizeof(buf));
+ value.getBits(buf);
+ memcpy(dest, buf, field.getByteSize());
+}
+
+Literal readField(const void* src, const Field& field, bool signed_) {
+ if (field.isPacked()) {
+ assert(field.type == Type::i32);
+ if (field.packedType == Field::i8) {
+ int8_t val = Bits::readLE<int8_t>(src);
+ return Literal(signed_ ? int32_t(val)
+ : int32_t(static_cast<uint8_t>(val)));
+ } else if (field.packedType == Field::i16) {
+ int16_t val = Bits::readLE<int16_t>(src);
+ return Literal(signed_ ? int32_t(val)
+ : int32_t(static_cast<uint16_t>(val)));
+ } else {
+ WASM_UNREACHABLE("invalid packed type");
+ }
+ }
+
+ return Literal::makeFromMemory(src, field.type);
+}
+
+} // anonymous namespace
+
+Literal Literal::getElement(size_t index, bool signed_) const {
+ assert(isData());
+ if (gcData->isRawBytes()) {
+ auto field = type.getHeapType().getArray().element;
+ size_t elemSize = field.getByteSize();
+ assert((index + 1) * elemSize <= gcData->getRawBytes().size());
+ return readField(&gcData->getRawBytes()[index * elemSize], field, signed_);
+ }
+ return gcData->getLiterals()[index];
+}
+
+void Literal::setElement(size_t index, Literal value) {
+ assert(isData());
+ if (gcData->isRawBytes()) {
+ auto field = type.getHeapType().getArray().element;
+ size_t elemSize = field.getByteSize();
+ assert((index + 1) * elemSize <= gcData->getRawBytes().size());
+ writeField(&gcData->getRawBytes()[index * elemSize], field, value);
+ return;
+ }
+ gcData->getLiterals()[index] = value;
+}
+
} // namespace wasm
