Introduce an OOM-handling Error type for Wasmtime (#12163)

* Introduce an OOM-handling `Error` type for Wasmtime

This new `Error` has an API that is 99% identical to `anyhow::Error`'s API, but
additionally handles memory exhaustion.

This commit only introduces the `wasmtime_internal_error` crate into our
workspace, along with its regular tests and OOM tests. This commit does not,
however, migrate Wasmtime's internals or public-facing API over to the new error
type yet. That is left for follow up work.

In order to continue fitting `Result<(), Error>` in one word, there is quite a
bit of unsafe code in `Error`'s implementation, mostly surrounding the manual
creation of our own moral equivalent of `Box<dyn Error>` with explicit vtables
and type erasure so that we get a thin pointer to a trait object rather than
`Box<dyn Error>`'s fat pointer. To alleviate the associated risks, I've been
testing this code under MIRI throughout its whole development, as well as
thoroughly testing the API so that MIRI can dynamically exercise all the code
paths. Furthermore, I've enabled testing this crate under MIRI in CI.

* Fill out more cargo.toml info

* Add cargo vet entries for new crate

* Use single quotes for string in Cargo.toml to work around publish script bug

See #12164

* Add the `wasmtime_error::Ok` function

For compatibility with `anyhow`'s API.

This unfortunately means we have to move the actual error implementation out
into a submodule so that this function isn't visible, or else every `match .. {
Ok(_) => ... }` stops type checking.

* fix some stuff that was broken in the move to a module

* maybe fix cargo vet?

* Add comments about backtraces and OOM

* Debug assert `ConcreteError<E>` and `DynError` layouts are compatible for all `ConcreteError<E>`s that we actually allocate

* Add reference to layout test and assertions to docs about compatible layout requirements

* Use `#[track_caller]` to hide internal frame from error's backtrace

* Fix typo

* Pull various internal `ErrorExt` implementations out to top level

* Switch to `0x1` as the representation of OOM in `OomOrDynError` packing

* add audit-as-crates-io for wasmtime-internal-error

Author: fitzgen

.github/workflows/main.yml

@@ -1144,6 +1144,8 @@ jobs:
           - crate: "wasmtime-cli"
           - crate: "wasmtime-environ --all-features"
           - crate: "pulley-interpreter --all-features"
+          - crate: "wasmtime-internal-error"
+          - crate: "wasmtime-internal-error --all-features"
           - script: ./ci/miri-provenance-test.sh
           - script: ./ci/miri-wast.sh ./tests/spec_testsuite/table.wast
     needs: determine

Cargo.lock

@@ -158,6 +158,7 @@ members = [
   "crates/bench-api",
   "crates/c-api/artifact",
   "crates/environ/fuzz",
+  "crates/error",
   "crates/misc/component-async-tests",
   "crates/test-programs",
   "crates/wasi-preview1-component-adapter",
@@ -258,6 +259,7 @@ wasmtime-wast = { path = "crates/wast", version = "=41.0.0" }
 # that these are internal unsupported crates for external use. These exist as
 # part of the project organization of other public crates in Wasmtime and are
 # otherwise not supported in terms of CVEs for example.
+wasmtime-error = { path = "crates/error", version = "=41.0.0", package = 'wasmtime-internal-error' }
 wasmtime-wmemcheck = { path = "crates/wmemcheck", version = "=41.0.0", package = 'wasmtime-internal-wmemcheck' }
 wasmtime-c-api-macros = { path = "crates/c-api-macros", version = "=41.0.0", package = 'wasmtime-internal-c-api-macros' }
 wasmtime-cache = { path = "crates/cache", version = "=41.0.0", package = 'wasmtime-internal-cache' }

Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+authors.workspace = true
+description = "INTERNAL: Wasmtime's universal error type's implementation"
+edition.workspace = true
+license = "Apache-2.0 WITH LLVM-exception"
+name = "wasmtime-internal-error"
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+anyhow = { workspace = true, optional = true }
+
+[lints]
+workspace = true
+
+[features]
+# Enable the use of the `std` crate.
+std = []
+# Enable backtraces.
+backtrace = ["std"]
+# Enable the `From<Error> for anyhow::Error` implementation and
+# `Error::from_anyhow` constructor.
+anyhow = ["dep:anyhow"]

crates/error/Cargo.toml

@@ -0,0 +1,28 @@
+use std::backtrace::Backtrace;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+static ENABLED: AtomicBool = AtomicBool::new(true);
+
+fn enabled() -> bool {
+    ENABLED.load(Ordering::Acquire)
+}
+
+/// Forcibly disable capturing backtraces dynamically.
+///
+/// XXX: This is only exposed for internal testing, to work around cargo
+/// workspaces and feature resolution. This method may disappear or change
+/// at any time. Instead of using this method, you should disable the
+/// `backtrace` cargo feature.
+#[doc(hidden)]
+pub fn disable_backtrace() {
+    ENABLED.store(false, Ordering::Release)
+}
+
+#[track_caller]
+pub fn capture() -> Backtrace {
+    if enabled() {
+        Backtrace::capture()
+    } else {
+        Backtrace::disabled()
+    }
+}

crates/error/src/backtrace.rs

@@ -0,0 +1,50 @@
+use super::{OutOfMemory, Result};
+use alloc::boxed::Box;
+use core::alloc::Layout;
+use core::ptr::NonNull;
+
+/// Try to allocate a block of memory that fits the given layout, or return an
+/// `OutOfMemory` error.
+///
+/// # Safety
+///
+/// Same as `alloc::alloc::alloc`: layout must have non-zero size.
+#[inline]
+pub(crate) unsafe fn try_alloc(layout: Layout) -> Result<NonNull<u8>, OutOfMemory> {
+    // Safety: same as our safety conditions.
+    debug_assert!(layout.size() > 0);
+    let ptr = unsafe { alloc::alloc::alloc(layout) };
+
+    if let Some(ptr) = NonNull::new(ptr) {
+        Ok(ptr)
+    } else {
+        out_of_line_slow_path!(Err(OutOfMemory::new()))
+    }
+}
+
+/// Create a `Box<T>`, or return an `OutOfMemory` error.
+#[inline]
+pub(crate) fn try_box<T>(value: T) -> Result<Box<T>, OutOfMemory> {
+    let layout = alloc::alloc::Layout::new::<T>();
+
+    if layout.size() == 0 {
+        // Safety: `Box` explicitly allows construction from dangling pointers
+        // (which are guaranteed non-null and aligned) for zero-sized types.
+        return Ok(unsafe { Box::from_raw(core::ptr::dangling::<T>().cast_mut()) });
+    }
+
+    // Safety: layout size is non-zero.
+    let ptr = unsafe { try_alloc(layout)? };
+
+    let ptr = ptr.cast::<T>();
+
+    // Safety: The allocation succeeded, and it has `T`'s layout, so the pointer
+    // is valid for writing a `T`.
+    unsafe {
+        ptr.write(value);
+    }
+
+    // Safety: The pointer's memory block was allocated by the global allocator,
+    // is valid for `T`, and is initialized.
+    Ok(unsafe { Box::from_raw(ptr.as_ptr()) })
+}