Compare juniper with async-graphql and apollo-compiler in the benchmarks

.github/workflows/ci.yml

@@ -100,7 +100,8 @@ jobs:
           toolchain: stable
           components: clippy
 
-      - run: cargo clippy -p juniper_benchmarks --benches -- -D warnings
+      - run: cargo clippy -p juniper_benchmarks --benches --tests -- -D warnings
+      - run: cargo test -p juniper_benchmarks
       - run: cargo bench -p juniper_benchmarks
 
   feature:

benches/Cargo.toml

@@ -5,12 +5,20 @@ edition = "2024"
 authors = ["Christoph Herzog <chris@theduke.at>"]
 publish = false
 
+[lib]
+# Keeps the `libtest` harness from rejecting Criterion arguments in `cargo bench`.
+bench = false
+
 [dependencies]
+apollo-compiler = "1.33"
+async-graphql = { version = "7.2", default-features = false }
+futures = "0.3"
 juniper = { path = "../juniper" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 
 [dev-dependencies]
 criterion = "0.8"
-tokio = { version = "1.0", features = ["rt-multi-thread"] }
 
 [lints.clippy]
 allow_attributes = "warn"
@@ -24,5 +32,5 @@ unsafe_code = "forbid"
 unused_crate_dependencies = "warn"
 
 [[bench]]
-name = "benchmark"
+name = "comparison"
 harness = false

benches/benches/comparison.rs

@@ -0,0 +1,45 @@
+//! Compares the engines on every operation in [`operation::ALL`].
+
+#![allow(
+    unused_crate_dependencies,
+    reason = "the library uses the other dependencies"
+)]
+
+use criterion::{
+    BenchmarkGroup, Criterion, criterion_group, criterion_main, measurement::WallTime,
+};
+use juniper_benchmarks::{
+    engine::{
+        Engine,
+        apollo_compiler::ApolloCompiler,
+        async_graphql::AsyncGraphql,
+        juniper::{JuniperAsync, JuniperAsyncFn, JuniperSync},
+    },
+    operation::{self, Operation},
+};
+
+fn compare(c: &mut Criterion) {
+    for operation in operation::ALL {
+        operation.check();
+        let request = operation.request();
+        let mut group = c.benchmark_group(operation.name);
+        bench::<JuniperAsync>(&mut group, operation, &request);
+        bench::<JuniperSync>(&mut group, operation, &request);
+        bench::<JuniperAsyncFn>(&mut group, operation, &request);
+        bench::<AsyncGraphql>(&mut group, operation, &request);
+        bench::<ApolloCompiler>(&mut group, operation, &request);
+        group.finish();
+    }
+}
+
+/// Measures the `E` engine on the `operation`, if the engine supports the operation.
+fn bench<E: Engine>(group: &mut BenchmarkGroup<'_, WallTime>, operation: Operation, request: &str) {
+    if !operation.runs_on::<E>() {
+        return;
+    }
+    let engine = E::default();
+    group.bench_function(E::NAME, |b| b.iter(|| engine.execute(request)));
+}
+
+criterion_group!(benches, compare);
+criterion_main!(benches);

benches/src/data.rs

@@ -0,0 +1,200 @@
+//! Data served by every engine.
+//!
+//! All data is `'static`. Resolvers only read fields and follow references, so the benchmarks
+//! measure the engines and not the data access.
+
+use std::sync::LazyLock;
+
+/// Number of users in [`users()`].
+pub const USER_COUNT: usize = 1_000;
+
+/// User of a social network.
+#[derive(Debug)]
+pub struct User {
+    pub id: i32,
+    pub name: String,
+    pub email: String,
+    pub active: bool,
+    pub score: f64,
+    pub role: Role,
+}
+
+impl User {
+    fn new(id: i32) -> Self {
+        let roles = [Role::Admin, Role::Moderator, Role::Member, Role::Guest];
+        Self {
+            id,
+            name: format!("User {id}"),
+            email: format!("user{id}@example.com"),
+            active: id % 2 == 0,
+            score: f64::from(id) / 2.0,
+            role: roles[id.unsigned_abs() as usize % roles.len()],
+        }
+    }
+
+    /// Returns up to `count` users that follow this one in [`users()`].
+    ///
+    /// The sequence wraps around to the start of [`users()`]. A negative `count` returns no users.
+    pub fn friends(&self, count: i32) -> impl Iterator<Item = &'static User> {
+        let all = users();
+        let start = self.id.unsigned_abs() as usize + 1;
+        let count = usize::try_from(count).unwrap_or_default();
+        (start..start + count).map(move |index| &all[index % all.len()])
+    }
+}
+
+/// Access level of a [`User`].
+#[derive(Clone, Copy, Debug, Eq, PartialEq, async_graphql::Enum, juniper::GraphQLEnum)]
+pub enum Role {
+    Admin,
+    Moderator,
+    Member,
+    Guest,
+}
+
+/// Returns all users, ordered by ID from `0`.
+pub fn users() -> &'static [User] {
+    static USERS: LazyLock<Vec<User>> = LazyLock::new(|| {
+        let count = i32::try_from(USER_COUNT).expect("`USER_COUNT` fits in `i32`");
+        (0..count).map(User::new).collect()
+    });
+    &USERS
+}
+
+/// Returns up to `count` first users. A negative `count` returns no users.
+pub fn first_users(count: i32) -> impl Iterator<Item = &'static User> {
+    users()
+        .iter()
+        .take(usize::try_from(count).unwrap_or_default())
+}
+
+/// Film of the original Star Wars trilogy.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, async_graphql::Enum, juniper::GraphQLEnum)]
+pub enum Episode {
+    NewHope,
+    Empire,
+    Jedi,
+}
+
+/// Character of the Star Wars films.
+#[derive(Clone, Copy, Debug)]
+pub enum Character {
+    Human(&'static Human),
+    Droid(&'static Droid),
+}
+
+/// Human character of the Star Wars films.
+#[derive(Debug)]
+pub struct Human {
+    pub id: &'static str,
+    pub name: &'static str,
+    pub friends: &'static [Character],
+    pub appears_in: &'static [Episode],
+    pub home_planet: Option<&'static str>,
+}
+
+/// Droid character of the Star Wars films.
+#[derive(Debug)]
+pub struct Droid {
+    pub id: &'static str,
+    pub name: &'static str,
+    pub friends: &'static [Character],
+    pub appears_in: &'static [Episode],
+    pub primary_function: &'static str,
+}
+
+/// Human characters in the order of their IDs.
+pub static HUMANS: [&Human; 5] = [&LUKE, &VADER, &HAN, &LEIA, &TARKIN];
+
+/// Droid characters in the order of their IDs.
+pub static DROIDS: [&Droid; 2] = [&THREEPIO, &ARTOO];
+
+/// Returns the hero of `episode`, or the hero of the whole saga if `episode` is [`None`].
+pub fn hero(episode: Option<Episode>) -> Character {
+    if episode == Some(Episode::Empire) {
+        Character::Human(&LUKE)
+    } else {
+        Character::Droid(&ARTOO)
+    }
+}
+
+const ALL_EPISODES: &[Episode] = &[Episode::NewHope, Episode::Empire, Episode::Jedi];
+
+static LUKE: Human = Human {
+    id: "1000",
+    name: "Luke Skywalker",
+    friends: &[
+        Character::Human(&HAN),
+        Character::Human(&LEIA),
+        Character::Droid(&THREEPIO),
+        Character::Droid(&ARTOO),
+    ],
+    appears_in: ALL_EPISODES,
+    home_planet: Some("Tatooine"),
+};
+
+static VADER: Human = Human {
+    id: "1001",
+    name: "Darth Vader",
+    friends: &[Character::Human(&TARKIN)],
+    appears_in: ALL_EPISODES,
+    home_planet: Some("Tatooine"),
+};
+
+static HAN: Human = Human {
+    id: "1002",
+    name: "Han Solo",
+    friends: &[
+        Character::Human(&LUKE),
+        Character::Human(&LEIA),
+        Character::Droid(&ARTOO),
+    ],
+    appears_in: ALL_EPISODES,
+    home_planet: None,
+};
+
+static LEIA: Human = Human {
+    id: "1003",
+    name: "Leia Organa",
+    friends: &[
+        Character::Human(&LUKE),
+        Character::Human(&HAN),
+        Character::Droid(&THREEPIO),
+        Character::Droid(&ARTOO),
+    ],
+    appears_in: ALL_EPISODES,
+    home_planet: Some("Alderaan"),
+};
+
+static TARKIN: Human = Human {
+    id: "1004",
+    name: "Wilhuff Tarkin",
+    friends: &[Character::Human(&VADER)],
+    appears_in: &[Episode::NewHope],
+    home_planet: None,
+};
+
+static THREEPIO: Droid = Droid {
+    id: "2000",
+    name: "C-3PO",
+    friends: &[
+        Character::Human(&LUKE),
+        Character::Human(&HAN),
+        Character::Human(&LEIA),
+        Character::Droid(&ARTOO),
+    ],
+    appears_in: ALL_EPISODES,
+    primary_function: "Protocol",
+};
+
+static ARTOO: Droid = Droid {
+    id: "2001",
+    name: "R2-D2",
+    friends: &[
+        Character::Human(&LUKE),
+        Character::Human(&HAN),
+        Character::Human(&LEIA),
+    ],
+    appears_in: ALL_EPISODES,
+    primary_function: "Astromech",
+};