From dcf48fc5db6dcc1f4b135782c7d9d43b3c620810 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Wed, 28 Oct 2026 19:43:11 +0100 Subject: [PATCH 1/2] ci: add benchmark workflow Cases: - cache hit - cache miss - cache hit 304 via If-None-Match - cache hit with brotli compression Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yml | 132 ++++++++++++++++++++ .gitignore | 3 + benchmark/index.js | 215 ++++++++++++++++++++++++++++++++ package.json | 1 + 4 files changed, 351 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/index.js diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..bc704ad --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,132 @@ +name: benchmark + +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +env: + pnpm_config_dangerously_allow_all_builds: true + SKIP_INSTALL_SIMPLE_GIT_HOOKS: true + +jobs: + benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout head + uses: actions/checkout@v7 + with: + path: head + persist-credentials: false + - name: Checkout base + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: lts/* + - name: Setup PNPM + uses: pnpm/action-setup@v6 + with: + version: latest + - name: Install head + working-directory: head + run: pnpm install + - name: Install base + working-directory: base + run: | + [ -f benchmark/index.js ] || exit 0 + pnpm install || { + echo "::warning::base install failed, comparing without it" + rm benchmark/index.js + } + - name: Run base and head alternately + env: + ROUNDS: 5 + run: | + set -euo pipefail + mkdir -p results + run_side () { + if [ "$1" = base ] && [ ! -f base/benchmark/index.js ]; then return 0; fi + if (cd "$1" && node benchmark/index.js) && [ -f "$1/benchmark-output.json" ]; then + mv "$1/benchmark-output.json" "results/$1-$2.json" + elif [ "$1" = base ]; then + echo "::warning::base benchmark failed in round $2, comparing without it" + else + exit 1 + fi + } + for round in $(seq "$ROUNDS"); do + if [ $((round % 2)) -eq 1 ]; then run_side base "$round"; run_side head "$round" + else run_side head "$round"; run_side base "$round"; fi + done + - name: Compare + env: + NAME: cacheable-response + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + node - <<'EOF' + const { readdirSync, readFileSync, writeFileSync, mkdirSync, appendFileSync } = require('fs') + const median = values => { + const sorted = [...values].sort((a, b) => a - b) + const middle = sorted.length >> 1 + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2 + } + const round = value => Number(value.toFixed(2)) + const readSide = side => + readdirSync('results') + .filter(file => file.startsWith(`${side}-`)) + .map(file => JSON.parse(readFileSync(`results/${file}`, 'utf8'))) + const merge = runs => { + const names = [...new Set(runs.flatMap(run => run.map(result => result.name)))] + return names.map(name => { + const matches = runs.flatMap(run => run.filter(result => result.name === name)) + const values = matches.map(result => result.value) + return { + name, + unit: matches[0].unit, + value: round(median(values)), + range: `${round(Math.min(...values))}-${round(Math.max(...values))}`, + extra: `median of ${values.length} rounds: ${values.join(', ')}` + } + }) + } + const head = merge(readSide('head')) + const baseRuns = readSide('base') + const base = baseRuns.length ? merge(baseRuns) : [] + writeFileSync('head-output.json', JSON.stringify(head, null, 2)) + mkdirSync('cache', { recursive: true }) + const seed = base.length + ? { [process.env.NAME]: [{ commit: { id: process.env.BASE_SHA }, date: Date.now(), tool: 'customSmallerIsBetter', benches: base }] } + : {} + writeFileSync('cache/benchmark-data.json', JSON.stringify({ lastUpdate: Date.now(), repoUrl: '', entries: seed })) + const rows = head.map(current => { + const previous = base.find(result => result.name === current.name) + const ratio = previous ? (current.value / previous.value).toFixed(2) : '' + const format = result => (result ? `${result.value} ${result.unit}` : '') + return `| \`${current.name}\` | ${format(previous)} | ${format(current)} | ${ratio} |` + }) + const summary = ['| case | base | head | head/base |', '|-|-|-|-|', ...rows].join('\n') + console.log(summary) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary + '\n') + EOF + - name: Comment + if: github.event.pull_request.head.repo.full_name == github.repository + uses: benchmark-action/github-action-benchmark@v1.22.2 + with: + name: cacheable-response + tool: customSmallerIsBetter + output-file-path: head-output.json + external-data-json-path: cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: false + comment-always: true + comment-on-alert: false + fail-on-alert: false + summary-always: false diff --git a/.gitignore b/.gitignore index cee5ab4..ae77e4a 100755 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ coverage .env .envrc .envrc +benchmark-output.json +head-output.json +cache/ diff --git a/benchmark/index.js b/benchmark/index.js new file mode 100644 index 0000000..f10adfc --- /dev/null +++ b/benchmark/index.js @@ -0,0 +1,215 @@ +'use strict' + +const { writeFileSync } = require('fs') +const { join } = require('path') +const { setFlagsFromString } = require('v8') +const { runInNewContext } = require('vm') + +const cacheableResponse = require('..') + +const RUNS = 7 +const HIT_REQUESTS = 20_000 +const MISS_REQUESTS = 10_000 +const NOT_MODIFIED_REQUESTS = 20_000 +const COMPRESSED_REQUESTS = 2_000 +const SMALL_PAGE_SECTIONS = 2 +const LARGE_PAGE_SECTIONS = 200 + +setFlagsFromString('--expose-gc') +const collectGarbage = runInNewContext('gc') + +const now = () => process.hrtime.bigint() + +const elapsed = started => Number(now() - started) / 1e6 + +const median = values => { + const sorted = values.slice().sort((a, b) => a - b) + return sorted[Math.floor(sorted.length / 2)] +} + +const createPage = sections => + Array.from( + { length: sections }, + (_, section) => + `

Section ${section}

Lorem ipsum dolor sit amet ${section}

` + ).join('') + +const SMALL_PAGE = createPage(SMALL_PAGE_SECTIONS) +const LARGE_PAGE = createPage(LARGE_PAGE_SECTIONS) + +const createRequest = (url, headers = {}) => ({ url, headers }) + +const createResponse = () => { + const response = { + statusCode: 200, + writableEnded: false, + headers: {}, + body: undefined, + setHeader: (name, value) => { + response.headers[name.toLowerCase()] = value + }, + end: body => { + response.body = body + response.writableEnded = true + } + } + return response +} + +const createHandler = ({ page = SMALL_PAGE, ...options } = {}) => + cacheableResponse({ + get: ({ req }) => ({ data: `${req.url}${page}` }), + send: ({ data, res }) => res.end(data), + ...options + }) + +const serve = async (handler, url, headers) => { + const res = createResponse() + await handler({ req: createRequest(url, headers), res }) + return res +} + +const serveSequentially = async (handler, urlOf, count, headersOf) => { + const tally = { statuses: {}, bytes: 0 } + for (let request = 0; request < count; request++) { + const res = await serve(handler, urlOf(request), headersOf?.(request)) + const status = res.headers['x-cache-status'] + tally.statuses[status] = (tally.statuses[status] || 0) + 1 + if (res.statusCode === 200) tally.bytes += res.body.length + } + return tally +} + +const pageBytes = (url, page = SMALL_PAGE) => url.length + page.length + +const HIT_URL = '/blog/post?utm_source=newsletter&page=2' + +const hitCase = { + name: 'cache hit', + extra: `${HIT_REQUESTS} requests for one cached URL`, + setup: async state => { + state.handler = createHandler() + await serve(state.handler, HIT_URL) + }, + timed: state => + serveSequentially(state.handler, () => HIT_URL, HIT_REQUESTS), + verify: ({ statuses, bytes }) => + statuses.HIT === HIT_REQUESTS && + bytes === HIT_REQUESTS * pageBytes(HIT_URL) +} + +const missCase = { + name: 'cache miss', + extra: `${MISS_REQUESTS} requests for distinct URLs into an empty cache`, + setup: () => {}, + timed: () => + serveSequentially( + createHandler(), + request => `/item/${request}?ref=home`, + MISS_REQUESTS + ), + verify: ({ statuses, bytes }) => + statuses.MISS === MISS_REQUESTS && + bytes > MISS_REQUESTS * SMALL_PAGE.length +} + +const notModifiedCase = { + name: 'cache hit 304 via If-None-Match', + extra: `${NOT_MODIFIED_REQUESTS} conditional requests for one cached URL`, + setup: async state => { + state.handler = createHandler() + const res = await serve(state.handler, HIT_URL) + state.headers = { 'if-none-match': res.headers.etag } + }, + timed: state => + serveSequentially( + state.handler, + () => HIT_URL, + NOT_MODIFIED_REQUESTS, + () => state.headers + ), + verify: ({ statuses, bytes }) => + statuses.HIT === NOT_MODIFIED_REQUESTS && bytes === 0 +} + +const compressedHitCase = { + name: 'cache hit with brotli compression', + extra: `${COMPRESSED_REQUESTS} requests decompressing a ${LARGE_PAGE.length} byte page`, + setup: async state => { + state.handler = createHandler({ compress: true, page: LARGE_PAGE }) + await serve(state.handler, HIT_URL) + }, + timed: state => + serveSequentially(state.handler, () => HIT_URL, COMPRESSED_REQUESTS), + verify: ({ statuses, bytes }) => + statuses.HIT === COMPRESSED_REQUESTS && + bytes === COMPRESSED_REQUESTS * pageBytes(HIT_URL, LARGE_PAGE) +} + +const CASES = [hitCase, missCase, notModifiedCase, compressedHitCase] + +const fmt = value => value.toFixed(1) + +const report = slot => { + const med = slot.walls.length ? median(slot.walls) : Number.NaN + const min = slot.walls.length ? Math.min(...slot.walls) : Number.NaN + const max = slot.walls.length ? Math.max(...slot.walls) : Number.NaN + const extra = `${slot.entry.extra}, errors ${slot.errors}, runs ${RUNS}` + console.log( + `${slot.entry.name} median ${fmt(med)}ms range ${fmt(min)}-${fmt( + max + )} ${extra}` + ) + return { + name: slot.entry.name, + unit: 'ms', + value: Number(med.toFixed(1)), + range: `${fmt(min)}-${fmt(max)}`, + extra + } +} + +const takeSample = async (slot, run) => { + try { + collectGarbage() + const started = now() + const outcome = await slot.entry.timed(slot.state) + const wall = elapsed(started) + if (!slot.entry.verify(outcome)) { + console.error(`${slot.entry.name} run ${run} produced unexpected output`) + slot.errors++ + return + } + slot.walls.push(wall) + } catch (error) { + console.error(error) + slot.errors++ + } +} + +const main = async () => { + const slots = [] + for (const entry of CASES) { + const state = {} + await entry.setup(state) + await entry.timed(state) + slots.push({ entry, state, walls: [], errors: 0 }) + } + for (let run = 0; run < RUNS; run++) { + for (const slot of slots) await takeSample(slot, run) + } + const results = slots.map(report) + writeFileSync( + join(__dirname, '..', 'benchmark-output.json'), + JSON.stringify(results, null, 2) + '\n' + ) + const errors = slots.reduce((total, slot) => total + slot.errors, 0) + if (errors > 0 || results.some(result => !Number.isFinite(result.value))) { + process.exitCode = 1 + } +} + +main().catch(error => { + console.error(error) + process.exit(1) +}) diff --git a/package.json b/package.json index 93c1b15..aefabc0 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "src" ], "scripts": { + "benchmark": "node benchmark/index.js", "clean": "rm -rf node_modules", "contributors": "(git-authors-cli && --sort-ignore-object-at ava && git add package.json && git commit -m 'build: contributors' --no-verify) || true", "lint": "standard", From 5ec96bb6bedf88b816391ccf56013ca1927e767f Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Wed, 28 Oct 2026 19:53:29 +0100 Subject: [PATCH 2/2] ci(benchmark): assert the 304 status in the conditional case Co-Authored-By: Claude Opus 5.5 --- benchmark/index.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmark/index.js b/benchmark/index.js index f10adfc..d27fd80 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -70,11 +70,12 @@ const serve = async (handler, url, headers) => { } const serveSequentially = async (handler, urlOf, count, headersOf) => { - const tally = { statuses: {}, bytes: 0 } + const tally = { statuses: {}, codes: {}, bytes: 0 } for (let request = 0; request < count; request++) { const res = await serve(handler, urlOf(request), headersOf?.(request)) const status = res.headers['x-cache-status'] tally.statuses[status] = (tally.statuses[status] || 0) + 1 + tally.codes[res.statusCode] = (tally.codes[res.statusCode] || 0) + 1 if (res.statusCode === 200) tally.bytes += res.body.length } return tally @@ -128,8 +129,9 @@ const notModifiedCase = { NOT_MODIFIED_REQUESTS, () => state.headers ), - verify: ({ statuses, bytes }) => - statuses.HIT === NOT_MODIFIED_REQUESTS && bytes === 0 + verify: ({ statuses, codes }) => + statuses.HIT === NOT_MODIFIED_REQUESTS && + codes[304] === NOT_MODIFIED_REQUESTS } const compressedHitCase = {