bynk_driver/
dev_run.rs

1//! #2138 (end-to-end testing track §3.5): `bynk test --against dev`, the
2//! `system` suites run against a local `bynk dev` the runner starts itself.
3//!
4//! For each distinct `env` the `system` cases run with (one group), this:
5//!
6//! 1. starts `bynk dev --ready-json` (local `wrangler dev`/workerd only, never
7//!    `-- --remote`) with the group's `env` and a fixed test value for every
8//!    other secret the Workers declare, passed as `-- --var NAME:VALUE`;
9//! 2. waits for the readiness file, whose URLs must all be loopback;
10//! 3. runs the group's cases against those URLs (this binary again, with
11//!    [`crate::remote::DEV_URLS_ENV`] and [`crate::remote::DEV_ENV_ENV`]),
12//!    signing with the same secret values;
13//! 4. stops `bynk dev` by its own pid (it stops its wranglers and their
14//!    `workerd`s), then its process group if anything is left.
15//!
16//! Groups run in turn; their results are reported together.
17
18use std::collections::{BTreeMap, BTreeSet};
19use std::path::{Path, PathBuf};
20use std::process::{Child, Command, ExitCode, Stdio};
21use std::time::{Duration, Instant};
22
23use bynk_emit::project::{BuildTarget, RemoteTests};
24
25use crate::remote::{DEV, DEV_ENV_ENV, DEV_SECRETS_ENV, DEV_URLS_ENV, is_loopback_http};
26use crate::test_json::TestRun;
27
28/// The value every declared secret the case doesn't set takes, both in the
29/// Workers (`-- --var`) and in the runner that signs for them.
30pub const TEST_SECRET: &str = "__bynk_test_secret";
31
32/// How long to wait for a `bynk dev` to answer before giving up.
33const READY_TIMEOUT: Duration = Duration::from_secs(180);
34
35/// How long a stopped `bynk dev` gets to stop its Workers before its process
36/// group is killed.
37const STOP_GRACE: Duration = Duration::from_secs(30);
38
39/// What the orchestration needs from the `test` invocation.
40#[derive(Debug, Clone)]
41pub struct DevRunArgs {
42    pub input: PathBuf,
43    pub output: PathBuf,
44    pub json: bool,
45    /// `--format ndjson`: the merged result is printed as a stream. The groups
46    /// run as `--format json` children, so it is written when they have finished.
47    pub stream: bool,
48    pub seed: Option<String>,
49    /// The `--suite`/`--case` set, forwarded to each group's run.
50    pub filter: crate::test_filter::TestFilter,
51    pub skip_unsupported: bool,
52    /// `--settle`, forwarded to each group's run.
53    pub settle: Option<String>,
54    /// The `run.id` every group's run shares.
55    pub run_id: String,
56    /// The `bynk` binary that starts `bynk dev`.
57    pub bynk: PathBuf,
58}
59
60/// Run every `system` suite against a local `bynk dev`, one per distinct
61/// `env`.
62pub fn run(program: &str, args: &DevRunArgs) -> ExitCode {
63    let fail = |message: String| {
64        if args.json {
65            crate::test_stream::print_run(args.stream, &TestRun::runtime_error(message, None));
66        } else {
67            eprintln!("{program} test: {message}");
68        }
69        ExitCode::FAILURE
70    };
71    let plan = match plan(args) {
72        Ok(plan) => plan,
73        Err(Planned::Failed(code)) => return code,
74        Err(Planned::Message(m)) => return fail(m),
75    };
76    // The groups run in separate processes, each seeing only its own cases, so
77    // the values that select nothing are judged here, over all of them, before
78    // anything starts (#2273).
79    let unmatched = args.filter.unmatched_in(&plan.cases);
80    if !unmatched.is_empty() {
81        return fail(unmatched.messages().join("; "));
82    }
83    if plan.groups.is_empty() {
84        if args.json {
85            crate::test_stream::print_run(
86                args.stream,
87                &TestRun::empty().with_remote(format!("remote:{DEV}"), &[]),
88            );
89        } else {
90            eprintln!("{program} test: no `system` suites to run against `bynk dev`");
91        }
92        return ExitCode::SUCCESS;
93    }
94    let mut docs: Vec<serde_json::Value> = Vec::new();
95    let mut all_ok = true;
96    for (i, group) in plan.groups.iter().enumerate() {
97        let vars = dev_vars(group, &plan.secrets);
98        if !args.json {
99            eprintln!(
100                "{program} test: starting `bynk dev`{}",
101                if group.is_empty() {
102                    String::new()
103                } else {
104                    format!(
105                        " with env {}",
106                        group
107                            .iter()
108                            .map(|(k, v)| format!("{k}={v:?}"))
109                            .collect::<Vec<_>>()
110                            .join(", ")
111                    )
112                }
113            );
114        }
115        let ready = args.output.join(format!(".bynk-dev-ready-{i}.json"));
116        let log = args.output.join(format!(".bynk-dev-{i}.log"));
117        let mut dev = match Dev::start(&args.bynk, &args.input, &ready, &log, &vars) {
118            Ok(dev) => dev,
119            Err(e) => return fail(format!("could not start `bynk dev`: {e}")),
120        };
121        let urls = match dev.wait_ready(&ready) {
122            Ok(urls) => urls,
123            Err(e) => {
124                drop(dev);
125                return fail(format!("{e} (its output is in {})", log.display()));
126            }
127        };
128        let outcome = run_group(args, group, &urls, &vars);
129        dev.stop();
130        match outcome {
131            Ok((ok, doc)) => {
132                all_ok &= ok;
133                docs.extend(doc);
134            }
135            Err(e) => return fail(e),
136        }
137    }
138    if args.json {
139        crate::test_stream::print_run(args.stream, &merge(&docs));
140    }
141    if all_ok {
142        ExitCode::SUCCESS
143    } else {
144        ExitCode::FAILURE
145    }
146}
147
148/// The groups to run, and the secrets the Workers declare.
149struct Plan {
150    groups: Vec<BTreeMap<String, String>>,
151    secrets: BTreeSet<String>,
152    /// Every (suite, case) a group can run, to judge `--suite`/`--case` against.
153    cases: Vec<(String, String)>,
154}
155
156enum Planned {
157    /// A compile failure, already reported.
158    Failed(ExitCode),
159    Message(String),
160}
161
162/// Compile the project as a `--against dev` build once: refuse what a real
163/// runtime can't honour (as the group runs would), and read the distinct
164/// `env` maps of its `system` cases and the secrets its Workers declare.
165fn plan(args: &DevRunArgs) -> Result<Plan, Planned> {
166    let options = crate::try_project_options(&args.input)
167        .map_err(|e| Planned::Message(e.to_string()))?
168        .contracts(true)
169        .remote_tests(Some(RemoteTests {
170            skip_unsupported: args.skip_unsupported,
171            dev: true,
172            ..RemoteTests::default()
173        }))
174        .target(BuildTarget::Workers);
175    let out = match bynk_emit::project::compile_project(&options) {
176        Ok(out) => out,
177        Err(failure) => {
178            if args.json {
179                crate::test_stream::print_run(
180                    args.stream,
181                    &TestRun::compile_error(crate::project_failure_short_lines(&failure)),
182                );
183            } else {
184                crate::print_project_failure(&failure);
185            }
186            return Err(Planned::Failed(ExitCode::FAILURE));
187        }
188    };
189    let mut groups: Vec<BTreeMap<String, String>> = Vec::new();
190    for env in out.discovered.iter().flat_map(|s| s.envs.iter()) {
191        if !groups.contains(env) {
192            groups.push(env.clone());
193        }
194    }
195    // A suite the dev run executes has `envs` (the groups are built from them),
196    // so its cases are the ones a value can select. A case the build skipped
197    // (`--skip-unsupported`) still exists, so a value naming one is not unmatched.
198    let cases = out
199        .discovered
200        .iter()
201        .flat_map(|s| {
202            s.cases
203                .iter()
204                .filter(|_| !s.envs.is_empty())
205                .map(|c| (s.name.clone(), c.name.clone()))
206                .chain(s.skipped.iter().map(|k| (s.name.clone(), k.case.clone())))
207        })
208        .collect();
209    let mut secrets = BTreeSet::new();
210    for (path, doc) in &out.artefacts.docs {
211        if path.file_name().is_some_and(|n| n == "bynk-secrets.json") {
212            let manifest: serde_json::Value = serde_json::from_str(&doc.text()).unwrap_or_default();
213            for key in ["declared", "read"] {
214                for name in manifest[key].as_array().into_iter().flatten() {
215                    if let Some(name) = name.as_str() {
216                        secrets.insert(name.to_string());
217                    }
218                }
219            }
220        }
221    }
222    Ok(Plan {
223        groups,
224        secrets,
225        cases,
226    })
227}
228
229/// The variables one `bynk dev` serves: the group's `env`, then each declared
230/// secret the group doesn't set at the fixed test value.
231pub fn dev_vars(
232    group: &BTreeMap<String, String>,
233    secrets: &BTreeSet<String>,
234) -> BTreeMap<String, String> {
235    let mut vars = group.clone();
236    for s in secrets {
237        vars.entry(s.clone())
238            .or_insert_with(|| TEST_SECRET.to_string());
239    }
240    vars
241}
242
243/// Run one group's cases against `urls`: this binary again, as a run
244/// against `dev` with the URLs and the group's `env`, and the group's
245/// variables in its environment so the drivers sign with them. Returns
246/// whether it passed and, under `--format json`, its document.
247fn run_group(
248    args: &DevRunArgs,
249    group: &BTreeMap<String, String>,
250    urls: &BTreeMap<String, String>,
251    vars: &BTreeMap<String, String>,
252) -> Result<(bool, Option<serde_json::Value>), String> {
253    let me = std::env::current_exe().map_err(|e| format!("could not find this binary: {e}"))?;
254    let mut cmd = Command::new(me);
255    cmd.arg("test")
256        .arg(&args.input)
257        .arg("--output")
258        .arg(&args.output);
259    cmd.arg("--format")
260        .arg(if args.json { "json" } else { "rich" });
261    cmd.args(["--against", DEV]);
262    if let Some(seed) = &args.seed {
263        cmd.args(["--seed", seed]);
264    }
265    cmd.args(args.filter.args());
266    if !args.filter.is_empty() {
267        cmd.arg("--partial-selection");
268    }
269    if args.skip_unsupported {
270        cmd.arg("--skip-unsupported");
271    }
272    if let Some(settle) = &args.settle {
273        cmd.args(["--settle", settle]);
274    }
275    cmd.env_remove(crate::remote::DEV_COMMAND_ENV);
276    // One `run.id` for every group of this run, through the orchestration's
277    // own variable: the group's runner always sets the harness's.
278    cmd.env(crate::remote::DEV_RUN_ID_ENV, &args.run_id);
279    cmd.env(
280        DEV_URLS_ENV,
281        serde_json::to_string(urls).expect("a string map serialises"),
282    );
283    cmd.env(
284        DEV_ENV_ENV,
285        serde_json::to_string(group).expect("a string map serialises"),
286    );
287    // One JSON variable, never a variable per name: an `env` entry or secret
288    // named `PATH` or `NODE_OPTIONS` must not change the runner itself.
289    cmd.env(
290        DEV_SECRETS_ENV,
291        serde_json::to_string(vars).expect("a string map serialises"),
292    );
293    if args.json {
294        let out = cmd
295            .stderr(Stdio::inherit())
296            .output()
297            .map_err(|e| format!("could not run the suites: {e}"))?;
298        let doc = serde_json::from_slice(&out.stdout)
299            .map_err(|e| format!("the run printed no JSON document: {e}"))?;
300        Ok((out.status.success(), Some(doc)))
301    } else {
302        let status = cmd
303            .status()
304            .map_err(|e| format!("could not run the suites: {e}"))?;
305        Ok((status.success(), None))
306    }
307}
308
309/// Fold the groups' documents into one: counts summed, suites of the same
310/// name merged, the first error kept, `transport: "remote:dev"`.
311fn merge(docs: &[serde_json::Value]) -> TestRun {
312    use crate::test_json::{Case, Location, Suite, TestError};
313    let count = |d: &serde_json::Value, k: &str| d[k].as_u64().unwrap_or(0) as u32;
314    let mut run = TestRun::empty();
315    run.transport = Some(format!("remote:{DEV}"));
316    let suites = run.suites.get_or_insert_with(Vec::new);
317    for doc in docs {
318        run.passed += count(doc, "passed");
319        run.failed += count(doc, "failed");
320        run.skipped += count(doc, "skipped");
321        for s in doc["suites"].as_array().into_iter().flatten() {
322            let name = s["name"].as_str().unwrap_or_default().to_string();
323            let cases = s["cases"].as_array().into_iter().flatten().map(|c| Case {
324                name: c["name"].as_str().unwrap_or_default().to_string(),
325                outcome: c["outcome"].as_str().unwrap_or_default().to_string(),
326                message: c["message"].as_str().map(str::to_string),
327                location: c["location"].as_object().map(|l| Location {
328                    path: l["path"].as_str().unwrap_or_default().to_string(),
329                    line: l["line"].as_u64().unwrap_or(0) as u32,
330                    col: l["col"].as_u64().unwrap_or(0) as u32,
331                }),
332                retries: c["retries"].as_u64().map(|n| n as u32),
333                tier: c["tier"].as_str().map(str::to_string),
334                kind: c["kind"].as_str().map(str::to_string),
335                seed: c["seed"].as_str().map(str::to_string),
336                counterexample: c["counterexample"].as_str().map(str::to_string),
337                violation: c
338                    .get("violation")
339                    .map(crate::test_json::violation_from_json),
340                refusals: crate::test_json::refusals_from_json(c.get("refusals")),
341                // The groups run as `json`, whose document holds no durations.
342                duration_ms: None,
343            });
344            match suites.iter_mut().find(|x| x.name == name) {
345                Some(existing) => {
346                    // Every group's build reports the same skipped cases;
347                    // list each once.
348                    for case in cases {
349                        let repeat = case.outcome == "skipped"
350                            && existing
351                                .cases
352                                .iter()
353                                .any(|c| c.outcome == "skipped" && c.name == case.name);
354                        if repeat {
355                            run.skipped = run.skipped.saturating_sub(1);
356                        } else {
357                            existing.cases.push(case);
358                        }
359                    }
360                }
361                None => suites.push(Suite {
362                    name,
363                    kind: s["kind"].as_str().unwrap_or("integration").to_string(),
364                    cases: cases.collect(),
365                }),
366            }
367        }
368        if run.error.is_none()
369            && let Some(e) = doc["error"].as_object()
370        {
371            run.error = Some(TestError {
372                kind: e["kind"].as_str().unwrap_or("runtime").to_string(),
373                message: e
374                    .get("message")
375                    .and_then(|m| m.as_str())
376                    .map(str::to_string),
377                diagnostics: e
378                    .get("diagnostics")
379                    .and_then(|d| d.as_array())
380                    .into_iter()
381                    .flatten()
382                    .filter_map(|d| d.as_str().map(str::to_string))
383                    .collect(),
384                stderr: e.get("stderr").and_then(|m| m.as_str()).map(str::to_string),
385            });
386        }
387    }
388    run
389}
390
391/// Remove a readiness file left by an earlier run, so it is never read as
392/// this session being ready. A missing file is fine.
393pub fn remove_stale(ready: &Path) -> std::io::Result<()> {
394    match std::fs::remove_file(ready) {
395        Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e),
396        _ => Ok(()),
397    }
398}
399
400/// Two distinct localhost ports, free when asked: the base of the HTTP and
401/// the inspector allocations `bynk dev` makes.
402fn free_ports() -> std::io::Result<(u16, u16)> {
403    let a = std::net::TcpListener::bind("127.0.0.1:0")?;
404    let b = std::net::TcpListener::bind("127.0.0.1:0")?;
405    Ok((a.local_addr()?.port(), b.local_addr()?.port()))
406}
407
408/// A running `bynk dev`, stopped on drop.
409struct Dev {
410    child: Child,
411    stopped: bool,
412}
413
414impl Dev {
415    fn start(
416        bynk: &Path,
417        input: &Path,
418        ready: &Path,
419        log: &Path,
420        vars: &BTreeMap<String, String>,
421    ) -> std::io::Result<Dev> {
422        if let Some(dir) = ready.parent() {
423            std::fs::create_dir_all(dir)?;
424        }
425        // A file left by an earlier run that was killed must never read as
426        // this session being ready.
427        crate::dev_run::remove_stale(ready)?;
428        let (port, inspector) = free_ports()?;
429        let log_file = std::fs::File::create(log)?;
430        let mut cmd = Command::new(bynk);
431        cmd.arg("dev")
432            .arg(input)
433            .arg("--base-port")
434            .arg(port.to_string())
435            .arg("--inspect-port")
436            .arg(inspector.to_string())
437            .arg("--ready-json")
438            .arg(ready)
439            .arg("--");
440        // Only variables are passed through: never `--remote`.
441        for (k, v) in vars {
442            cmd.arg("--var").arg(format!("{k}:{v}"));
443        }
444        #[cfg(unix)]
445        std::os::unix::process::CommandExt::process_group(&mut cmd, 0);
446        let child = cmd
447            .stdin(Stdio::null())
448            .stdout(log_file.try_clone()?)
449            .stderr(log_file)
450            .spawn()?;
451        Ok(Dev {
452            child,
453            stopped: false,
454        })
455    }
456
457    /// Wait for the readiness file and read its URLs, which must all be
458    /// loopback.
459    fn wait_ready(&mut self, ready: &Path) -> Result<BTreeMap<String, String>, String> {
460        let deadline = Instant::now() + READY_TIMEOUT;
461        loop {
462            if let Ok(text) = std::fs::read_to_string(ready)
463                && let Ok(urls) = serde_json::from_str::<BTreeMap<String, String>>(&text)
464            {
465                if let Some(bad) = urls.values().find(|u| !is_loopback_http(u)) {
466                    return Err(format!("`bynk dev` reported `{bad}`, which isn't loopback"));
467                }
468                return Ok(urls);
469            }
470            if let Ok(Some(status)) = self.child.try_wait() {
471                return Err(format!("`bynk dev` exited before it was ready ({status})"));
472            }
473            if Instant::now() >= deadline {
474                return Err(format!(
475                    "`bynk dev` wasn't ready within {}s",
476                    READY_TIMEOUT.as_secs()
477                ));
478            }
479            std::thread::sleep(Duration::from_millis(250));
480        }
481    }
482
483    /// Stop `bynk dev`: SIGTERM to its own pid, so it stops its Workers; then,
484    /// always, its whole process group, so a wrangler or `workerd` it leaves
485    /// behind (because it exited first, or didn't stop them in time) never
486    /// leaks. Runs once; a second call (the drop after an explicit stop) does
487    /// nothing.
488    fn stop(&mut self) {
489        if self.stopped {
490            return;
491        }
492        self.stopped = true;
493        #[cfg(unix)]
494        {
495            let pid = self.child.id().to_string();
496            if !matches!(self.child.try_wait(), Ok(Some(_))) {
497                let _ = Command::new("kill").args(["-s", "TERM", &pid]).status();
498                let deadline = Instant::now() + STOP_GRACE;
499                while Instant::now() < deadline {
500                    if matches!(self.child.try_wait(), Ok(Some(_))) {
501                        break;
502                    }
503                    std::thread::sleep(Duration::from_millis(100));
504                }
505            }
506            // `bynk dev` leads its own process group, so `-<pid>` names
507            // exactly what it started and nothing else on the machine.
508            let _ = Command::new("kill")
509                .args(["-s", "KILL", "--", &format!("-{pid}")])
510                .stderr(Stdio::null())
511                .status();
512        }
513        let _ = self.child.kill();
514        let _ = self.child.wait();
515    }
516}
517
518impl Drop for Dev {
519    fn drop(&mut self) {
520        self.stop();
521    }
522}
523
524#[cfg(test)]
525mod tests {
526    use super::*;
527
528    /// A merged `--against dev` document keeps each case's `tier`, which the
529    /// groups' own documents carry.
530    #[test]
531    fn merging_the_groups_keeps_each_cases_tier() {
532        let doc = serde_json::json!({
533            "passed": 1, "failed": 0,
534            "suites": [{"name": "shop", "kind": "integration", "cases": [
535                {"name": "wired", "outcome": "pass", "tier": "system"},
536                {"name": "plain", "outcome": "pass"}
537            ]}]
538        });
539        let merged = merge(&[doc]);
540        let cases = &merged.suites.as_ref().unwrap()[0].cases;
541        assert_eq!(cases[0].tier.as_deref(), Some("system"));
542        assert_eq!(cases[1].tier, None);
543    }
544
545    #[test]
546    fn a_group_s_env_wins_and_other_secrets_get_the_test_value() {
547        let group = BTreeMap::from([
548            ("PAYMENTS".to_string(), "fake".to_string()),
549            ("AUTH_SECRET".to_string(), "s3".to_string()),
550        ]);
551        let secrets = BTreeSet::from(["AUTH_SECRET".to_string(), "HOOK_SECRET".to_string()]);
552        let vars = dev_vars(&group, &secrets);
553        assert_eq!(vars["PAYMENTS"], "fake");
554        assert_eq!(vars["AUTH_SECRET"], "s3");
555        assert_eq!(vars["HOOK_SECRET"], TEST_SECRET);
556    }
557
558    #[test]
559    fn a_stale_ready_file_is_removed_before_a_session_starts() {
560        let dir = std::env::temp_dir().join(format!("bynk-dev-stale-{}", std::process::id()));
561        std::fs::create_dir_all(&dir).unwrap();
562        let ready = dir.join(".bynk-dev-ready-0.json");
563        // An earlier, killed run's file, pointing somewhere bogus.
564        std::fs::write(&ready, r#"{"svc":"http://localhost:1"}"#).unwrap();
565        // `Dev::start` removes it before spawning, even when the spawn then
566        // fails (here: no such binary), so it can never read as ready.
567        let started = Dev::start(
568            &dir.join("no-such-bynk"),
569            &dir,
570            &ready,
571            &dir.join("dev.log"),
572            &BTreeMap::new(),
573        );
574        assert!(started.is_err());
575        assert!(!ready.exists(), "the stale file is gone");
576        remove_stale(&ready).expect("removing an absent file is fine");
577        let _ = std::fs::remove_dir_all(&dir);
578    }
579
580    #[test]
581    fn group_documents_merge_into_one() {
582        let a = serde_json::json!({"passed": 1, "failed": 0, "suites": [{"name": "s", "kind": "integration", "cases": [{"name": "a", "outcome": "pass"}]}], "transport": "remote:dev"});
583        let b = serde_json::json!({"passed": 0, "failed": 1, "suites": [{"name": "s", "kind": "integration", "cases": [{"name": "b", "outcome": "fail", "message": "m"}]}], "transport": "remote:dev"});
584        let run = merge(&[a, b]);
585        assert_eq!((run.passed, run.failed), (1, 1));
586        let suites = run.suites.expect("suites");
587        assert_eq!(suites.len(), 1);
588        assert_eq!(suites[0].cases.len(), 2);
589        assert_eq!(run.transport.as_deref(), Some("remote:dev"));
590    }
591}
bynk_driver/dev_run.rs

bynk_driver/
dev_run.rs
