bynk_driver/
remote.rs

1//! #2138 (end-to-end testing track §3.5, §3.7, §3.8): `test --against`, a run
2//! of the `system` suites against a real runtime.
3//!
4//! This module owns the **safety** of that run, before any request is made:
5//!
6//! - `--against` takes only a name. `dev` is reserved for the local `bynk dev`
7//!   the runner starts itself; anything else must be a target named under
8//!   `[test.targets]` in `bynk.toml`. A bare URL is refused.
9//! - A target marked `production = true` is refused.
10//! - A target's secrets are read only from the runner's environment, by the
11//!   names it lists, and each must be set.
12//! - The target and each Worker's URL are printed before the first request
13//!   ([`RemoteRun::banner`]).
14
15use std::collections::BTreeMap;
16use std::path::Path;
17
18use bynk_emit::project::{RemoteSkip, TestTarget};
19
20/// The reserved `--against` value that starts a local `bynk dev`.
21pub const DEV: &str = "dev";
22
23/// Internal: the `bynk` binary that can start `bynk dev`, set by `bynk test
24/// --against dev` for the `bynkc` it delegates to. Without it, `bynkc test
25/// --against dev` is refused.
26pub const DEV_COMMAND_ENV: &str = "BYNK_DEV_COMMAND";
27
28/// Internal: the URLs of a running `bynk dev`, by worker, for one group's run
29/// (set by the `--against dev` orchestration). Every one must be loopback.
30pub const DEV_URLS_ENV: &str = "BYNK_TEST_DEV_URLS";
31
32/// Internal: the `env` map of the cases one `bynk dev` serves.
33pub const DEV_ENV_ENV: &str = "BYNK_TEST_DEV_ENV";
34
35/// Internal: the secret values one `bynk dev` serves, by name, which the
36/// runner signs with. One JSON variable, so a name such as `PATH` never
37/// becomes a real environment variable of the runner.
38pub const DEV_SECRETS_ENV: &str = "BYNK_TEST_DEV_SECRETS";
39
40/// The variable the remote harness reads every secret value from, by name.
41pub const SECRETS_ENV: &str = "BYNK_TEST_SECRETS";
42
43/// The variable the remote harness reads `run.id` from. The runner always
44/// sets it for a remote run, so a value exported in the shell or a CI job is
45/// overwritten, never reused: each run's id is its own.
46pub const RUN_ID_ENV: &str = "BYNK_TEST_RUN_ID";
47
48/// Internal: the `run.id` every `bynk dev` group of one `--against dev` run
49/// shares (set by the orchestration). Read only with [`DEV_URLS_ENV`].
50pub const DEV_RUN_ID_ENV: &str = "BYNK_TEST_DEV_RUN_ID";
51
52/// A fresh `run.id` for a remote run: the time in milliseconds and the
53/// process id, in hex, so two runs don't collide on the same deployment.
54pub fn fresh_run_id() -> String {
55    let millis = std::time::SystemTime::now()
56        .duration_since(std::time::UNIX_EPOCH)
57        .map_or(0, |d| d.as_millis());
58    format!("r{millis:x}{:x}", std::process::id())
59}
60
61/// `--settle`'s default: a failing `expect` in a remote run is retried for up
62/// to five seconds (track §3.6).
63pub const DEFAULT_SETTLE_MS: u64 = 5_000;
64
65/// Parse a `--settle` duration (`5s`, `500ms`, `2m`, `0`) to milliseconds.
66pub fn parse_settle(value: &str) -> Result<u64, String> {
67    let bad =
68        || format!("`--settle {value}` is not a duration; write one such as `5s`, `500ms` or `0`");
69    let v = value.trim();
70    if v == "0" {
71        return Ok(0);
72    }
73    let (digits, factor) = if let Some(n) = v.strip_suffix("ms") {
74        (n, 1)
75    } else if let Some(n) = v.strip_suffix('s') {
76        (n, 1_000)
77    } else if let Some(n) = v.strip_suffix('m') {
78        (n, 60_000)
79    } else {
80        return Err(bad());
81    };
82    digits
83        .parse::<u64>()
84        .ok()
85        .and_then(|n| n.checked_mul(factor))
86        .ok_or_else(bad)
87}
88
89/// What `--against` asks for, once it passed the safety checks.
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub enum Against {
92    /// Run against this target now.
93    Run(RemoteRun),
94    /// `--against dev` from `bynk test`: start `bynk dev` with this `bynk`
95    /// binary, once per distinct `env`, and run each group against it.
96    StartDev(std::path::PathBuf),
97}
98
99/// Whether `url` is plain HTTP on a loopback host, the only kind of URL a
100/// `bynk dev` run accepts.
101pub fn is_loopback_http(url: &str) -> bool {
102    let Some(rest) = url.strip_prefix("http://") else {
103        return false;
104    };
105    let authority = rest.split('/').next().unwrap_or_default();
106    let host = match authority.rsplit_once(':') {
107        Some((h, port)) if !h.is_empty() && port.chars().all(|c| c.is_ascii_digit()) => h,
108        _ => authority,
109    };
110    matches!(host, "localhost" | "127.0.0.1" | "[::1]")
111}
112
113/// A remote run that passed every safety check.
114#[derive(Debug, Clone, PartialEq, Eq)]
115pub struct RemoteRun {
116    /// The target's name, as passed to `--against`.
117    pub target: String,
118    /// Each Worker's base URL, keyed by worker-directory name.
119    pub urls: BTreeMap<String, String>,
120    /// The non-secret values the target was deployed with.
121    pub vars: BTreeMap<String, String>,
122    /// The names of the target's secrets.
123    pub secrets: Vec<String>,
124    /// Under `--against dev`, the `env` map of the cases this run covers.
125    pub dev_env: Option<BTreeMap<String, String>>,
126    /// The values the drivers sign with, by secret name: a named target's
127    /// from the environment, a `bynk dev` group's from the orchestration.
128    pub secret_values: BTreeMap<String, String>,
129    /// `--settle`: how long a failing `expect` is retried, in milliseconds.
130    pub settle_ms: u64,
131    /// `run.id`: fresh for each run, or a `bynk dev` group's shared one.
132    pub run_id: String,
133    /// The `system` cases the build skipped (`--skip-unsupported`), with the
134    /// suite each belongs to.
135    pub skipped: Vec<(String, RemoteSkip)>,
136}
137
138impl RemoteRun {
139    /// `BYNK_TEST_TARGETS`: the URLs as the emitted remote harness reads them.
140    pub fn targets_json(&self) -> String {
141        serde_json::to_string(&self.urls).expect("a string map serialises")
142    }
143
144    /// `BYNK_TEST_SECRETS`: the secret values as the remote harness reads them.
145    pub fn secrets_json(&self) -> String {
146        serde_json::to_string(&self.secret_values).expect("a string map serialises")
147    }
148
149    /// The `transport` the JSON document reports.
150    pub fn transport(&self) -> String {
151        format!("remote:{}", self.target)
152    }
153
154    /// The lines printed before the first request: the target and each
155    /// Worker's URL, so a run against the wrong deployment is visible at once.
156    pub fn banner(&self) -> String {
157        let mut out = format!(
158            "running the `system` suites against target `{}`:\n",
159            self.target
160        );
161        let width = self.urls.keys().map(String::len).max().unwrap_or(0);
162        for (worker, url) in &self.urls {
163            out.push_str(&format!("  {worker:<width$}  {url}\n"));
164        }
165        out
166    }
167}
168
169/// Why `--against` was refused.
170#[derive(Debug, Clone, PartialEq, Eq)]
171pub enum Refusal {
172    /// `--against dev` from a front end that can't start `bynk dev`.
173    Dev(String),
174    /// A `bynk dev` URL that isn't plain HTTP on loopback.
175    NotLoopback(String),
176    /// A bare URL instead of a target name.
177    BareUrl(String),
178    /// No `[test.targets.<name>]` of that name.
179    Unknown { name: String, known: Vec<String> },
180    /// The target is marked `production = true`.
181    Production(String),
182    /// Secrets the target lists that the environment doesn't set.
183    MissingSecrets { target: String, names: Vec<String> },
184    /// `bynk.toml` couldn't be read.
185    Manifest(String),
186}
187
188impl std::fmt::Display for Refusal {
189    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
190        match self {
191            Refusal::Dev(why) => write!(f, "`--against dev` {why}"),
192            Refusal::NotLoopback(url) => write!(
193                f,
194                "`--against dev` runs only against a local `bynk dev`, but was given `{url}`, which isn't plain HTTP on localhost"
195            ),
196            Refusal::BareUrl(url) => write!(
197                f,
198                "`--against {url}` names a URL; a run against a deployment must use a target named under `[test.targets]` in `bynk.toml`, so a URL can't be pointed at by accident"
199            ),
200            Refusal::Unknown { name, known } if known.is_empty() => write!(
201                f,
202                "`--against {name}` names no target: `bynk.toml` has no `[test.targets]`. Add `[test.targets.{name}]` with each Worker's `url`, or use `--against dev`"
203            ),
204            Refusal::Unknown { name, known } => write!(
205                f,
206                "`--against {name}` names no target under `[test.targets]`; the targets are {}",
207                known
208                    .iter()
209                    .map(|k| format!("`{k}`"))
210                    .collect::<Vec<_>>()
211                    .join(", ")
212            ),
213            Refusal::Production(name) => write!(
214                f,
215                "target `{name}` is marked `production = true`; the test runner never runs against production"
216            ),
217            Refusal::MissingSecrets { target, names } => write!(
218                f,
219                "target `{target}` needs the secret{} {} from the environment, and {} not set",
220                if names.len() == 1 { "" } else { "s" },
221                names
222                    .iter()
223                    .map(|n| format!("`{n}`"))
224                    .collect::<Vec<_>>()
225                    .join(", "),
226                if names.len() == 1 {
227                    "it is"
228                } else {
229                    "they are"
230                }
231            ),
232            Refusal::Manifest(e) => write!(f, "{e}"),
233        }
234    }
235}
236
237/// Whether `value` reads as a URL rather than a target name.
238fn looks_like_url(value: &str) -> bool {
239    value.contains("://")
240        || value.starts_with("localhost")
241        || value.contains('/')
242        || value.parse::<std::net::SocketAddr>().is_ok()
243        || value.contains(':')
244}
245
246/// Check `--against <value>` for a project rooted at `project_root`, against
247/// the environment `env` (by name), and resolve what it asks for.
248///
249/// `dev` is a run against a local `bynk dev`: one group's run when
250/// [`DEV_URLS_ENV`] names the running Workers (each URL must be loopback),
251/// the orchestration when [`DEV_COMMAND_ENV`] names a `bynk` that can start
252/// it, and otherwise refused with `dev_refusal`.
253pub fn resolve(
254    value: &str,
255    project_root: &Path,
256    env: impl Fn(&str) -> Option<String>,
257    dev_refusal: &str,
258) -> Result<Against, Refusal> {
259    if value == DEV {
260        if let Some(urls) = env(DEV_URLS_ENV) {
261            let urls: BTreeMap<String, String> = serde_json::from_str(&urls).map_err(|e| {
262                Refusal::Manifest(format!("{DEV_URLS_ENV} is not a JSON object: {e}"))
263            })?;
264            if let Some(bad) = urls.values().find(|u| !is_loopback_http(u)) {
265                return Err(Refusal::NotLoopback(bad.clone()));
266            }
267            let json_map = |name: &str| -> Result<BTreeMap<String, String>, Refusal> {
268                match env(name) {
269                    Some(e) => serde_json::from_str(&e).map_err(|e| {
270                        Refusal::Manifest(format!("{name} is not a JSON object: {e}"))
271                    }),
272                    None => Ok(BTreeMap::new()),
273                }
274            };
275            let dev_env = json_map(DEV_ENV_ENV)?;
276            let secret_values = json_map(DEV_SECRETS_ENV)?;
277            let run_id = env(DEV_RUN_ID_ENV)
278                .filter(|id| !id.is_empty())
279                .unwrap_or_else(fresh_run_id);
280            return Ok(Against::Run(RemoteRun {
281                target: DEV.to_string(),
282                urls,
283                vars: BTreeMap::new(),
284                secrets: Vec::new(),
285                dev_env: Some(dev_env),
286                secret_values,
287                settle_ms: DEFAULT_SETTLE_MS,
288                run_id,
289                skipped: Vec::new(),
290            }));
291        }
292        if let Some(bynk) = env(DEV_COMMAND_ENV).filter(|p| !p.is_empty()) {
293            return Ok(Against::StartDev(bynk.into()));
294        }
295        return Err(Refusal::Dev(dev_refusal.to_string()));
296    }
297    if looks_like_url(value) {
298        return Err(Refusal::BareUrl(value.to_string()));
299    }
300    let targets = bynk_emit::project::try_read_test_targets(project_root)
301        .map_err(|e| Refusal::Manifest(e.to_string()))?;
302    let Some(target) = targets.get(value) else {
303        return Err(Refusal::Unknown {
304            name: value.to_string(),
305            known: targets.keys().cloned().collect(),
306        });
307    };
308    check_target(value, target, env).map(Against::Run)
309}
310
311/// The checks on a named target: not production, every listed secret set.
312pub fn check_target(
313    name: &str,
314    target: &TestTarget,
315    env: impl Fn(&str) -> Option<String>,
316) -> Result<RemoteRun, Refusal> {
317    if target.production {
318        return Err(Refusal::Production(name.to_string()));
319    }
320    let mut secret_values = BTreeMap::new();
321    let mut missing: Vec<String> = Vec::new();
322    for name in &target.secrets {
323        match env(name).filter(|v| !v.is_empty()) {
324            Some(value) => {
325                secret_values.insert(name.clone(), value);
326            }
327            None => missing.push(name.clone()),
328        }
329    }
330    if !missing.is_empty() {
331        return Err(Refusal::MissingSecrets {
332            target: name.to_string(),
333            names: missing,
334        });
335    }
336    Ok(RemoteRun {
337        target: name.to_string(),
338        urls: target.urls.clone(),
339        vars: target.vars.clone(),
340        secrets: target.secrets.clone(),
341        dev_env: None,
342        secret_values,
343        settle_ms: DEFAULT_SETTLE_MS,
344        run_id: fresh_run_id(),
345        skipped: Vec::new(),
346    })
347}
348
349#[cfg(test)]
350mod tests {
351    use super::*;
352
353    fn target(production: bool, secrets: &[&str]) -> TestTarget {
354        TestTarget {
355            urls: BTreeMap::from([("shop-orders".to_string(), "http://127.0.0.1:9".to_string())]),
356            secrets: secrets.iter().map(|s| s.to_string()).collect(),
357            production,
358            vars: BTreeMap::new(),
359        }
360    }
361
362    fn manifest(text: &str) -> std::path::PathBuf {
363        let dir =
364            std::env::temp_dir().join(format!("bynk-remote-{}-{}", std::process::id(), text.len()));
365        std::fs::create_dir_all(&dir).unwrap();
366        std::fs::write(dir.join("bynk.toml"), text).unwrap();
367        dir
368    }
369
370    #[test]
371    fn settle_durations_parse() {
372        assert_eq!(parse_settle("0"), Ok(0));
373        assert_eq!(parse_settle("500ms"), Ok(500));
374        assert_eq!(parse_settle("5s"), Ok(5_000));
375        assert_eq!(parse_settle("2m"), Ok(120_000));
376        for bad in ["5", "fast", "-1s", "1.5s", "ms"] {
377            assert!(parse_settle(bad).is_err(), "{bad}");
378        }
379    }
380
381    #[test]
382    fn a_bare_url_is_refused() {
383        for url in [
384            "https://shop.example.workers.dev",
385            "localhost:8787",
386            "127.0.0.1:8787",
387            "shop.example/x",
388        ] {
389            assert!(
390                matches!(
391                    resolve(url, Path::new("/nonexistent"), |_| None, "x"),
392                    Err(Refusal::BareUrl(_))
393                ),
394                "{url}"
395            );
396        }
397    }
398
399    #[test]
400    fn dev_is_reserved() {
401        let r = resolve("dev", Path::new("/nonexistent"), |_| None, "is not here");
402        assert_eq!(r, Err(Refusal::Dev("is not here".to_string())));
403    }
404
405    #[test]
406    fn dev_urls_must_be_loopback() {
407        let env = |urls: &'static str| move |n: &str| (n == DEV_URLS_ENV).then(|| urls.to_string());
408        let ok = resolve(
409            "dev",
410            Path::new("/x"),
411            env(r#"{"a":"http://localhost:8787","b":"http://127.0.0.1:9"}"#),
412            "x",
413        );
414        assert!(
415            matches!(ok, Ok(Against::Run(ref r)) if r.dev_env.is_some()),
416            "{ok:?}"
417        );
418        for bad in [
419            r#"{"a":"https://localhost:8787"}"#,
420            r#"{"a":"http://shop.example.workers.dev"}"#,
421            r#"{"a":"http://localhost.example.com:80"}"#,
422        ] {
423            assert!(
424                matches!(
425                    resolve("dev", Path::new("/x"), env(bad), "x"),
426                    Err(Refusal::NotLoopback(_))
427                ),
428                "{bad}"
429            );
430        }
431    }
432
433    #[test]
434    fn a_dev_group_s_secrets_arrive_in_one_json_variable() {
435        let env = |n: &str| match n {
436            DEV_URLS_ENV => Some(r#"{"a":"http://localhost:8787"}"#.to_string()),
437            DEV_SECRETS_ENV => Some(r#"{"PATH":"not-a-path","AUTH":"s"}"#.to_string()),
438            _ => None,
439        };
440        let Ok(Against::Run(run)) = resolve("dev", Path::new("/x"), env, "x") else {
441            panic!("a dev group run resolves");
442        };
443        // A secret named `PATH` is only ever a value for the drivers.
444        assert_eq!(run.secret_values["PATH"], "not-a-path");
445        assert!(run.secrets_json().contains("\"AUTH\":\"s\""));
446    }
447
448    /// #2175: a `bynk dev` group takes the orchestration's shared id; any
449    /// other run mints its own, whatever `BYNK_TEST_RUN_ID` says.
450    #[test]
451    fn each_run_gets_its_own_run_id_and_dev_groups_share_one() {
452        let group = |shared: Option<&'static str>| {
453            move |n: &str| match n {
454                DEV_URLS_ENV => Some(r#"{"a":"http://localhost:8787"}"#.to_string()),
455                DEV_RUN_ID_ENV => shared.map(str::to_string),
456                RUN_ID_ENV => Some("stale".to_string()),
457                _ => None,
458            }
459        };
460        let Ok(Against::Run(run)) = resolve("dev", Path::new("/x"), group(Some("shared")), "x")
461        else {
462            panic!("a dev group run resolves");
463        };
464        assert_eq!(run.run_id, "shared");
465        let Ok(Against::Run(run)) = resolve("dev", Path::new("/x"), group(None), "x") else {
466            panic!("a dev group run resolves");
467        };
468        assert!(
469            run.run_id.starts_with('r') && run.run_id != "stale",
470            "{}",
471            run.run_id
472        );
473    }
474
475    #[test]
476    fn dev_starts_only_with_a_bynk_that_can() {
477        let r = resolve(
478            "dev",
479            Path::new("/x"),
480            |n| (n == DEV_COMMAND_ENV).then(|| "/bin/bynk".to_string()),
481            "x",
482        );
483        assert_eq!(r, Ok(Against::StartDev("/bin/bynk".into())));
484    }
485
486    #[test]
487    fn an_unknown_name_lists_the_targets() {
488        let dir = manifest("[test.targets.preview]\nurl.shop-orders = \"http://127.0.0.1:9\"\n");
489        let r = resolve("staging", &dir, |_| None, "x");
490        assert_eq!(
491            r,
492            Err(Refusal::Unknown {
493                name: "staging".to_string(),
494                known: vec!["preview".to_string()]
495            })
496        );
497        let Ok(Against::Run(ok)) = resolve("preview", &dir, |_| None, "x") else {
498            panic!("preview resolves");
499        };
500        assert_eq!(ok.urls["shop-orders"], "http://127.0.0.1:9");
501        let _ = std::fs::remove_dir_all(&dir);
502    }
503
504    #[test]
505    fn a_production_target_is_refused() {
506        assert_eq!(
507            check_target("prod", &target(true, &[]), |_| None),
508            Err(Refusal::Production("prod".to_string()))
509        );
510    }
511
512    #[test]
513    fn secrets_come_from_the_environment_and_must_be_set() {
514        let t = target(false, &["AUTH_JWT_SECRET", "HOOK_SECRET"]);
515        let r = check_target("preview", &t, |n| {
516            (n == "AUTH_JWT_SECRET").then(|| "s".to_string())
517        });
518        assert_eq!(
519            r,
520            Err(Refusal::MissingSecrets {
521                target: "preview".to_string(),
522                names: vec!["HOOK_SECRET".to_string()]
523            })
524        );
525        assert!(check_target("preview", &t, |_| Some("s".to_string())).is_ok());
526    }
527
528    #[test]
529    fn the_banner_names_the_target_and_each_url() {
530        let run = check_target("preview", &target(false, &[]), |_| None).unwrap();
531        let banner = run.banner();
532        assert!(banner.contains("target `preview`"), "{banner}");
533        assert!(
534            banner.contains("shop-orders  http://127.0.0.1:9"),
535            "{banner}"
536        );
537    }
538}
bynk_driver/remote.rs

bynk_driver/
remote.rs
