shell: expand ${NAME} and the ${NAME<op>word} forms

docs/runtime/shell.mdx

@@ -327,6 +327,68 @@ const foo = "bar123; rm -rf /tmp";
 await $`FOO=${foo} bun -e 'console.log(process.env.FOO)'`; // bar123; rm -rf /tmp\n
 ```
 
+### Reading variables
+
+Read a variable with `$NAME` or `${NAME}`. Use the braces when text follows the name, because `$FOObaz` reads the variable `FOObaz`:
+
+```sh script.sh icon="file-code"
+FOO=bar
+echo ${FOO}baz
+```
+
+```txt
+barbaz
+```
+
+`${0}` to `${9}` read the same positional arguments as `$0` to `$9`.
+
+A form with an operator gives a default value, an alternative value, an assignment or an error:
+
+| Form                            | `NAME` is set | `NAME` is not set                            |
+| ------------------------------- | ------------- | -------------------------------------------- |
+| `${NAME:-word}`, `${NAME-word}` | its value     | `word`                                       |
+| `${NAME:+word}`, `${NAME+word}` | `word`        | nothing                                      |
+| `${NAME:=word}`, `${NAME=word}` | its value     | `word`, and `NAME` is set to `word`          |
+| `${NAME:?word}`, `${NAME?word}` | its value     | the command fails, and `word` is the message |
+
+With the `:`, a variable that is empty also counts as not set.
+
+```json package.json icon="file-json"
+{
+  "scripts": {
+    "dev": "next dev --port ${PORT:-3000}",
+    "deploy": "./deploy.sh --env ${DEPLOY_ENV:?set DEPLOY_ENV first}"
+  }
+}
+```
+
+The word is expanded only when it is the result. It can hold quotes, variables, `$(...)` and other forms. Unquoted whitespace in the word separates arguments, so `${TOKEN:+--token $TOKEN}` gives two arguments or none.
+
+These forms differ from `sh` in a few ways:
+
+- `${NAME:?word}` fails its command with exit code 1 before the command runs. The script then continues with the next command. Put `&&` after the command to stop what follows it: `true "${NAME:?}" && ./deploy.sh`.
+- Three cases do not fail the command. After `export NAME` with no value, `${NAME?word}` sees `NAME` as set. `VAR=$(cmd "${NAME:?}")` prints the message, but the assignment succeeds. An assignment that is a member of a pipeline, as in `VAR=${NAME:?} | cmd`, is not expanded.
+- The value of a variable is one argument. Bun Shell does not split it at whitespace, and a `*` in it is not a glob. `${NAME:=a b}` is one argument.
+- A form in double quotes that expands to nothing gives no argument, as `"$NAME"` does.
+- A `*` or a `{a,b}` expands only in the last argument that a word gives. In `*.md${EXTRA:+ $EXTRA}`, the `*.md` stays as written when `EXTRA` is set. Put a space before the form.
+- An assignment before a command does not read an earlier one of the same command. In `A=1 B=${A:-2} cmd`, `B` is `2`.
+- Other `${...}` forms, such as `${#NAME}`, `${NAME%pattern}`, `${NAME/a/b}` and `${10}`, are not supported. Bun Shell does not expand them: the `$` is text, and the rest is read as ordinary shell syntax.
+- When a form has no closing `}`, Bun Shell reads every `${` of the script as text. When that script also has a `:?` or a `?` form, Bun Shell reports `Unclosed parameter expansion` and runs nothing.
+
+<Note>
+
+In a `$` template literal, `${...}` is a JavaScript interpolation. `\${...}` is text there, because Bun reads the [`raw`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals#raw_strings) template. Write `$NAME` to read a variable, and put it in double quotes when text follows the name. To use a form with an operator, put it in a constant `{ raw }` object (see `$.escape` under Utilities):
+
+```js
+import { $ } from "bun";
+
+await $`FOO=bar; echo "$FOO"baz`; // barbaz\n
+await $`echo ${{ raw: "${PORT:-3000}" }}`; // 3000\n
+await $`echo \${PORT:-3000}`; // ${PORT:-3000}\n
+```
+
+</Note>
+
 ### Changing the environment variables
 
 By default, all commands use `process.env` as their environment variables.

src/runtime/shell/IOWriter.rs

@@ -86,6 +86,7 @@ pub(crate) enum WriterTag {
     Builtin,
     Cmd,
     CondExpr,
+    Assigns,
     Pipeline,
     /// `subproc::PipeReader::CapturedWriter` — heap-allocated, addressed via
     /// `ChildPtr::raw` rather than `node`.
@@ -1161,13 +1162,16 @@ pub(crate) fn on_io_writer_chunk(
     err: Option<sys::SystemError>,
 ) -> Yield {
     use crate::shell::builtin::Builtin;
-    use crate::shell::states::{cmd, cond_expr, pipeline};
+    use crate::shell::states::{assigns, cmd, cond_expr, pipeline};
     match child.tag {
         WriterTag::Builtin => Builtin::on_io_writer_chunk(interp, child.node, written, err),
         WriterTag::Cmd => cmd::Cmd::on_io_writer_chunk(interp, child.node, written, err),
         WriterTag::CondExpr => {
             cond_expr::CondExpr::on_io_writer_chunk(interp, child.node, written, err)
         }
+        WriterTag::Assigns => {
+            assigns::Assigns::on_io_writer_chunk(interp, child.node, written, err)
+        }
         WriterTag::Pipeline => {
             pipeline::Pipeline::on_io_writer_chunk(interp, child.node, written, err)
         }

src/runtime/shell/interpreter.rs

@@ -119,6 +119,8 @@ pub(crate) enum Node {
     Subshell(Subshell),
 }
 
+const _: () = assert!(size_of::<Node>() == 176);
+
 impl Node {
     pub(crate) fn kind(&self) -> StateKind {
         match self {
@@ -172,6 +174,22 @@ impl Node {
             Node::Subshell(s) => Some(&mut s.base),
         }
     }
+
+    /// The streams of a state. `Assigns` and `Expansion` have none of their own.
+    pub(crate) fn io(&self) -> Option<&IO> {
+        match self {
+            Node::Free | Node::Assigns(_) | Node::Expansion(_) => None,
+            Node::Script(s) => Some(&s.io),
+            Node::Stmt(s) => Some(&s.io),
+            Node::Binary(s) => Some(&s.io),
+            Node::Pipeline(s) => Some(&s.io),
+            Node::Cmd(s) => Some(&s.io),
+            Node::If(s) => Some(&s.io),
+            Node::CondExpr(s) => Some(&s.io),
+            Node::Async(s) => Some(&s.io),
+            Node::Subshell(s) => Some(&s.io),
+        }
+    }
 }
 
 /// Generate `Interpreter::as_<kind>{,_mut}` typed accessors. These panic on
@@ -419,22 +437,14 @@ impl Interpreter {
         out_parser: &mut Option<bun_shell_parser::Parser<'a>>,
         out_lex_result: &mut Option<bun_shell_parser::LexResult<'a>>,
     ) -> crate::Result<bun_shell_parser::ast::Script<'a>> {
-        use crate::shell::shell_body::{LexerAscii, LexerUnicode, ParseError, Parser};
-        let lex_result = if bun_core::is_all_ascii(src) {
-            let mut lexer = LexerAscii::new(arena, src, jsstrings_to_escape, jsobjs_len);
-            lexer.lex().map_err(crate::Error::from)?;
-            lexer.get_result()
-        } else {
-            let mut lexer = LexerUnicode::new(arena, src, jsstrings_to_escape, jsobjs_len);
-            lexer.lex().map_err(crate::Error::from)?;
-            lexer.get_result()
-        };
-        if !lex_result.errors.is_empty() {
-            *out_lex_result = Some(lex_result);
-            return Err(ParseError::Lex.into());
-        }
-        *out_parser = Some(Parser::new(arena, lex_result));
-        Ok(out_parser.as_mut().unwrap().parse()?)
+        bun_shell_parser::parse_script(
+            arena,
+            src,
+            jsstrings_to_escape,
+            jsobjs_len,
+            out_parser,
+            out_lex_result,
+        )
     }
 
     /// Builds the root `ShellExecEnv` (export env from the event loop's
@@ -1467,51 +1477,52 @@ impl Interpreter {
     }
 
     /// Appends the value of `$N` to
-    /// `out`. Takes the relevant interpreter fields by-part so callers can
+    /// `out`, and returns whether `$N` is set. Takes the relevant interpreter fields by-part so callers can
     /// split-borrow alongside the node arena.
     ///
     /// # Safety
     /// `command_ctx` must be null or point to a live `ContextData` that
     /// outlives this call.
+    #[inline(always)] // `Expansion` runs this for each `$N`, and `${N<op>word}` is a second caller.
     pub(crate) fn append_var_argv(
         out: &mut Vec<u8>,
         original_int: u8,
         event_loop: EventLoopHandle,
         command_ctx: *mut bun_options_types::context::ContextData,
         vm_args_utf8: &mut Vec<bun_core::Utf8Bytes<'static>>,
-    ) {
+    ) -> bool {
         let mut int = original_int;
         match event_loop {
             EventLoopHandle::Js { .. } => {
                 if int == 0 {
                     if let Ok(p) = bun_core::self_exe_path() {
                         out.extend_from_slice(p.as_bytes());
                     }
-                    return;
+                    return true;
                 }
                 int -= 1;
 
                 let vm_ptr = event_loop
                     .bun_vm()
                     .cast::<bun_jsc::virtual_machine::VirtualMachine>();
                 if vm_ptr.is_null() {
-                    return;
+                    return false;
                 }
                 // SAFETY: `bun_vm()` on a JS event loop returns the live
                 // `*VirtualMachine` owning that loop.
                 let vm = unsafe { &*vm_ptr };
                 if let Some(main) = vm.main() {
                     if int == 0 {
                         out.extend_from_slice(main);
-                        return;
+                        return true;
                     }
                     int -= 1;
                 }
 
                 if let Some(worker) = vm.worker_ref() {
                     let argv = worker.argv();
                     if int as usize >= argv.len() {
-                        return;
+                        return false;
                     }
                     if vm_args_utf8.len() != argv.len() {
                         vm_args_utf8.reserve(argv.len());
@@ -1521,30 +1532,33 @@ impl Interpreter {
                         }
                     }
                     out.extend_from_slice(vm_args_utf8[int as usize].slice());
-                    return;
+                    return true;
                 }
 
                 if (int as usize) < vm.argv.len() {
                     out.extend_from_slice(&vm.argv[int as usize]);
+                    return true;
                 }
+                false
             }
             EventLoopHandle::Mini(_) => {
                 if command_ctx.is_null() {
-                    return;
+                    return false;
                 }
                 // SAFETY: `command_ctx` is the process-global `ContextData`
                 // (see `init`); it outlives the interpreter.
                 let ctx = unsafe { &*command_ctx };
                 if int as usize > ctx.passthrough.len() {
-                    return;
+                    return false;
                 }
                 if int == 0 {
                     if let Some(last) = ctx.positionals.last() {
                         out.extend_from_slice(last);
                     }
-                    return;
+                    return true;
                 }
                 out.extend_from_slice(&ctx.passthrough[int as usize - 1]);
+                true
             }
         }
     }

src/runtime/shell/mod.rs

@@ -143,7 +143,7 @@ pub(crate) use bun_shell_parser::{escape_8bit, needs_escape_utf8_ascii_latin1};
 // layout) at the arena/state-machine boundary.
 pub(crate) mod ast {
     use bun_shell_parser::parse::ast as p;
-    pub(crate) use p::{BinaryOp, CondExprOp, IoKind, RedirectFlags};
+    pub(crate) use p::{BinaryOp, CondExprOp, IoKind, ParamAction, RedirectFlags};
 
     pub(crate) type SmolList<T, const N: usize> = bun_shell_parser::parse::SmolList<'static, T, N>;
     pub(crate) type Script = p::Script<'static>;
@@ -160,6 +160,9 @@ pub(crate) mod ast {
     pub(crate) type Assign = p::Assign<'static>;
     pub(crate) type Atom = p::Atom<'static>;
     pub(crate) type SimpleAtom = p::SimpleAtom<'static>;
+    pub(crate) type CmdSubst = p::CmdSubst<'static>;
+    pub(crate) type ParamExp = p::ParamExp<'static>;
+    pub(crate) type ParamTarget = p::ParamTarget<'static>;
 }
 
 // Canonical 4-variant shell error enum. Defined in

src/runtime/shell/shell_body.rs

@@ -29,9 +29,8 @@ pub(crate) use super::subproc; // declared once in `shell/mod.rs`
 // (still-draft) JSC bridge below. This file keeps the JSC-coupled half
 // (ShellErr, GlobalJS/Mini, shell_cmd_from_js, ShellSrcBuilder, TestingAPIs).
 pub(crate) use bun_shell_parser::parse::{
-    IfClauseTok, LEX_JS_OBJREF_PREFIX, LEX_JS_REF_TERMINATOR, LEX_JS_STRING_PREFIX, LexerAscii,
-    LexerUnicode, ParseError, Parser, Token, ast, is_if_clause_keyword_bunstr, needs_escape_bunstr,
-    needs_escape_utf8_ascii_latin1,
+    IfClauseTok, LEX_JS_OBJREF_PREFIX, LEX_JS_REF_TERMINATOR, LEX_JS_STRING_PREFIX, Token, ast,
+    is_if_clause_keyword_bunstr, needs_escape_bunstr, needs_escape_utf8_ascii_latin1,
 };
 
 #[cfg(windows)]
@@ -127,6 +126,9 @@ pub(crate) mod test {
         CloseParen,
         Var(&'a [u8]),
         VarArgv(u8),
+        ParamBegin(ast::ParamOp),
+        ParamSpace(&'a [u8]),
+        ParamEnd,
         Text(&'a [u8]),
         SingleQuotedText(&'a [u8]),
         DoubleQuotedText(&'a [u8]),
@@ -142,6 +144,11 @@ pub(crate) mod test {
             match the_token {
                 Token::Var(txt) => TestToken::Var(&buf[txt.start as usize..txt.end as usize]),
                 Token::VarArgv(int) => TestToken::VarArgv(int),
+                Token::ParamBegin(op) => TestToken::ParamBegin(op),
+                Token::ParamSpace(txt) => {
+                    TestToken::ParamSpace(&buf[txt.start as usize..txt.end as usize])
+                }
+                Token::ParamEnd => TestToken::ParamEnd,
                 Token::Text(txt) => TestToken::Text(&buf[txt.start as usize..txt.end as usize]),
                 Token::SingleQuotedText(txt) => {
                     TestToken::SingleQuotedText(&buf[txt.start as usize..txt.end as usize])
@@ -214,6 +221,13 @@ pub(crate) mod test {
                     w.write_char('}')
                 }
                 T::VarArgv(n) => write!(w, "{{\"VarArgv\":{}}}", n),
+                T::ParamBegin(op) => write!(w, "{{\"ParamBegin\":\"{}\"}}", op.as_str()),
+                T::ParamSpace(s) => {
+                    w.write_str("{\"ParamSpace\":")?;
+                    encode_json_string(w, s)?;
+                    w.write_char('}')
+                }
+                T::ParamEnd => unit!("ParamEnd"),
                 T::Text(s) => {
                     w.write_str("{\"Text\":")?;
                     encode_json_string(w, s)?;
@@ -551,7 +565,7 @@ impl<'a> ShellSrcBuilder<'a> {
             // marker recognized regardless of quote context (e.g. inside single quotes).
             if needs_escape_bunstr(&bunstr)
                 || is_if_clause_keyword_bunstr(&bunstr)
-                || self.outbuf_ends_with_var_ref()
+                || bun_shell_parser::ends_in_param_head(self.outbuf)
             {
                 self.append_js_str_ref(bunstr)?;
                 return Ok(true);
@@ -581,7 +595,7 @@ impl<'a> ShellSrcBuilder<'a> {
         if ALLOW_ESCAPE {
             if needs_escape_utf8_ascii_latin1(utf8)
                 || IfClauseTok::from_text(utf8).is_some()
-                || self.outbuf_ends_with_var_ref()
+                || bun_shell_parser::ends_in_param_head(self.outbuf)
             {
                 self.append_js_str_ref(BunString::clone_utf8(utf8))?;
                 return Ok(true);
@@ -592,17 +606,6 @@ impl<'a> ShellSrcBuilder<'a> {
         Ok(true)
     }
 
-    fn outbuf_ends_with_var_ref(&self) -> bool {
-        match self
-            .outbuf
-            .iter()
-            .rposition(|b| !(b.is_ascii_alphanumeric() || *b == b'_'))
-        {
-            Some(i) => self.outbuf[i] == b'$',
-            None => false,
-        }
-    }
-
     pub(crate) fn append_utf16_impl(&mut self, utf16: &[u16]) -> Result<(), bun_alloc::AllocError> {
         let size = simdutf::length::utf8::from::utf16::le(utf16);
         self.outbuf.reserve(size);
@@ -706,20 +709,13 @@ pub(crate) mod testing_apis {
         )?;
 
         let jsobjs_len: u32 = u32::try_from(marked_argument_buffer.size()).expect("int cast");
-        let lex_result = 'brk: {
-            if strings::is_all_ascii(&script[..]) {
-                let mut lexer = LexerAscii::new(&arena, &script[..], &jsstrings[..], jsobjs_len);
-                if let Err(err) = lexer.lex() {
+        let lex_result =
+            match bun_shell_parser::lex_script(&arena, &script[..], &jsstrings[..], jsobjs_len) {
+                Ok(lex_result) => lex_result,
+                Err(err) => {
                     return Err(global.throw_error(crate::Error::from(err), "failed to lex shell"));
                 }
-                break 'brk lexer.get_result();
-            }
-            let mut lexer = LexerUnicode::new(&arena, &script[..], &jsstrings[..], jsobjs_len);
-            if let Err(err) = lexer.lex() {
-                return Err(global.throw_error(crate::Error::from(err), "failed to lex shell"));
-            }
-            lexer.get_result()
-        };
+            };
 
         if !lex_result.errors.is_empty() {
             let str = lex_result.combine_errors(&arena);