From f184a8bcd9c70c2e0e4c853e37f82771bee928ff Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 10:11:40 -0500 Subject: [PATCH 1/8] add --shell-quote option --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/command_line_args.rs | 6 ++++++ src/parser.rs | 11 ++++++++++- 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index cc3f2ba..b169210 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,6 +523,7 @@ dependencies = [ "num_cpus", "predicates", "regex", + "shlex", "thiserror", "tokio", "tracing", @@ -571,6 +572,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook-registry" version = "1.4.8" diff --git a/Cargo.toml b/Cargo.toml index e36602d..ac4bf51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ indicatif = "0.18" itertools = "0.15" num_cpus = "1" regex = "1" +shlex = "2" thiserror = "2" tokio = { version = "1", features = ["full"] } tracing = "0.1" diff --git a/src/command_line_args.rs b/src/command_line_args.rs index 32158bd..d334222 100644 --- a/src/command_line_args.rs +++ b/src/command_line_args.rs @@ -52,6 +52,12 @@ pub struct CommandLineArgs { #[arg(short, long)] pub shell: bool, + /// Use shell quoting for command arguments. + /// + /// Each command line is passed to " " as a single argument with shell quoting applied to each argument. + #[arg(long)] + pub shell_quote: bool, + /// Timeout seconds for running commands. Defaults to infinite timeout if not specified. #[arg(short, long, value_parser = Self::parse_timeout_seconds)] pub timeout_seconds: Option, diff --git a/src/parser.rs b/src/parser.rs index b97efe0..c0c5cd0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -39,6 +39,7 @@ impl ParsedCommand { /// Applies shell wrapping (if configured) and converts to OwnedCommandAndArgs. pub struct CommandBuilder { shell_command_and_args: Option>, + shell_quote: bool, } impl CommandBuilder { @@ -53,6 +54,7 @@ impl CommandBuilder { }; Self { shell_command_and_args, + shell_quote: command_line_args.shell_quote, } } @@ -62,7 +64,14 @@ impl CommandBuilder { Some(shell_command_and_args) => { let mut result = Vec::with_capacity(shell_command_and_args.len() + 1); result.extend(shell_command_and_args.iter().cloned()); - result.push(parsed.command_and_args.join(" ")); + + if self.shell_quote { + result.push( + shlex::try_join(parsed.command_and_args.iter().map(|s| s.as_str())).ok()?, + ); + } else { + result.push(parsed.command_and_args.join(" ")); + } result } }; From f4e979579b1f27d4a15ec62cd14cd713d20f91b0 Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 10:24:59 -0500 Subject: [PATCH 2/8] adding tests for --shell-quote --- tests/integration_tests.rs | 103 +++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 494d5ab..c05b097 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -733,3 +733,106 @@ E"#; .stdout(predicate::str::contains("1\n").count(5)) .stderr(predicate::str::is_empty()); } + +#[test] +fn test_shell_quote_with_spaces() { + // Test shell_quote with arguments containing spaces + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("echo") + .arg(":::") + .arg("hello world") + .arg("foo bar") + .assert() + .success() + .stdout( + (predicate::str::contains("hello world\n").count(1)) + .and(predicate::str::contains("foo bar\n").count(1)), + ) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_with_special_characters() { + // Test shell_quote with special shell characters like $, ", ', etc. + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("echo") + .arg(":::") + .arg("$VAR") + .arg("'single quotes'") + .arg(" 'single quote and spaces' ") + .arg(r#" "double quote and spaces" "#) + .assert() + .success() + .stdout( + (predicate::str::contains("$VAR\n").count(1)) + .and(predicate::str::contains("'single quotes'\n").count(1)) + .and(predicate::str::contains(" 'single quote and spaces' \n").count(1)) + .and( + predicate::str::contains(r#" "double quote and spaces" "#.to_owned() + "\n") + .count(1), + ), + ) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_with_placeholders() { + // Test shell_quote combined with placeholder substitution + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("echo") + .arg("arg={}") + .arg(":::") + .arg("hello world") + .arg("foo bar") + .assert() + .success() + .stdout( + (predicate::str::contains("arg=hello world\n").count(1)) + .and(predicate::str::contains("arg=foo bar\n").count(1)), + ) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_with_regex() { + // Test shell_quote combined with regex capture groups + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("-r") + .arg("(.*):(.*)") + .arg("echo") + .arg("key={1}") + .arg("val={2}") + .arg(":::") + .arg("my key:my value") + .assert() + .success() + .stdout(predicate::str::contains("key=my key val=my value\n")) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_without_shell_option() { + // shell_quote should have no effect when shell is not enabled + rust_parallel() + .arg("-j1") + .arg("--shell-quote") + .arg("echo") + .arg(":::") + .arg("hello world") + .assert() + .success() + .stdout(predicate::str::contains("hello world\n")) + .stderr(predicate::str::is_empty()); +} From 5356dc7534adc37a2c314849644927e34695bc6f Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 10:28:47 -0500 Subject: [PATCH 3/8] add test --- tests/integration_tests.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index c05b097..02758e4 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -286,6 +286,28 @@ fn runs_shell_function_from_stdin_j1() { .stderr(predicate::str::is_empty()); } +#[test] +fn runs_shell_function_from_stdin_with_shell_quote_j1() { + // Test with null-separated stdin containing special characters: ' " and backticks + // When shell_quote is enabled, these special characters should be properly quoted + let stdin = "hello'world\0foo\"bar\0baz`cmd`"; + + rust_parallel() + .write_stdin(stdin) + .arg("-0") + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("--shell-path=./dummy_shell.sh") + .arg("shell_function") + .assert() + .success() + .stdout(predicate::eq( + "dummy_shell arg1=-c arg2=shell_function \"hello'world\"\ndummy_shell arg1=-c arg2=shell_function 'foo\"bar'\ndummy_shell arg1=-c arg2=shell_function 'baz`cmd`'\n", + )) + .stderr(predicate::str::is_empty()); +} + #[test] fn runs_shell_function_from_file_j1() { rust_parallel() From f8abe36065da6337dc3dc4ae5b66d679b10b540c Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 10:31:02 -0500 Subject: [PATCH 4/8] add shell quote to manual --- scripts/generate_manual.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/generate_manual.sh b/scripts/generate_manual.sh index 7e989ca..e74f400 100755 --- a/scripts/generate_manual.sh +++ b/scripts/generate_manual.sh @@ -29,6 +29,7 @@ echo ' 1. [Numbered Capture Groups](#numbered-capture-groups) 1. [Capture Group Special Characters](#capture-group-special-characters) 1. [Shell Commands](#shell-commands) +1. [Shell Quote](#shell-quote) 1. [Bash Function](#bash-function) 1. [Function Setup](#function-setup) 1. [Demo of command line arguments](#demo-of-command-line-arguments) @@ -425,6 +426,26 @@ echo -e '$ rust-parallel -s -r \x27(?P.*) (?P.*)\x27 \x27FOO={arg1}; $RUST_PARALLEL -s -r '(?P.*) (?P.*)' 'FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"' ::: A B ::: CAT DOG echo '```' +echo '## Shell Quote + +When using shell mode (`-s`), the `--shell-quote` option applies shell escaping to each argument before passing it to the shell. + +This is useful when arguments contain special shell characters like `$`, backticks, single quotes, or double quotes that should be treated as literal strings and not interpreted by the shell. + +Without `--shell-quote`, special characters can cause command injection or unexpected shell behavior. With `--shell-quote` enabled, arguments are properly escaped. +' + +echo '```' +echo "$ echo -e \"hello\$world\nfoo\`cmd\`\" | rust-parallel -s -0 --shell-quote echo" +echo -e "hello\$world\nfoo\`cmd\`" | $RUST_PARALLEL -s -0 --shell-quote echo +echo '```' + +echo 'Without `--shell-quote`, the special characters would be interpreted by the shell:' +echo '```' +echo "$ echo -e \"hello\$world\nfoo\`cmd\`\" | rust-parallel -s -0 echo" +echo -e "hello\$world\nfoo\`cmd\`" | $RUST_PARALLEL -s -0 echo +echo '```' + echo '## Bash Function `-s` shell mode can be used to invoke an arbitrary bash function. From 449bbd641ad74a43b63562056b04826afbc67be8 Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 10:43:53 -0500 Subject: [PATCH 5/8] update manual for shell-quote --- scripts/generate_manual.sh | 84 +++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 14 deletions(-) diff --git a/scripts/generate_manual.sh b/scripts/generate_manual.sh index e74f400..d064dc3 100755 --- a/scripts/generate_manual.sh +++ b/scripts/generate_manual.sh @@ -29,12 +29,12 @@ echo ' 1. [Numbered Capture Groups](#numbered-capture-groups) 1. [Capture Group Special Characters](#capture-group-special-characters) 1. [Shell Commands](#shell-commands) -1. [Shell Quote](#shell-quote) 1. [Bash Function](#bash-function) 1. [Function Setup](#function-setup) 1. [Demo of command line arguments](#demo-of-command-line-arguments) 1. [Demo of function and command line arguments from stdin](#demo-of-function-and-command-line-arguments-from-stdin) 1. [Demo of function and initial arguments on command line, additional arguments from stdin](#demo-of-function-and-initial-arguments-on-command-line-additional-arguments-from-stdin) +1. [Shell Quote](#shell-quote) ' echo '## Command line options' @@ -426,24 +426,17 @@ echo -e '$ rust-parallel -s -r \x27(?P.*) (?P.*)\x27 \x27FOO={arg1}; $RUST_PARALLEL -s -r '(?P.*) (?P.*)' 'FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"' ::: A B ::: CAT DOG echo '```' -echo '## Shell Quote - -When using shell mode (`-s`), the `--shell-quote` option applies shell escaping to each argument before passing it to the shell. +echo '## Shell Commands -This is useful when arguments contain special shell characters like `$`, backticks, single quotes, or double quotes that should be treated as literal strings and not interpreted by the shell. +Shell commands can be written using `-s` shell mode. -Without `--shell-quote`, special characters can cause command injection or unexpected shell behavior. With `--shell-quote` enabled, arguments are properly escaped. -' +Multiline commands can be written using `;`. -echo '```' -echo "$ echo -e \"hello\$world\nfoo\`cmd\`\" | rust-parallel -s -0 --shell-quote echo" -echo -e "hello\$world\nfoo\`cmd\`" | $RUST_PARALLEL -s -0 --shell-quote echo -echo '```' +Environment variables, `$` characters, nested commands and much more are possible:' -echo 'Without `--shell-quote`, the special characters would be interpreted by the shell:' echo '```' -echo "$ echo -e \"hello\$world\nfoo\`cmd\`\" | rust-parallel -s -0 echo" -echo -e "hello\$world\nfoo\`cmd\`" | $RUST_PARALLEL -s -0 echo +echo -e '$ rust-parallel -s -r \x27(?P.*) (?P.*)\x27 \x27FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"\x27 ::: A B ::: CAT DOG' +$RUST_PARALLEL -s -r '(?P.*) (?P.*)' 'FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"' ::: A B ::: CAT DOG echo '```' echo '## Bash Function @@ -525,3 +518,66 @@ cat test | $RUST_PARALLEL -s logargs hello rm -f test echo '```' + +echo '## Shell Quote + +When using shell mode (`-s`), the `--shell-quote` option applies shell escaping to each argument before passing it to the shell. + +This is useful when arguments contain special shell characters like `$`, backticks, single quotes, or double quotes that should be treated as literal strings and not interpreted by the shell. + +Without `--shell-quote`, special characters can cause command injection or unexpected shell behavior. With `--shell-quote` enabled, arguments are properly escaped. + +Consider a bash function that uses special characters: +' + +echo '```' + +echo '$ demonstrate_shell_quote() { + for arg in "$@"; do + echo "arg: $arg" + done +}' + +demonstrate_shell_quote() { + for arg in "$@"; do + echo "arg: $arg" + done +} + +echo ' +$ export -f demonstrate_shell_quote' +export -f demonstrate_shell_quote + +echo '``` + +With `--shell-quote`, special characters in arguments are properly escaped: +' + +echo '```' +echo '$ cat >./test <<'"'"'EOL'"'"' +hello$world +foo`cmd` +bar'"'"'baz'"'"' +EOL' +cat >./test <<'EOL' +hello$world +foo`cmd` +bar'baz' +EOL + +echo ' +$ cat test | rust-parallel -s --shell-quote demonstrate_shell_quote' +cat test | $RUST_PARALLEL -s --shell-quote demonstrate_shell_quote +echo '```' + +echo 'Without `--shell-quote`, special characters are interpreted by the shell, causing errors: +' +echo '```' +echo '$ cat test | rust-parallel -s demonstrate_shell_quote' +set +e +cat test | $RUST_PARALLEL -s demonstrate_shell_quote 2>&1 | head -20 +set -e + +rm -f test + +echo '```' From 109598736321edfbee75ef9609113d11b717e162 Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 10:46:07 -0500 Subject: [PATCH 6/8] Remove extra Shell Commands section --- scripts/generate_manual.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/generate_manual.sh b/scripts/generate_manual.sh index d064dc3..9eacb15 100755 --- a/scripts/generate_manual.sh +++ b/scripts/generate_manual.sh @@ -426,19 +426,6 @@ echo -e '$ rust-parallel -s -r \x27(?P.*) (?P.*)\x27 \x27FOO={arg1}; $RUST_PARALLEL -s -r '(?P.*) (?P.*)' 'FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"' ::: A B ::: CAT DOG echo '```' -echo '## Shell Commands - -Shell commands can be written using `-s` shell mode. - -Multiline commands can be written using `;`. - -Environment variables, `$` characters, nested commands and much more are possible:' - -echo '```' -echo -e '$ rust-parallel -s -r \x27(?P.*) (?P.*)\x27 \x27FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"\x27 ::: A B ::: CAT DOG' -$RUST_PARALLEL -s -r '(?P.*) (?P.*)' 'FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"' ::: A B ::: CAT DOG -echo '```' - echo '## Bash Function `-s` shell mode can be used to invoke an arbitrary bash function. From 00b9b9d51bad3725c909edc2dfbfdb7b653c9e6e Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 11:18:35 -0500 Subject: [PATCH 7/8] update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8700b59..0b2f8bd 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Similar interface to [GNU Parallel](https://www.gnu.org/software/parallel/parall * Automatic parallelism to all cpus, or [configure manually](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#parallelism) * Transform inputs with [variables](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#automatic-variables) or [regular expressions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#regular-expression) * Prevent [output interleaving](https://github.com/aaronriekenberg/rust-parallel/wiki/Output-Interleaving) and maintain input order with `-k`/`--keep-order` -* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands) +* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands), configurable (shell quoting)[https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-quote] * [TUI progress bar](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#progress-bar) using [indicatif](https://github.com/console-rs/indicatif) * [Path cache](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#path-cache) * [Command timeouts](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#timeout) @@ -75,6 +75,7 @@ The same `cargo install rust-parallel` command will also update to the latest ve * [itertools](https://docs.rs/itertools/latest/itertools/) using [`multi_cartesian_product`](https://docs.rs/itertools/latest/itertools/trait.Itertools.html#method.multi_cartesian_product) to process `:::` command line inputs. * [indicatif](https://github.com/console-rs/indicatif) optional TUI progress bar. * [regex](https://github.com/rust-lang/regex) optional regular expression capture groups processing for `-r`/`--regex` option. +* [rust-shlex](https://github.com/comex/rust-shlex) Using [`shlex::try_join`](https://docs.rs/shlex/latest/shlex/fn.try_join.html) for `--shell-quote` mode. * [tokio](https://tokio.rs/) asynchronous runtime for rust. From tokio this app uses: * `async` / `await` functions (aka coroutines) * Singleton `CommandLineArgs` instance using [`tokio::sync::OnceCell`](https://docs.rs/tokio/latest/tokio/sync/struct.OnceCell.html). From 1c714519de2eed8794f08b8cc838ba1d94ab20f0 Mon Sep 17 00:00:00 2001 From: Aaron Riekenberg Date: Sat, 15 Aug 2026 11:20:05 -0500 Subject: [PATCH 8/8] fix syntax --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0b2f8bd..59870ed 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Similar interface to [GNU Parallel](https://www.gnu.org/software/parallel/parall * Automatic parallelism to all cpus, or [configure manually](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#parallelism) * Transform inputs with [variables](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#automatic-variables) or [regular expressions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#regular-expression) * Prevent [output interleaving](https://github.com/aaronriekenberg/rust-parallel/wiki/Output-Interleaving) and maintain input order with `-k`/`--keep-order` -* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands), configurable (shell quoting)[https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-quote] +* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands), configurable [shell quoting](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-quote) * [TUI progress bar](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#progress-bar) using [indicatif](https://github.com/console-rs/indicatif) * [Path cache](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#path-cache) * [Command timeouts](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#timeout) @@ -75,7 +75,7 @@ The same `cargo install rust-parallel` command will also update to the latest ve * [itertools](https://docs.rs/itertools/latest/itertools/) using [`multi_cartesian_product`](https://docs.rs/itertools/latest/itertools/trait.Itertools.html#method.multi_cartesian_product) to process `:::` command line inputs. * [indicatif](https://github.com/console-rs/indicatif) optional TUI progress bar. * [regex](https://github.com/rust-lang/regex) optional regular expression capture groups processing for `-r`/`--regex` option. -* [rust-shlex](https://github.com/comex/rust-shlex) Using [`shlex::try_join`](https://docs.rs/shlex/latest/shlex/fn.try_join.html) for `--shell-quote` mode. +* [rust-shlex](https://github.com/comex/rust-shlex) using [`shlex::try_join`](https://docs.rs/shlex/latest/shlex/fn.try_join.html) for `--shell-quote` mode. * [tokio](https://tokio.rs/) asynchronous runtime for rust. From tokio this app uses: * `async` / `await` functions (aka coroutines) * Singleton `CommandLineArgs` instance using [`tokio::sync::OnceCell`](https://docs.rs/tokio/latest/tokio/sync/struct.OnceCell.html).