diff --git a/Cargo.lock b/Cargo.lock index cc3f2ba..b169210 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,6 +523,7 @@ dependencies = [ "num_cpus", "predicates", "regex", + "shlex", "thiserror", "tokio", "tracing", @@ -571,6 +572,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook-registry" version = "1.4.8" diff --git a/Cargo.toml b/Cargo.toml index e36602d..ac4bf51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ indicatif = "0.18" itertools = "0.15" num_cpus = "1" regex = "1" +shlex = "2" thiserror = "2" tokio = { version = "1", features = ["full"] } tracing = "0.1" diff --git a/README.md b/README.md index 8700b59..59870ed 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Similar interface to [GNU Parallel](https://www.gnu.org/software/parallel/parall * Automatic parallelism to all cpus, or [configure manually](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#parallelism) * Transform inputs with [variables](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#automatic-variables) or [regular expressions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#regular-expression) * Prevent [output interleaving](https://github.com/aaronriekenberg/rust-parallel/wiki/Output-Interleaving) and maintain input order with `-k`/`--keep-order` -* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands) +* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands), configurable [shell quoting](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-quote) * [TUI progress bar](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#progress-bar) using [indicatif](https://github.com/console-rs/indicatif) * [Path cache](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#path-cache) * [Command timeouts](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#timeout) @@ -75,6 +75,7 @@ The same `cargo install rust-parallel` command will also update to the latest ve * [itertools](https://docs.rs/itertools/latest/itertools/) using [`multi_cartesian_product`](https://docs.rs/itertools/latest/itertools/trait.Itertools.html#method.multi_cartesian_product) to process `:::` command line inputs. * [indicatif](https://github.com/console-rs/indicatif) optional TUI progress bar. * [regex](https://github.com/rust-lang/regex) optional regular expression capture groups processing for `-r`/`--regex` option. +* [rust-shlex](https://github.com/comex/rust-shlex) using [`shlex::try_join`](https://docs.rs/shlex/latest/shlex/fn.try_join.html) for `--shell-quote` mode. * [tokio](https://tokio.rs/) asynchronous runtime for rust. From tokio this app uses: * `async` / `await` functions (aka coroutines) * Singleton `CommandLineArgs` instance using [`tokio::sync::OnceCell`](https://docs.rs/tokio/latest/tokio/sync/struct.OnceCell.html). diff --git a/scripts/generate_manual.sh b/scripts/generate_manual.sh index 7e989ca..9eacb15 100755 --- a/scripts/generate_manual.sh +++ b/scripts/generate_manual.sh @@ -34,6 +34,7 @@ echo ' 1. [Demo of command line arguments](#demo-of-command-line-arguments) 1. [Demo of function and command line arguments from stdin](#demo-of-function-and-command-line-arguments-from-stdin) 1. [Demo of function and initial arguments on command line, additional arguments from stdin](#demo-of-function-and-initial-arguments-on-command-line-additional-arguments-from-stdin) +1. [Shell Quote](#shell-quote) ' echo '## Command line options' @@ -504,3 +505,66 @@ cat test | $RUST_PARALLEL -s logargs hello rm -f test echo '```' + +echo '## Shell Quote + +When using shell mode (`-s`), the `--shell-quote` option applies shell escaping to each argument before passing it to the shell. + +This is useful when arguments contain special shell characters like `$`, backticks, single quotes, or double quotes that should be treated as literal strings and not interpreted by the shell. + +Without `--shell-quote`, special characters can cause command injection or unexpected shell behavior. With `--shell-quote` enabled, arguments are properly escaped. + +Consider a bash function that uses special characters: +' + +echo '```' + +echo '$ demonstrate_shell_quote() { + for arg in "$@"; do + echo "arg: $arg" + done +}' + +demonstrate_shell_quote() { + for arg in "$@"; do + echo "arg: $arg" + done +} + +echo ' +$ export -f demonstrate_shell_quote' +export -f demonstrate_shell_quote + +echo '``` + +With `--shell-quote`, special characters in arguments are properly escaped: +' + +echo '```' +echo '$ cat >./test <<'"'"'EOL'"'"' +hello$world +foo`cmd` +bar'"'"'baz'"'"' +EOL' +cat >./test <<'EOL' +hello$world +foo`cmd` +bar'baz' +EOL + +echo ' +$ cat test | rust-parallel -s --shell-quote demonstrate_shell_quote' +cat test | $RUST_PARALLEL -s --shell-quote demonstrate_shell_quote +echo '```' + +echo 'Without `--shell-quote`, special characters are interpreted by the shell, causing errors: +' +echo '```' +echo '$ cat test | rust-parallel -s demonstrate_shell_quote' +set +e +cat test | $RUST_PARALLEL -s demonstrate_shell_quote 2>&1 | head -20 +set -e + +rm -f test + +echo '```' diff --git a/src/command_line_args.rs b/src/command_line_args.rs index 32158bd..d334222 100644 --- a/src/command_line_args.rs +++ b/src/command_line_args.rs @@ -52,6 +52,12 @@ pub struct CommandLineArgs { #[arg(short, long)] pub shell: bool, + /// Use shell quoting for command arguments. + /// + /// Each command line is passed to " " as a single argument with shell quoting applied to each argument. + #[arg(long)] + pub shell_quote: bool, + /// Timeout seconds for running commands. Defaults to infinite timeout if not specified. #[arg(short, long, value_parser = Self::parse_timeout_seconds)] pub timeout_seconds: Option, diff --git a/src/parser.rs b/src/parser.rs index b97efe0..c0c5cd0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -39,6 +39,7 @@ impl ParsedCommand { /// Applies shell wrapping (if configured) and converts to OwnedCommandAndArgs. pub struct CommandBuilder { shell_command_and_args: Option>, + shell_quote: bool, } impl CommandBuilder { @@ -53,6 +54,7 @@ impl CommandBuilder { }; Self { shell_command_and_args, + shell_quote: command_line_args.shell_quote, } } @@ -62,7 +64,14 @@ impl CommandBuilder { Some(shell_command_and_args) => { let mut result = Vec::with_capacity(shell_command_and_args.len() + 1); result.extend(shell_command_and_args.iter().cloned()); - result.push(parsed.command_and_args.join(" ")); + + if self.shell_quote { + result.push( + shlex::try_join(parsed.command_and_args.iter().map(|s| s.as_str())).ok()?, + ); + } else { + result.push(parsed.command_and_args.join(" ")); + } result } }; diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 494d5ab..02758e4 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -286,6 +286,28 @@ fn runs_shell_function_from_stdin_j1() { .stderr(predicate::str::is_empty()); } +#[test] +fn runs_shell_function_from_stdin_with_shell_quote_j1() { + // Test with null-separated stdin containing special characters: ' " and backticks + // When shell_quote is enabled, these special characters should be properly quoted + let stdin = "hello'world\0foo\"bar\0baz`cmd`"; + + rust_parallel() + .write_stdin(stdin) + .arg("-0") + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("--shell-path=./dummy_shell.sh") + .arg("shell_function") + .assert() + .success() + .stdout(predicate::eq( + "dummy_shell arg1=-c arg2=shell_function \"hello'world\"\ndummy_shell arg1=-c arg2=shell_function 'foo\"bar'\ndummy_shell arg1=-c arg2=shell_function 'baz`cmd`'\n", + )) + .stderr(predicate::str::is_empty()); +} + #[test] fn runs_shell_function_from_file_j1() { rust_parallel() @@ -733,3 +755,106 @@ E"#; .stdout(predicate::str::contains("1\n").count(5)) .stderr(predicate::str::is_empty()); } + +#[test] +fn test_shell_quote_with_spaces() { + // Test shell_quote with arguments containing spaces + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("echo") + .arg(":::") + .arg("hello world") + .arg("foo bar") + .assert() + .success() + .stdout( + (predicate::str::contains("hello world\n").count(1)) + .and(predicate::str::contains("foo bar\n").count(1)), + ) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_with_special_characters() { + // Test shell_quote with special shell characters like $, ", ', etc. + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("echo") + .arg(":::") + .arg("$VAR") + .arg("'single quotes'") + .arg(" 'single quote and spaces' ") + .arg(r#" "double quote and spaces" "#) + .assert() + .success() + .stdout( + (predicate::str::contains("$VAR\n").count(1)) + .and(predicate::str::contains("'single quotes'\n").count(1)) + .and(predicate::str::contains(" 'single quote and spaces' \n").count(1)) + .and( + predicate::str::contains(r#" "double quote and spaces" "#.to_owned() + "\n") + .count(1), + ), + ) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_with_placeholders() { + // Test shell_quote combined with placeholder substitution + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("echo") + .arg("arg={}") + .arg(":::") + .arg("hello world") + .arg("foo bar") + .assert() + .success() + .stdout( + (predicate::str::contains("arg=hello world\n").count(1)) + .and(predicate::str::contains("arg=foo bar\n").count(1)), + ) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_with_regex() { + // Test shell_quote combined with regex capture groups + rust_parallel() + .arg("-j1") + .arg("-s") + .arg("--shell-quote") + .arg("-r") + .arg("(.*):(.*)") + .arg("echo") + .arg("key={1}") + .arg("val={2}") + .arg(":::") + .arg("my key:my value") + .assert() + .success() + .stdout(predicate::str::contains("key=my key val=my value\n")) + .stderr(predicate::str::is_empty()); +} + +#[test] +fn test_shell_quote_without_shell_option() { + // shell_quote should have no effect when shell is not enabled + rust_parallel() + .arg("-j1") + .arg("--shell-quote") + .arg("echo") + .arg(":::") + .arg("hello world") + .assert() + .success() + .stdout(predicate::str::contains("hello world\n")) + .stderr(predicate::str::is_empty()); +}