Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ indicatif = "0.18"
itertools = "0.15"
num_cpus = "1"
regex = "1"
shlex = "2"
thiserror = "2"
tokio = { version = "1", features = ["full"] }
tracing = "0.1"
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Similar interface to [GNU Parallel](https://www.gnu.org/software/parallel/parall
* Automatic parallelism to all cpus, or [configure manually](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#parallelism)
* Transform inputs with [variables](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#automatic-variables) or [regular expressions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#regular-expression)
* Prevent [output interleaving](https://github.com/aaronriekenberg/rust-parallel/wiki/Output-Interleaving) and maintain input order with `-k`/`--keep-order`
* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands)
* Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands), configurable [shell quoting](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-quote)
* [TUI progress bar](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#progress-bar) using [indicatif](https://github.com/console-rs/indicatif)
* [Path cache](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#path-cache)
* [Command timeouts](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#timeout)
Expand Down Expand Up @@ -75,6 +75,7 @@ The same `cargo install rust-parallel` command will also update to the latest ve
* [itertools](https://docs.rs/itertools/latest/itertools/) using [`multi_cartesian_product`](https://docs.rs/itertools/latest/itertools/trait.Itertools.html#method.multi_cartesian_product) to process `:::` command line inputs.
* [indicatif](https://github.com/console-rs/indicatif) optional TUI progress bar.
* [regex](https://github.com/rust-lang/regex) optional regular expression capture groups processing for `-r`/`--regex` option.
* [rust-shlex](https://github.com/comex/rust-shlex) using [`shlex::try_join`](https://docs.rs/shlex/latest/shlex/fn.try_join.html) for `--shell-quote` mode.
* [tokio](https://tokio.rs/) asynchronous runtime for rust. From tokio this app uses:
* `async` / `await` functions (aka coroutines)
* Singleton `CommandLineArgs` instance using [`tokio::sync::OnceCell`](https://docs.rs/tokio/latest/tokio/sync/struct.OnceCell.html).
Expand Down
64 changes: 64 additions & 0 deletions scripts/generate_manual.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ echo '
1. [Demo of command line arguments](#demo-of-command-line-arguments)
1. [Demo of function and command line arguments from stdin](#demo-of-function-and-command-line-arguments-from-stdin)
1. [Demo of function and initial arguments on command line, additional arguments from stdin](#demo-of-function-and-initial-arguments-on-command-line-additional-arguments-from-stdin)
1. [Shell Quote](#shell-quote)
'

echo '## Command line options'
Expand Down Expand Up @@ -504,3 +505,66 @@ cat test | $RUST_PARALLEL -s logargs hello
rm -f test

echo '```'

echo '## Shell Quote

When using shell mode (`-s`), the `--shell-quote` option applies shell escaping to each argument before passing it to the shell.

This is useful when arguments contain special shell characters like `$`, backticks, single quotes, or double quotes that should be treated as literal strings and not interpreted by the shell.

Without `--shell-quote`, special characters can cause command injection or unexpected shell behavior. With `--shell-quote` enabled, arguments are properly escaped.

Consider a bash function that uses special characters:
'

echo '```'

echo '$ demonstrate_shell_quote() {
for arg in "$@"; do
echo "arg: $arg"
done
}'

demonstrate_shell_quote() {
for arg in "$@"; do
echo "arg: $arg"
done
}

echo '
$ export -f demonstrate_shell_quote'
export -f demonstrate_shell_quote

echo '```

With `--shell-quote`, special characters in arguments are properly escaped:
'

echo '```'
echo '$ cat >./test <<'"'"'EOL'"'"'
hello$world
foo`cmd`
bar'"'"'baz'"'"'
EOL'
cat >./test <<'EOL'
hello$world
foo`cmd`
bar'baz'
EOL

echo '
$ cat test | rust-parallel -s --shell-quote demonstrate_shell_quote'
cat test | $RUST_PARALLEL -s --shell-quote demonstrate_shell_quote
echo '```'

echo 'Without `--shell-quote`, special characters are interpreted by the shell, causing errors:
'
echo '```'
echo '$ cat test | rust-parallel -s demonstrate_shell_quote'
set +e
cat test | $RUST_PARALLEL -s demonstrate_shell_quote 2>&1 | head -20
set -e

rm -f test

echo '```'
6 changes: 6 additions & 0 deletions src/command_line_args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ pub struct CommandLineArgs {
#[arg(short, long)]
pub shell: bool,

/// Use shell quoting for command arguments.
///
/// Each command line is passed to "<shell-path> <shell-argument>" as a single argument with shell quoting applied to each argument.
#[arg(long)]
pub shell_quote: bool,

/// Timeout seconds for running commands. Defaults to infinite timeout if not specified.
#[arg(short, long, value_parser = Self::parse_timeout_seconds)]
pub timeout_seconds: Option<f64>,
Expand Down
11 changes: 10 additions & 1 deletion src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ impl ParsedCommand {
/// Applies shell wrapping (if configured) and converts to OwnedCommandAndArgs.
pub struct CommandBuilder {
shell_command_and_args: Option<Vec<String>>,
shell_quote: bool,
}

impl CommandBuilder {
Expand All @@ -53,6 +54,7 @@ impl CommandBuilder {
};
Self {
shell_command_and_args,
shell_quote: command_line_args.shell_quote,
}
}

Expand All @@ -62,7 +64,14 @@ impl CommandBuilder {
Some(shell_command_and_args) => {
let mut result = Vec::with_capacity(shell_command_and_args.len() + 1);
result.extend(shell_command_and_args.iter().cloned());
result.push(parsed.command_and_args.join(" "));

if self.shell_quote {
result.push(
shlex::try_join(parsed.command_and_args.iter().map(|s| s.as_str())).ok()?,
);
} else {
result.push(parsed.command_and_args.join(" "));
}
result
}
};
Expand Down
125 changes: 125 additions & 0 deletions tests/integration_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,28 @@ fn runs_shell_function_from_stdin_j1() {
.stderr(predicate::str::is_empty());
}

#[test]
fn runs_shell_function_from_stdin_with_shell_quote_j1() {
// Test with null-separated stdin containing special characters: ' " and backticks
// When shell_quote is enabled, these special characters should be properly quoted
let stdin = "hello'world\0foo\"bar\0baz`cmd`";

rust_parallel()
.write_stdin(stdin)
.arg("-0")
.arg("-j1")
.arg("-s")
.arg("--shell-quote")
.arg("--shell-path=./dummy_shell.sh")
.arg("shell_function")
.assert()
.success()
.stdout(predicate::eq(
"dummy_shell arg1=-c arg2=shell_function \"hello'world\"\ndummy_shell arg1=-c arg2=shell_function 'foo\"bar'\ndummy_shell arg1=-c arg2=shell_function 'baz`cmd`'\n",
))
.stderr(predicate::str::is_empty());
}

#[test]
fn runs_shell_function_from_file_j1() {
rust_parallel()
Expand Down Expand Up @@ -733,3 +755,106 @@ E"#;
.stdout(predicate::str::contains("1\n").count(5))
.stderr(predicate::str::is_empty());
}

#[test]
fn test_shell_quote_with_spaces() {
// Test shell_quote with arguments containing spaces
rust_parallel()
.arg("-j1")
.arg("-s")
.arg("--shell-quote")
.arg("echo")
.arg(":::")
.arg("hello world")
.arg("foo bar")
.assert()
.success()
.stdout(
(predicate::str::contains("hello world\n").count(1))
.and(predicate::str::contains("foo bar\n").count(1)),
)
.stderr(predicate::str::is_empty());
}

#[test]
fn test_shell_quote_with_special_characters() {
// Test shell_quote with special shell characters like $, ", ', etc.
rust_parallel()
.arg("-j1")
.arg("-s")
.arg("--shell-quote")
.arg("echo")
.arg(":::")
.arg("$VAR")
.arg("'single quotes'")
.arg(" 'single quote and spaces' ")
.arg(r#" "double quote and spaces" "#)
.assert()
.success()
.stdout(
(predicate::str::contains("$VAR\n").count(1))
.and(predicate::str::contains("'single quotes'\n").count(1))
.and(predicate::str::contains(" 'single quote and spaces' \n").count(1))
.and(
predicate::str::contains(r#" "double quote and spaces" "#.to_owned() + "\n")
.count(1),
),
)
.stderr(predicate::str::is_empty());
}

#[test]
fn test_shell_quote_with_placeholders() {
// Test shell_quote combined with placeholder substitution
rust_parallel()
.arg("-j1")
.arg("-s")
.arg("--shell-quote")
.arg("echo")
.arg("arg={}")
.arg(":::")
.arg("hello world")
.arg("foo bar")
.assert()
.success()
.stdout(
(predicate::str::contains("arg=hello world\n").count(1))
.and(predicate::str::contains("arg=foo bar\n").count(1)),
)
.stderr(predicate::str::is_empty());
}

#[test]
fn test_shell_quote_with_regex() {
// Test shell_quote combined with regex capture groups
rust_parallel()
.arg("-j1")
.arg("-s")
.arg("--shell-quote")
.arg("-r")
.arg("(.*):(.*)")
.arg("echo")
.arg("key={1}")
.arg("val={2}")
.arg(":::")
.arg("my key:my value")
.assert()
.success()
.stdout(predicate::str::contains("key=my key val=my value\n"))
.stderr(predicate::str::is_empty());
}

#[test]
fn test_shell_quote_without_shell_option() {
// shell_quote should have no effect when shell is not enabled
rust_parallel()
.arg("-j1")
.arg("--shell-quote")
.arg("echo")
.arg(":::")
.arg("hello world")
.assert()
.success()
.stdout(predicate::str::contains("hello world\n"))
.stderr(predicate::str::is_empty());
}