Merge pull request #2761 from testssl/mac_runner

PoC github action runner macos-14

Author: drwetter

.github/workflows/unit_tests_macos.yml

@@ -0,0 +1,54 @@
+---
+name: CI run MacOS
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'utils/**'
+      - 'doc/**'
+      - 'bin/**'
+      - '**.md'
+      - '**.pem'
+      - '**.pdf'
+      - '**.html'
+      - 'LICENSE'
+      - 'Dockerfile'
+      - 'Dockerfile.alpine'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    name: PoC unit test on MacOS
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up system (perl, curl is installed already)
+        run: |
+          brew install socat jsonlint cpanminus
+          printf "%s\n" "----------"
+          mv -v /opt/homebrew/bin/openssl /opt/homebrew/bin/openssl.NOPE
+          printf "%s\n" "----------"
+          perl -V
+          printf "%s\n" "----------"
+          curl --version
+          printf "%s\n" "----------"
+          openssl version -a
+          printf "%s\n" "----------"
+          bash --version
+          printf "%s\n" "----------"
+
+      - name: Install perl modules
+        run: |
+          cpanm --notest Test::More
+          cpanm --notest Data::Dumper
+          cpanm --notest JSON
+          cpanm --notest Text::Diff
+
+      - name: run it
+        run: |
+          prove -v t

.github/workflows/unit_tests_ubuntu.yml

@@ -1,5 +1,5 @@
 ---
-name: testssl.sh CI with Ubuntu
+name: CI run Ubuntu
 
 on:
   pull_request:

Readme.md

@@ -2,12 +2,13 @@
 ## Intro
 ![GitHub Tag](https://img.shields.io/github/v/tag/testssl/testssl.sh)
 ![Static Badge](https://img.shields.io/badge/%2Fbin%2Fbash_-blue)
-![Static Badge](https://img.shields.io/badge/OpenSSL_-blue)
+![Static Badge](https://img.shields.io/badge/Libre+OpenSSL_-blue)
 [![License](https://img.shields.io/github/license/testssl/testssl.sh)](https://github.com/testssl/testssl.sh/LICENSE)
 ![GitHub Created At](https://img.shields.io/github/created-at/testssl/testssl.sh)
 ![GitHub last commit](https://img.shields.io/github/last-commit/testssl/testssl.sh)
 ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/testssl/testssl.sh)
-[![CI tests](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml/badge.svg)](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml)
+[![CI test Ubuntu](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml/badge.svg)](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml)
+[![CI test MacOS](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_macos.yml/badge.svg)](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_macos.yml)
 [![Docker](https://img.shields.io/docker/pulls/drwetter/testssl.sh)](https://github.com/testssl/testssl.sh/blob/3.2/Dockerfile.md)
 ![Mastodon Follow](https://img.shields.io/mastodon/follow/109319848143024146?domain=infosec.exchange)
 [![Bluesky](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpublic.api.bsky.app%2Fxrpc%2Fapp.bsky.actor.getProfile%2F%3Factor%3Dtestssl.bsky.social&query=%24.followersCount&style=social&logo=bluesky&label=Follow%20%40testssl.sh)
@@ -30,6 +31,7 @@ cryptographic flaws.
 * Privacy: It's only you who sees the result, not a third party.
 * Freedom: It's 100% open source. You can look at the code, see what's going on.
 * The development is free and open @ GitHub and participation is welcome.
+* Unit tests ensure maturity (output is consistent, JSON is valid, runs under Linux+MacOS etc)
 
 ### License
 
@@ -52,7 +54,7 @@ but you can also use any LibreSSL or OpenSSL version.
 (silent) check for binaries is done when you start testssl.sh . System V needs probably
 to have GNU grep installed. Windows (using MSYS2, Cygwin or WSL) work too.
 
-Update notification here or @ [mastodon](https://infosec.exchange/@testssl) or [bluesky](https://bsky.app/profile/testssl.bsky.social). [twitter](https://twitter.com/drwetter) is not being used anymore.
+Update notification can be found here or most important ones @ [mastodon](https://infosec.exchange/@testssl) or [bluesky](https://bsky.app/profile/testssl.bsky.social). [twitter](https://twitter.com/drwetter) is not being used anymore.
 
 ### Installation
 

etc/ca_hashes.txt

@@ -24,33 +24,42 @@ printf "\n%s\n", "Testing whether just calling \"./testssl.sh\" produces no erro
 my $info    = stat($prg);
 my $retMode = $info->mode;
 
+#1
 is($retMode & 0400, 0400, "Checking \"./testssl.sh\" for read permission");
 $tests++;
 
+#2
 is($retMode & 0100, 0100, "Checking \"./testssl.sh\" for execute permission");
 $tests++;
 
-$fileout = `timeout 10 bash $prg 2>&1`;
+$fileout = `bash $prg 2>&1`;
 my $retval=$?;
 
+#3
 unlike($fileout, qr/$error_regexp1/, "regex 1");
 $tests++;
 
+#4
 unlike($fileout, qr/$error_regexp2/, "regex 2");
 $tests++;
 
+#5
 unlike($fileout, qr/$error_regexp3/, "regex 3");
 $tests++;
 
+#6
 unlike($fileout, qr/$error_regexp4/, "regex 4");
 $tests++;
 
+#7
 unlike($fileout, qr/$error_regexp5/, "regex 5");
 $tests++;
 
+#8
 is($retval, 0, "return value should be equal zero: \"$retval\"");
 $tests++;
 
+#9
 $out=`grep -E "$debug_regexp" $prg`;
 unlike($out, qr/$debug_regexp/, "Debug RegEx");
 $tests++;

t/00_testssl_help.t

@@ -17,27 +17,34 @@ my $error_regexp5='(syntax error|unexpected token)';
 my $good_regexp='free software([\s\S]*)USAGE w/o ANY WARRANTY([\s\S]*)OWN RISK([\s\S]*)Using([\s\S]*)ciphers([\s\S]*)built([\s\S]*)platform';
 
 printf "\n%s\n", "Testing whether just calling \"./testssl.sh --banner\" produces no error ...";
-$fileout = `timeout 10 bash ./testssl.sh --banner 2>&1`;
+$fileout = `bash ./testssl.sh --banner 2>&1`;
 my $retval=$?;
 
+#1
 unlike($fileout, qr/$error_regexp1/, "regex 1");
 $tests++;
 
+#2
 unlike($fileout, qr/$error_regexp2/, "regex 2");
 $tests++;
 
+#3
 unlike($fileout, qr/$error_regexp3/, "regex 3");
 $tests++;
 
+#4
 unlike($fileout, qr/$error_regexp4/, "regex 4");
 $tests++;
 
+#5
 unlike($fileout, qr/$error_regexp5/, "regex 5");
 $tests++;
 
+#6
 like($fileout, qr/$good_regexp/, "regex positive");
 $tests++;
 
+#7
 is($retval, 0, "return value should be equal zero: \"$retval\"");
 $tests++;
 

t/01_testssl_banner.t

@@ -14,9 +14,12 @@ my $error_regexp2='client-simulation.txt:';
 
 printf "\n%s\n", "Testing whether \"~/etc/client-simulation.txt\" isn't broken ...";
 $fileout = `bash ./etc/client-simulation.txt 2>&1`;
+
+#1
 unlike($fileout, qr/$error_regexp1/, "regex 1");
 $tests++;
 
+#2
 unlike($fileout, qr/$error_regexp2/, "regex 2");
 $tests++;
 

t/02_clientsim_txt_parsable.t

@@ -3,10 +3,12 @@
 use strict;
 use Test::More;
 
-printf "\n%s\n", "Testing whether CA certificates are newer their SPKI hashes \"~/etc/ca_hashes.txt\" ...";
+printf "\n%s\n", "Make sure CA certificate stores are older than their SPKI hashes \"~/etc/ca_hashes.txt\" ...";
 
 my $newer_bundles=`find etc/*.pem -newer etc/ca_hashes.txt`;
-is($newer_bundles,"","Checking if there's an output with a *.pem file run \"~/utils/create_ca_hashes.sh\"");
+
+#1
+is($newer_bundles,"","Checking if there's an output with a *.pem file. If so: run \"~/utils/create_ca_hashes.sh\"");
 
 printf "\n";
 done_testing;

t/05_ca_hashes_up_to_date.t

@@ -22,7 +22,7 @@ my $socket_out="";
 my $openssl_out="";
 my $socket_json="";
 my $openssl_json="";
-#FIXME: Blacklists we use to trigger an error, but likely we can skip that and instead we should?/could use
+#FIXME: Pattern we use to trigger an error, but likely we can skip that and instead we should?/could use the following??
 #       @args="$prg $check2run $uri >/dev/null";
 #       system("@args") == 0
 #           or die ("FAILED: \"@args\" ");
@@ -38,23 +38,26 @@ unlink $tmp_json;
 
 # Title
 printf "\n%s\n", "Baseline unit test IPv4 against \"$uri\"";
-
-#1
 $socket_out = `$prg $check2run $uri 2>&1`;
 $socket_json = json($tmp_json);
+
+#1
 unlike($socket_out, qr/$socket_errors≈/, "via sockets, checking terminal output");
 $tests++;
+
+#2
 unlike($socket_json, qr/$json_errors/, "via sockets checking JSON output");
 $tests++;
 
 unlink $tmp_json;
 
-
-#2
+#3
 $openssl_out = `$prg --ssl-native $check2run $uri 2>&1`;
 $openssl_json = json($tmp_json);
 unlike($openssl_out, qr/$openssl_errors/, "via (builtin) OpenSSL, checking terminal output");
 $tests++;
+
+#4
 unlike($openssl_json, qr/$json_errors/, "via OpenSSL (builtin) checking JSON output");
 $tests++;
 

t/10_baseline_ipv4_http.t

@@ -35,66 +35,71 @@ die "Unable to open $prg" unless -f $prg;
 
 $uri="smtp-relay.gmail.com:587";
 
-# we will have client simulations later, so we don't need to run everything again:
+
+#1
 # unlink "tmp.json";
+# we will have client simulations later, so we don't need to run everything again:
 printf "\n%s\n", "STARTTLS SMTP unit test via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run_smtp -t smtp $uri 2>&1`;
 # $socket_json = json('tmp.json');
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#2
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS SMTP unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run_smtp -t smtp $uri 2>&1`;
 # $openssl_json = json('tmp.json');
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="pop.gmx.net:110";
 
+#3
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS POP3 unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t pop3 $uri 2>&1`;
 # $socket_json = json('tmp.json');
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#4
 printf "\n%s\n", "STARTTLS POP3 unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run -t pop3 $uri 2>&1`;
 # $openssl_json = json('tmp.json');
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="imap.gmx.net:143";
 
+#5
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS IMAP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t imap $uri 2>&1`;
 # $socket_json = json('tmp.json');
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#6
 printf "\n%s\n", "STARTTLS IMAP unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run -t imap $uri 2>&1`;
 # $openssl_json = json('tmp.json');
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="mail.tigertech.net:4190";
 
+#7
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS MANAGE(SIEVE) unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t sieve $uri 2>&1`;
 # $socket_json = json('tmp.json');
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="jabber.org:5222";
 
+#8
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS XMPP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t xmpp $uri 2>&1`;
@@ -120,6 +125,7 @@ $tests++;
 
 $uri="ldap.uni-rostock.de:21";
 
+#9
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS FTP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t ftp $uri 2>&1`;
@@ -143,12 +149,14 @@ $tests++;
 # https://ldapwiki.com/wiki/Public%20LDAP%20Servers
 $uri="db.debian.org:389";
 
+#10
 printf "\n%s\n", "STARTTLS LDAP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t ldap $uri 2>&1`;
 # $socket_json = json('tmp.json');
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#11
 printf "\n%s\n", "STARTTLS LDAP unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run -t ldap $uri 2>&1`;
 # $openssl_json = json('tmp.json');
@@ -172,12 +180,10 @@ $tests++;
 #unlike($openssl_out, qr/$openssl_regex_bl/, "");
 #$tests++;
 
-
 # IRC: missing
 # LTMP, mysql, postgres
 
 
-
 done_testing($tests);
 # unlink "tmp.json";