Backport HTTP 'Age' header field

    See #2032 / #2067, kudos @Wahnes. This is a backport for 3.0

    Currently, when "HTTP clock skew" is calculated, this is taken to be the
    difference between the timestamp of the moment the HTTPS request was sent and
    the date given in the HTTP Date header. This does not yield valid results in
    case a HTTP cache is used, either on the client side or on the server side.
    According to the HTTP specs, the Date field will contain the timestamp the
    response was created, which may not be the timestamp the response was
    delivered.

    Consider the following example that queries the Varnish project's web server.
    Note that Varnish is a popular HTTP caching server, so HTTP caching will of
    course be used when serving HTTP responses from this project's web server.
    testssl.sh https://varnish-cache.org/ This will typically output a HTTP clock
    skew of some thousand seconds.

    The patch takes into account the HTTP Age header that caching servers add to
    the HTTP response to signal the response's freshness. As client-side caches
    normally do not cache HTTPS requests (except maybe for "enterprise" HTTP proxy
    caches that do MITM HTTPS proxying), this is mostly targeted to HTTPS websites
    that employ server side HTTP caching.

    Addtional polishing:

    * address my comments in #2032
    * add JSON field HTTP_headerAge if they exists
    * output HTTP_AGE if it was detected
    * do stripping of line feeds closer to where variables were set

Author: drwetter

testssl.sh

@@ -401,6 +401,7 @@ TLS_NOW=""                              # Similar
 TLS_DIFFTIME_SET=false                  # Tells TLS functions to measure the TLS difftime or not
 NOW_TIME=""
 HTTP_TIME=""
+HTTP_AGE=""                             # Age Header, see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Age + RFC 7234
 GET_REQ11=""
 START_TIME=0                            # time in epoch when the action started
 END_TIME=0                              # .. ended
@@ -2160,13 +2161,15 @@ run_http_header() {
           printf "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$OPTIMAL_PROTO $BUGS -quiet -ign_eof -connect $NODEIP:$PORT $PROXY $SNI") >$HEADERFILE 2>$ERRFILE
           NOW_TIME=$(date "+%s")
           HTTP_TIME=$(awk -F': ' '/^date:/ { print $2 }  /^Date:/ { print $2 }' $HEADERFILE)
+          HTTP_AGE=$(awk -F': ' '/^[aA][gG][eE]: / { print $2 }' $HEADERFILE)
           HAD_SLEPT=0
      else
           # 1st GET request hung and needed to be killed. Check whether it succeeded anyway:
           if grep -Eiaq "XML|HTML|DOCTYPE|HTTP|Connection" $HEADERFILE; then
                # correct by seconds we slept, HAD_SLEPT comes from wait_kill()
                NOW_TIME=$(($(date "+%s") - HAD_SLEPT))
                HTTP_TIME=$(awk -F': ' '/^date:/ { print $2 }  /^Date:/ { print $2 }' $HEADERFILE)
+               HTTP_AGE=$(awk -F': ' '/^[aA][gG][eE]: / { print $2 }' $HEADERFILE)
           else
                prln_warning " likely HTTP header requests failed (#lines: $(wc -l $HEADERFILE | awk '{ print $1 }'))"
                [[ "$DEBUG" -lt 1 ]] && outln "Rerun with DEBUG>=1 and inspect $HEADERFILE\n"
@@ -2198,7 +2201,9 @@ run_http_header() {
      fi
 
      # Populate vars for HTTP time
-     debugme echo "NOW_TIME: $NOW_TIME | HTTP_TIME: $HTTP_TIME"
+     [[ -n "$HTTP_AGE" ]] && HTTP_AGE="$(strip_lf "$HTTP_AGE")"
+     [[ -n "$HTTP_TIME" ]] && HTTP_TIME="$(strip_lf "$HTTP_TIME")"
+     debugme echo "NOW_TIME: $NOW_TIME | HTTP_AGE: $HTTP_AGE | HTTP_TIME: $HTTP_TIME"
 
      # Quit on first empty line to catch 98% of the cases. Next pattern is there because the SEDs tested
      # so far seem not to be fine with header containing x0d x0a (CRLF) which is the usual case.
@@ -2324,19 +2329,26 @@ run_http_date() {
                out "${spaces}local:  $(LC_ALL=C TZ=GMT date "+%a, %d %b %Y %T %Z")"
                fileout "$jsonID" "INFO" "$HTTP_TIME - $(TZ=GMT date "+%a, %d %b %Y %T %Z")"
           else
+               # modifying the global from string to a number
                HTTP_TIME="$(parse_date "$HTTP_TIME" "+%s" "%a, %d %b %Y %T %Z" 2>>$ERRFILE)"
-               difftime=$((HTTP_TIME - NOW_TIME))
+               difftime=$((HTTP_TIME + HTTP_AGE - NOW_TIME))
                [[ $difftime != "-"* ]] && [[ $difftime != "0" ]] && difftime="+$difftime"
                # process was killed, so we need to add an error
                [[ $HAD_SLEPT -ne 0 ]] && difftime="$difftime (± 1.5)"
                out "$difftime sec from localtime";
                fileout "$jsonID" "INFO" "$difftime seconds from localtime"
           fi
+          if [[ -n "$HTTP_AGE" ]]; then
+               outln
+               pr_bold " HTTP Age, RFC 7234           "
+               out "$HTTP_AGE"
+               fileout "HTTP_headerAge" "INFO" "$HTTP_AGE seconds"
+          fi
      else
           out "Got no HTTP time, maybe try different URL?";
           fileout "$jsonID" "INFO" "Got no HTTP time, maybe try different URL?"
      fi
-     debugme tm_out ", HTTP_TIME in epoch: $HTTP_TIME"
+     debugme tm_out ", HTTP_TIME + HTTP_AGE in epoch: $HTTP_TIME / $HTTP_AGE"
      outln
      match_ipv4_httpheader "$1"
      return 0