crypto: optimize C906 AES and add measured SG2002 offload

.github/workflows/ci.yml

@@ -160,6 +160,18 @@ jobs:
         run: |
           sudo apt-get install -y gcc-riscv64-linux-gnu qemu-user
           python3 tests/test-openssl-source.py --negative-control
+      - name: OpenSSL AES partial DMA continuation
+        run: python3 firmware/crypto/experimental/openssl-aes/test-blocks.py
+      - name: OpenSSL AES EVP state and counter carry
+        run: |
+          cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined firmware/crypto/experimental/openssl-aes/verify.c -lcrypto -o /tmp/openssl-aes-verify
+          /tmp/openssl-aes-verify
+          cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined firmware/crypto/experimental/openssl-aes/verify-gcm.c -lcrypto -o /tmp/openssl-gcm-verify
+          /tmp/openssl-gcm-verify
+          cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined -Iplatform/modules/sg2002-aes firmware/crypto/experimental/openssl-aes/verify-ctr.c -lcrypto -o /tmp/openssl-ctr-verify
+          /tmp/openssl-ctr-verify
+          cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined firmware/crypto/experimental/openssl-aes/verify-tails.c -lcrypto -o /tmp/openssl-aes-tails
+          /tmp/openssl-aes-tails bfe84c43ff49df11
       - name: Independent add-on repository index
         run: python3 tests/test-index-apk-repo.py
       - name: ECDSA package signing and upgrade

firmware/crypto/experimental/openssl-aes/README.md

@@ -0,0 +1,130 @@
+# OpenSSL C906 AES qualification
+
+`0005-aes-c906.patch` separately changes the existing scalar AES table lookups
+to `th.extu` followed by a scaled indexed load. Byte lookups in the 32-bit Te
+tables retain scale 2; the Td4 byte S-box uses scale 0. Tables, schedules and
+interfaces stay unchanged. Plain RV64GC and FIPS builds produce byte-identical
+AES fallback machine code to upstream.
+
+`0005-aes-dma.patch` adds SG2002 offload for AES-128/192/256 ECB/CBC/CTR and the
+CTR primitive inside GCM. OpenSSL keeps padding, streaming, GHASH, tags and TLS.
+The FIPS module is excluded. Both patches can be reviewed and qualified
+independently; no cryptodev or ENGINE integration is added.
+
+## Native selection and performance
+
+The performance measurements below use OpenSSL 3.5.9, the restricted C906/O3
+profile and physical hardware, before the SHA-1/DES/TDES scope cleanup.
+`native-measurement-identities.json` pins those binaries, patches, flags and
+harnesses. They are historical measurements of the recorded revisions, not
+new measurements of the restacked build. Rates are MiB per process CPU-second.
+AES assembly, thresholds and provider algorithms are unchanged by the cleanup.
+
+Five rotated primitive trials measure AES-128 encryption/decryption at
+0.700/0.703 us per block, versus upstream 1.030/1.030 us. AES-192 and AES-256
+also use about 32–33% less CPU per block. Key setup is essentially unchanged.
+The earlier indexed-load implementation is additionally compared with the
+new extraction/scaled-load sequence in `native-lookup-trials.jsonl`.
+
+Three final rotated EVP trials give:
+
+| Operation | #504 baseline | Patched, DMA off | Patched, DMA on |
+| --- | ---: | ---: | ---: |
+| AES-128 ECB encrypt, 1 KiB | 13.198 | 16.982 | 17.276 |
+| AES-128 CBC encrypt, 1 KiB | 12.046 | 15.103 | 15.149 |
+| AES-128 CTR encrypt, 1 KiB | 11.131 | 13.705 | 13.574 |
+| AES-128 GCM open, 1400 B | 7.779 | 11.071 | 11.034 |
+| AES-128 GCM open, 16 KiB | 8.036 | 11.208 | 14.141 |
+| AES-256 GCM open, 16 KiB | 6.695 | 9.462 | 13.620 |
+
+These GCM columns use upstream software GHASH; the separate vector GHASH patch
+#527 is not included. A 16 KiB GCM record still reaches the CTR callback as five
+3072-byte DMA calls plus a 1024-byte software tail. The complete EVP result
+includes that transport cost. The experimental combined-record ioctl and
+persistent-descriptor design is not selected by this patch; its earlier
+service/wake-lateness measurements do not establish end-to-end SRTP latency.
+
+The full EVP benchmark reinitializes key/IV and finalizes each operation.
+Complete 16–128-byte calls can be slower than #504 despite the faster AES
+primitive. No unconditional tiny-message improvement is claimed. Raw trials
+include these regressions. Assembly, provider dispatch and whole-library
+placement effects must not be conflated.
+
+`bench-stream.c` measures repeated EVP updates with an existing key and context,
+with DMA disabled. Across all key sizes/directions, the final software path
+improves 16-byte updates by 33–38% for ECB, 17–27% for CBC and 21–30% for CTR.
+At 64 bytes the corresponding ranges are 43–46%, 31–41% and 31–37%. This is a
+streaming-update measurement, distinct from the repeated-key-init benchmark
+and from a complete TLS connection. Three rotated trials are in
+`native-stream-trials.jsonl`.
+
+## DMA boundaries and tails
+
+Initial minima are 3072 bytes for ECB/CBC and 2048 for CTR/GCM, with a shared
+header for both CTR users. Against the selected software path, the smallest
+final measured gains across all key sizes and both directions are 8.0%, 17.7%
+and 13.2% respectively. `native-threshold-trials.jsonl` retains the three rotated
+256–4096-byte sweep with DMA forced from 256 bytes, including the 1024-byte
+regressions. That low-threshold DSO is an experiment, not a shipped profile.
+
+After opening the descriptor, AES uses a separate 512-byte continuation
+minimum. A 6000-byte operation therefore submits 4096 + 1904 bytes; a 4112-byte
+operation submits 4096 and handles 16 bytes in software. In the experiment,
+sending every final block to DMA lost up to 3.7% at a 16-byte tail, while a
+512-byte tail improved all measured key/mode/direction combinations. The
+initial descriptor cost and the cost of another RUN are different, and neither
+is zero. The helper accepts AES only; other algorithms are rejected before
+opening the device. The legacy provider no longer links the DMA transport.
+
+Provider size checks now skip the DMA helper entirely below the initial
+minimum. Request initialization occurs only after successful validation and
+descriptor acquisition; the former path zeroed request storage even when
+returning immediately to software.
+
+## Correctness and memory
+
+The AES-only ASan/UBSan helper suite passes 92,880 continuation cases plus 960
+CBC cases at every block boundary from 4096+16 through 4096+2560 and 25 GCM
+counter wrap/fault cases. Seven non-AES IDs must leave input, output and IV
+unchanged without opening the device. Negative controls reject full-width
+hardware-counter assumptions, repeating the setup minimum on a tail, and
+accepting a non-AES algorithm.
+
+`verify.c` retains the 972 AES cases from the former shared DES verifier:
+AES-128/192/256 ECB/CBC/CTR, both directions, split updates, context copying,
+in-place operation, offsets and output guards. Its independent host OpenSSL
+3.5.5 ciphertext fingerprint is `019ff5988780130c`; CI compiles and runs it with
+ASan/UBSan alongside the CTR, GCM and tail verifiers. It needs only the default
+provider. A wrong-fingerprint negative control is also checked locally.
+
+The earlier native DMA-on/off checks each passed 1944 mixed-algorithm block
+cases (including those 972 AES cases), 288 CTR carry cases,
+408 AEAD known-answer/rejected-tag cases, 2160 extended GCM cases, 20,736 real
+EVP tail cases and upstream AES vectors (313 tests, one FIPS-only skip). Tail
+outputs and updated IVs match independent host OpenSSL 3.5.5 fingerprint
+`bfe84c43ff49df11`, across all key sizes, modes, directions, offsets and in-place
+operation. GCM retains its independent `a2d7088437b53472` fingerprint and tests
+context copy/free-original, split updates and rejected tags/ciphertext.
+
+`scope-cleanup-validation.json` records the current AES-only host sanitizer
+checks and clean C906 cross-build, with all 5808 public exports unchanged and
+no SHA-1 offload or SG2002 objects in the legacy provider. Native checks were
+not repeated for this cleanup; `native-final-validation.json` retains the
+exact earlier library and patch identities. No new performance claim is made.
+
+Only completed whole-block prefixes are committed. CBC in-place decryption
+preserves the last ciphertext, and IVs advance after success. Every CTR request
+splits before its low 32-bit word wraps, even for ordinary 128-bit CTR; software
+advances the external counter at the required width between requests. Failed
+requests resume software from the last committed prefix.
+
+Provider contexts remain 472 bytes for AES and 984 for AES-GCM, versus 448/960
+before DMA key retention. The scalar patch adds no persistent storage and the
+DMA helper adds no per-context output buffer. Production retains all 5808
+libcrypto dynamic definitions. Native files and raw measurements are published
+alongside the standalone benchmark and verification sources.
+
+Four real 16 MiB HTTPS range downloads with the recorded pre-cleanup library pass CA,
+status, length and independent SHA-256 checks: AES-128/256-GCM under TLS 1.2
+and TLS 1.3. These are qualification runs, not a repeated network-throughput
+comparison; see `native-final-https.jsonl`.

firmware/crypto/experimental/openssl-aes/bench-primitive.c

@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+#define _POSIX_C_SOURCE 200809L
+#define OPENSSL_SUPPRESS_DEPRECATED
+#include <openssl/aes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+static volatile unsigned sink;
+static uint64_t ns(void){struct timespec t;if(clock_gettime(CLOCK_PROCESS_CPUTIME_ID,&t))abort();return (uint64_t)t.tv_sec*1000000000+t.tv_nsec;}
+int main(void){unsigned char key[32],input[16],out[16];AES_KEY ek,dk,temp;for(unsigned i=0;i<32;i++)key[i]=(unsigned char)(i*7+3);for(unsigned i=0;i<16;i++)input[i]=(unsigned char)(i*13+21);
+ const char *names[]={"set-encrypt-key","set-decrypt-key","encrypt-block","decrypt-block"};
+ for(int bits=128;bits<=256;bits+=64){if(AES_set_encrypt_key(key,bits,&ek)||AES_set_decrypt_key(key,bits,&dk))abort();
+  for(unsigned mode=0;mode<4;mode++){uint64_t start=ns(),end,n=0;
+   do{for(unsigned i=0;i<256;i++){if(mode<2){int rc=mode==0?AES_set_encrypt_key(key,bits,&temp):AES_set_decrypt_key(key,bits,&temp);if(rc)abort();sink^=temp.rd_key[0];}else{if(mode==2)AES_encrypt(input,out,&ek);else AES_decrypt(input,out,&dk);sink^=out[0];}n++;}end=ns();}while(end-start<30000000);
+   printf("{\"bits\":%d,\"operation\":\"%s\",\"iterations\":%llu,\"cpu_ns_per_call\":%.3f}\n",bits,names[mode],(unsigned long long)n,(double)(end-start)/n);
+  }
+ }return 0;}

firmware/crypto/experimental/openssl-aes/bench-stream.c

@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+#define _POSIX_C_SOURCE 200809L
+#include <openssl/evp.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+static volatile unsigned sink;
+static void check(int ok){if(!ok){fputs("FAIL stream benchmark\n",stderr);exit(2);}}
+static uint64_t ns(void){struct timespec t;check(clock_gettime(CLOCK_PROCESS_CPUTIME_ID,&t)==0);return (uint64_t)t.tv_sec*1000000000+t.tv_nsec;}
+int main(int argc,char **argv){check(argc==2);EVP_CIPHER *cipher=EVP_CIPHER_fetch(NULL,argv[1],NULL);EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();check(cipher&&ctx);unsigned char key[32]={0},iv[16]={0},in[16400],out[16400];memset(in,0x37,sizeof in);const int sizes[]={16,64,128,256,1024,16384};
+ for(unsigned s=0;s<6;s++)for(int enc=0;enc<=1;enc++){int n;check(EVP_CipherInit_ex2(ctx,cipher,key,iv,enc,NULL)==1);check(EVP_CIPHER_CTX_set_padding(ctx,0)==1);uint64_t start=ns(),end,rounds=0;
+  do{for(unsigned i=0;i<16;i++){check(EVP_CipherUpdate(ctx,out+5,&n,in+5,sizes[s])==1&&n==sizes[s]);sink^=out[5];rounds++;}end=ns();}while(end-start<25000000);
+  check(EVP_CipherFinal_ex(ctx,out,&n)==1&&n==0);
+  printf("{\"algorithm\":\"%s\",\"encrypt\":%d,\"bytes\":%d,\"rounds\":%llu,\"cpu_ns\":%llu,\"mib_per_cpu_s\":%.6f}\n",argv[1],enc,sizes[s],(unsigned long long)rounds,(unsigned long long)(end-start),(double)sizes[s]*rounds*1e9/((end-start)*1048576.0));
+ }EVP_CIPHER_CTX_free(ctx);EVP_CIPHER_free(cipher);return 0;}

firmware/crypto/experimental/openssl-aes/bench.c

@@ -0,0 +1,29 @@
+#define _POSIX_C_SOURCE 200809L
+#include <openssl/evp.h>
+#include <openssl/provider.h>
+#include <time.h>
+#include <sys/resource.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+static void ck(int x){if(!x){fputs("FAIL EVP benchmark\n",stderr);exit(2);}}
+static uint64_t ns(clockid_t id){struct timespec t;ck(clock_gettime(id,&t)==0);return (uint64_t)t.tv_sec*1000000000+t.tv_nsec;}
+int main(int argc,char **argv){
+ ck(argc>=2);double duration=argc>2?strtod(argv[2],0):0.1;ck(duration>0&&duration<=2);
+ OSSL_PROVIDER *def=OSSL_PROVIDER_load(0,"default"),*legacy=OSSL_PROVIDER_load(0,"legacy");ck(def&&legacy);
+ EVP_CIPHER *cipher=EVP_CIPHER_fetch(0,argv[1],0);EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();ck(cipher&&ctx);
+ unsigned char key[32]={0},iv[16]={0},input[65568],output[65568];memset(input,0x37,sizeof input);
+ int sizes[]={16,32,64,128,256,512,768,1024,1280,1536,1792,2048,2304,2560,2816,3072,3584,4096,4112,4160,4352,4608,4864,5120,5376,5632,6000,6144,6400,6656,8192,16384};
+ for(unsigned i=0;i<sizeof sizes/sizeof sizes[0];i++)for(int enc=0;enc<=1;enc++){
+  uint64_t cpu=ns(CLOCK_PROCESS_CPUTIME_ID),wall=ns(CLOCK_MONOTONIC),rounds=0,end;int n,m;
+  do{for(int k=0;k<16;k++){
+   ck(EVP_CipherInit_ex2(ctx,cipher,key,iv,enc,0)==1);ck(EVP_CIPHER_CTX_set_padding(ctx,0)==1);
+   ck(EVP_CipherUpdate(ctx,output+5,&n,input+5,sizes[i])==1);ck(EVP_CipherFinal_ex(ctx,output+5+n,&m)==1&&n+m==sizes[i]);rounds++;
+  }end=ns(CLOCK_PROCESS_CPUTIME_ID);}while(end-cpu<(uint64_t)(duration*1e9));
+  wall=ns(CLOCK_MONOTONIC)-wall;cpu=end-cpu;
+  printf("{\"algorithm\":\"%s\",\"encrypt\":%d,\"bytes\":%d,\"rounds\":%llu,\"cpu_ns\":%llu,\"wall_ns\":%llu,\"mib_per_cpu_s\":%.6f}\n",argv[1],enc,sizes[i],(unsigned long long)rounds,(unsigned long long)cpu,(unsigned long long)wall,(double)sizes[i]*rounds*1e9/(cpu*1048576.0));
+ }
+ struct rusage usage;ck(getrusage(RUSAGE_SELF,&usage)==0);printf("{\"kind\":\"memory\",\"maxrss_kib\":%ld}\n",usage.ru_maxrss);
+ EVP_CIPHER_CTX_free(ctx);EVP_CIPHER_free(cipher);OSSL_PROVIDER_unload(legacy);OSSL_PROVIDER_unload(def);return 0;
+}