diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bf09505..b2b11a7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,18 @@ jobs: run: | sudo apt-get install -y gcc-riscv64-linux-gnu qemu-user python3 tests/test-openssl-source.py --negative-control + - name: OpenSSL AES partial DMA continuation + run: python3 firmware/crypto/experimental/openssl-aes/test-blocks.py + - name: OpenSSL AES EVP state and counter carry + run: | + cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined firmware/crypto/experimental/openssl-aes/verify.c -lcrypto -o /tmp/openssl-aes-verify + /tmp/openssl-aes-verify + cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined firmware/crypto/experimental/openssl-aes/verify-gcm.c -lcrypto -o /tmp/openssl-gcm-verify + /tmp/openssl-gcm-verify + cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined -Iplatform/modules/sg2002-aes firmware/crypto/experimental/openssl-aes/verify-ctr.c -lcrypto -o /tmp/openssl-ctr-verify + /tmp/openssl-ctr-verify + cc -O1 -g -Wall -Wextra -Werror -fsanitize=address,undefined firmware/crypto/experimental/openssl-aes/verify-tails.c -lcrypto -o /tmp/openssl-aes-tails + /tmp/openssl-aes-tails bfe84c43ff49df11 - name: Independent add-on repository index run: python3 tests/test-index-apk-repo.py - name: ECDSA package signing and upgrade diff --git a/firmware/crypto/experimental/openssl-aes/README.md b/firmware/crypto/experimental/openssl-aes/README.md new file mode 100644 index 00000000..7cbc8a6a --- /dev/null +++ b/firmware/crypto/experimental/openssl-aes/README.md @@ -0,0 +1,130 @@ +# OpenSSL C906 AES qualification + +`0005-aes-c906.patch` separately changes the existing scalar AES table lookups +to `th.extu` followed by a scaled indexed load. Byte lookups in the 32-bit Te +tables retain scale 2; the Td4 byte S-box uses scale 0. Tables, schedules and +interfaces stay unchanged. Plain RV64GC and FIPS builds produce byte-identical +AES fallback machine code to upstream. + +`0005-aes-dma.patch` adds SG2002 offload for AES-128/192/256 ECB/CBC/CTR and the +CTR primitive inside GCM. OpenSSL keeps padding, streaming, GHASH, tags and TLS. +The FIPS module is excluded. Both patches can be reviewed and qualified +independently; no cryptodev or ENGINE integration is added. + +## Native selection and performance + +The performance measurements below use OpenSSL 3.5.9, the restricted C906/O3 +profile and physical hardware, before the SHA-1/DES/TDES scope cleanup. +`native-measurement-identities.json` pins those binaries, patches, flags and +harnesses. They are historical measurements of the recorded revisions, not +new measurements of the restacked build. Rates are MiB per process CPU-second. +AES assembly, thresholds and provider algorithms are unchanged by the cleanup. + +Five rotated primitive trials measure AES-128 encryption/decryption at +0.700/0.703 us per block, versus upstream 1.030/1.030 us. AES-192 and AES-256 +also use about 32–33% less CPU per block. Key setup is essentially unchanged. +The earlier indexed-load implementation is additionally compared with the +new extraction/scaled-load sequence in `native-lookup-trials.jsonl`. + +Three final rotated EVP trials give: + +| Operation | #504 baseline | Patched, DMA off | Patched, DMA on | +| --- | ---: | ---: | ---: | +| AES-128 ECB encrypt, 1 KiB | 13.198 | 16.982 | 17.276 | +| AES-128 CBC encrypt, 1 KiB | 12.046 | 15.103 | 15.149 | +| AES-128 CTR encrypt, 1 KiB | 11.131 | 13.705 | 13.574 | +| AES-128 GCM open, 1400 B | 7.779 | 11.071 | 11.034 | +| AES-128 GCM open, 16 KiB | 8.036 | 11.208 | 14.141 | +| AES-256 GCM open, 16 KiB | 6.695 | 9.462 | 13.620 | + +These GCM columns use upstream software GHASH; the separate vector GHASH patch +#527 is not included. A 16 KiB GCM record still reaches the CTR callback as five +3072-byte DMA calls plus a 1024-byte software tail. The complete EVP result +includes that transport cost. The experimental combined-record ioctl and +persistent-descriptor design is not selected by this patch; its earlier +service/wake-lateness measurements do not establish end-to-end SRTP latency. + +The full EVP benchmark reinitializes key/IV and finalizes each operation. +Complete 16–128-byte calls can be slower than #504 despite the faster AES +primitive. No unconditional tiny-message improvement is claimed. Raw trials +include these regressions. Assembly, provider dispatch and whole-library +placement effects must not be conflated. + +`bench-stream.c` measures repeated EVP updates with an existing key and context, +with DMA disabled. Across all key sizes/directions, the final software path +improves 16-byte updates by 33–38% for ECB, 17–27% for CBC and 21–30% for CTR. +At 64 bytes the corresponding ranges are 43–46%, 31–41% and 31–37%. This is a +streaming-update measurement, distinct from the repeated-key-init benchmark +and from a complete TLS connection. Three rotated trials are in +`native-stream-trials.jsonl`. + +## DMA boundaries and tails + +Initial minima are 3072 bytes for ECB/CBC and 2048 for CTR/GCM, with a shared +header for both CTR users. Against the selected software path, the smallest +final measured gains across all key sizes and both directions are 8.0%, 17.7% +and 13.2% respectively. `native-threshold-trials.jsonl` retains the three rotated +256–4096-byte sweep with DMA forced from 256 bytes, including the 1024-byte +regressions. That low-threshold DSO is an experiment, not a shipped profile. + +After opening the descriptor, AES uses a separate 512-byte continuation +minimum. A 6000-byte operation therefore submits 4096 + 1904 bytes; a 4112-byte +operation submits 4096 and handles 16 bytes in software. In the experiment, +sending every final block to DMA lost up to 3.7% at a 16-byte tail, while a +512-byte tail improved all measured key/mode/direction combinations. The +initial descriptor cost and the cost of another RUN are different, and neither +is zero. The helper accepts AES only; other algorithms are rejected before +opening the device. The legacy provider no longer links the DMA transport. + +Provider size checks now skip the DMA helper entirely below the initial +minimum. Request initialization occurs only after successful validation and +descriptor acquisition; the former path zeroed request storage even when +returning immediately to software. + +## Correctness and memory + +The AES-only ASan/UBSan helper suite passes 92,880 continuation cases plus 960 +CBC cases at every block boundary from 4096+16 through 4096+2560 and 25 GCM +counter wrap/fault cases. Seven non-AES IDs must leave input, output and IV +unchanged without opening the device. Negative controls reject full-width +hardware-counter assumptions, repeating the setup minimum on a tail, and +accepting a non-AES algorithm. + +`verify.c` retains the 972 AES cases from the former shared DES verifier: +AES-128/192/256 ECB/CBC/CTR, both directions, split updates, context copying, +in-place operation, offsets and output guards. Its independent host OpenSSL +3.5.5 ciphertext fingerprint is `019ff5988780130c`; CI compiles and runs it with +ASan/UBSan alongside the CTR, GCM and tail verifiers. It needs only the default +provider. A wrong-fingerprint negative control is also checked locally. + +The earlier native DMA-on/off checks each passed 1944 mixed-algorithm block +cases (including those 972 AES cases), 288 CTR carry cases, +408 AEAD known-answer/rejected-tag cases, 2160 extended GCM cases, 20,736 real +EVP tail cases and upstream AES vectors (313 tests, one FIPS-only skip). Tail +outputs and updated IVs match independent host OpenSSL 3.5.5 fingerprint +`bfe84c43ff49df11`, across all key sizes, modes, directions, offsets and in-place +operation. GCM retains its independent `a2d7088437b53472` fingerprint and tests +context copy/free-original, split updates and rejected tags/ciphertext. + +`scope-cleanup-validation.json` records the current AES-only host sanitizer +checks and clean C906 cross-build, with all 5808 public exports unchanged and +no SHA-1 offload or SG2002 objects in the legacy provider. Native checks were +not repeated for this cleanup; `native-final-validation.json` retains the +exact earlier library and patch identities. No new performance claim is made. + +Only completed whole-block prefixes are committed. CBC in-place decryption +preserves the last ciphertext, and IVs advance after success. Every CTR request +splits before its low 32-bit word wraps, even for ordinary 128-bit CTR; software +advances the external counter at the required width between requests. Failed +requests resume software from the last committed prefix. + +Provider contexts remain 472 bytes for AES and 984 for AES-GCM, versus 448/960 +before DMA key retention. The scalar patch adds no persistent storage and the +DMA helper adds no per-context output buffer. Production retains all 5808 +libcrypto dynamic definitions. Native files and raw measurements are published +alongside the standalone benchmark and verification sources. + +Four real 16 MiB HTTPS range downloads with the recorded pre-cleanup library pass CA, +status, length and independent SHA-256 checks: AES-128/256-GCM under TLS 1.2 +and TLS 1.3. These are qualification runs, not a repeated network-throughput +comparison; see `native-final-https.jsonl`. diff --git a/firmware/crypto/experimental/openssl-aes/bench-primitive.c b/firmware/crypto/experimental/openssl-aes/bench-primitive.c new file mode 100644 index 00000000..0d1021cb --- /dev/null +++ b/firmware/crypto/experimental/openssl-aes/bench-primitive.c @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +#define _POSIX_C_SOURCE 200809L +#define OPENSSL_SUPPRESS_DEPRECATED +#include +#include +#include +#include +#include +static volatile unsigned sink; +static uint64_t ns(void){struct timespec t;if(clock_gettime(CLOCK_PROCESS_CPUTIME_ID,&t))abort();return (uint64_t)t.tv_sec*1000000000+t.tv_nsec;} +int main(void){unsigned char key[32],input[16],out[16];AES_KEY ek,dk,temp;for(unsigned i=0;i<32;i++)key[i]=(unsigned char)(i*7+3);for(unsigned i=0;i<16;i++)input[i]=(unsigned char)(i*13+21); + const char *names[]={"set-encrypt-key","set-decrypt-key","encrypt-block","decrypt-block"}; + for(int bits=128;bits<=256;bits+=64){if(AES_set_encrypt_key(key,bits,&ek)||AES_set_decrypt_key(key,bits,&dk))abort(); + for(unsigned mode=0;mode<4;mode++){uint64_t start=ns(),end,n=0; + do{for(unsigned i=0;i<256;i++){if(mode<2){int rc=mode==0?AES_set_encrypt_key(key,bits,&temp):AES_set_decrypt_key(key,bits,&temp);if(rc)abort();sink^=temp.rd_key[0];}else{if(mode==2)AES_encrypt(input,out,&ek);else AES_decrypt(input,out,&dk);sink^=out[0];}n++;}end=ns();}while(end-start<30000000); + printf("{\"bits\":%d,\"operation\":\"%s\",\"iterations\":%llu,\"cpu_ns_per_call\":%.3f}\n",bits,names[mode],(unsigned long long)n,(double)(end-start)/n); + } + }return 0;} diff --git a/firmware/crypto/experimental/openssl-aes/bench-stream.c b/firmware/crypto/experimental/openssl-aes/bench-stream.c new file mode 100644 index 00000000..9fe1ec5a --- /dev/null +++ b/firmware/crypto/experimental/openssl-aes/bench-stream.c @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include +static volatile unsigned sink; +static void check(int ok){if(!ok){fputs("FAIL stream benchmark\n",stderr);exit(2);}} +static uint64_t ns(void){struct timespec t;check(clock_gettime(CLOCK_PROCESS_CPUTIME_ID,&t)==0);return (uint64_t)t.tv_sec*1000000000+t.tv_nsec;} +int main(int argc,char **argv){check(argc==2);EVP_CIPHER *cipher=EVP_CIPHER_fetch(NULL,argv[1],NULL);EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();check(cipher&&ctx);unsigned char key[32]={0},iv[16]={0},in[16400],out[16400];memset(in,0x37,sizeof in);const int sizes[]={16,64,128,256,1024,16384}; + for(unsigned s=0;s<6;s++)for(int enc=0;enc<=1;enc++){int n;check(EVP_CipherInit_ex2(ctx,cipher,key,iv,enc,NULL)==1);check(EVP_CIPHER_CTX_set_padding(ctx,0)==1);uint64_t start=ns(),end,rounds=0; + do{for(unsigned i=0;i<16;i++){check(EVP_CipherUpdate(ctx,out+5,&n,in+5,sizes[s])==1&&n==sizes[s]);sink^=out[5];rounds++;}end=ns();}while(end-start<25000000); + check(EVP_CipherFinal_ex(ctx,out,&n)==1&&n==0); + printf("{\"algorithm\":\"%s\",\"encrypt\":%d,\"bytes\":%d,\"rounds\":%llu,\"cpu_ns\":%llu,\"mib_per_cpu_s\":%.6f}\n",argv[1],enc,sizes[s],(unsigned long long)rounds,(unsigned long long)(end-start),(double)sizes[s]*rounds*1e9/((end-start)*1048576.0)); + }EVP_CIPHER_CTX_free(ctx);EVP_CIPHER_free(cipher);return 0;} diff --git a/firmware/crypto/experimental/openssl-aes/bench.c b/firmware/crypto/experimental/openssl-aes/bench.c new file mode 100644 index 00000000..8ec8c873 --- /dev/null +++ b/firmware/crypto/experimental/openssl-aes/bench.c @@ -0,0 +1,29 @@ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include +#include +#include +static void ck(int x){if(!x){fputs("FAIL EVP benchmark\n",stderr);exit(2);}} +static uint64_t ns(clockid_t id){struct timespec t;ck(clock_gettime(id,&t)==0);return (uint64_t)t.tv_sec*1000000000+t.tv_nsec;} +int main(int argc,char **argv){ + ck(argc>=2);double duration=argc>2?strtod(argv[2],0):0.1;ck(duration>0&&duration<=2); + OSSL_PROVIDER *def=OSSL_PROVIDER_load(0,"default"),*legacy=OSSL_PROVIDER_load(0,"legacy");ck(def&&legacy); + EVP_CIPHER *cipher=EVP_CIPHER_fetch(0,argv[1],0);EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();ck(cipher&&ctx); + unsigned char key[32]={0},iv[16]={0},input[65568],output[65568];memset(input,0x37,sizeof input); + int sizes[]={16,32,64,128,256,512,768,1024,1280,1536,1792,2048,2304,2560,2816,3072,3584,4096,4112,4160,4352,4608,4864,5120,5376,5632,6000,6144,6400,6656,8192,16384}; + for(unsigned i=0;i +#include +#include +#include +#include +#include +#include "sg2002_cipher.c" +struct cipher_case {const char *name;unsigned alg,mode,keylen,block;}; +static const struct cipher_case cases[]={ + {"AES-128-ECB",1,0,16,16},{"AES-128-CBC",1,1,16,16},{"AES-128-CTR",1,2,16,16}, + {"AES-192-ECB",1,0,24,16},{"AES-192-CBC",1,1,24,16},{"AES-192-CTR",1,2,24,16}, + {"AES-256-ECB",1,0,32,16},{"AES-256-CBC",1,1,32,16},{"AES-256-CTR",1,2,32,16}}; +static const struct cipher_case *current; +static EVP_CIPHER *cipher; +static unsigned char key[32]; +static int fail_after,calls,opens,closes; +static void software(int enc,const unsigned char *in,unsigned char *out,size_t len,unsigned char *iv) +{ + EVP_CIPHER_CTX *c=EVP_CIPHER_CTX_new();int n,k;assert(c); + assert(EVP_CipherInit_ex2(c,cipher,key,iv,enc,NULL));assert(EVP_CIPHER_CTX_set_padding(c,0)); + assert(EVP_CipherUpdate(c,out,&n,in,(int)len));assert(EVP_CipherFinal_ex(c,out+n,&k));assert((size_t)(n+k)==len); + if(current->mode!=0) { assert(EVP_CIPHER_CTX_get_updated_iv(c,iv,current->block)); } + EVP_CIPHER_CTX_free(c); +} +int ossl_sg2002_begin(unsigned alg){assert(alg==current->alg);opens++;return fail_after==-2?-1:37;} +void ossl_sg2002_end(int fd){assert(fd==37);closes++;} +int ossl_sg2002_run(int fd,struct sg2002_crypto_request *r) +{ + assert(fd==37&&r->algorithm==current->alg&&r->mode==current->mode); + assert(r->key_length==current->keylen&&!memcmp(r->key,key,current->keylen)); + /* Model a device which only advances the low 32 counter bits. A request + * must not rely on wider hardware carry, even for ordinary 128-bit CTR. */ + if(r->mode==SG2002_MODE_CTR){ + unsigned b=current->block; + uint32_t low=(uint32_t)r->iv[b-4]<<24|(uint32_t)r->iv[b-3]<<16|(uint32_t)r->iv[b-2]<<8|r->iv[b-1]; + assert((uint64_t)low+r->length/b <= (UINT64_C(1)<<32)); + } + if(calls++==fail_after)return 0; + unsigned char iv[16];memcpy(iv,r->iv,16); + software(r->encrypt,(const unsigned char*)(uintptr_t)r->input,(unsigned char*)(uintptr_t)r->output,r->length,iv); + return 1; +} +static void ctr32_reference(const unsigned char *in,unsigned char *out,size_t len,unsigned char iv[16]) +{ + EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();assert(ctx); + assert(EVP_EncryptInit_ex2(ctx,EVP_aes_128_ecb(),key,NULL,NULL));assert(EVP_CIPHER_CTX_set_padding(ctx,0)); + uint32_t counter=(uint32_t)iv[12]<<24|(uint32_t)iv[13]<<16|(uint32_t)iv[14]<<8|iv[15]; + for(size_t p=0;p>(24-8*k)); + assert(EVP_EncryptUpdate(ctx,stream,&n,c,16)&&n==16); + for(unsigned k=0;k<16;k++)out[p+k]=in[p+k]^stream[k]; + counter++; + } + for(unsigned k=0;k<4;k++)iv[12+k]=(unsigned char)(counter>>(24-8*k)); + EVP_CIPHER_CTX_free(ctx); +} +static void check_gcm_counter(void) +{ + unsigned char input[16384],want[16384],buf[16384],iv[16],next[16],expected[16]; + memset(input,0x4b,sizeof input);current=&cases[2];cipher=EVP_CIPHER_fetch(NULL,current->name,NULL);assert(cipher); + for(size_t len=1024;len<=16384;len*=2)for(fail_after=-2;fail_after<=2;fail_after++){ + memset(iv,0xff,16);iv[0]=0xa5;iv[15]=0xfe;memcpy(expected,iv,16); + ctr32_reference(input,want,len,expected);memcpy(next,iv,16);memcpy(buf,input,len);calls=opens=closes=0; + size_t n=ossl_sg2002_cipher(1,2,1,key,16,next,4,buf,buf,len,1024,512); + ctr32_reference(buf+n,buf+n,len-n,next); + assert(!memcmp(buf,want,len)&&!memcmp(next,expected,16)); + } + EVP_CIPHER_free(cipher); + puts("PASS 25 GCM CTR32 rollover/fault cases: nonce prefix never carries"); +} + +static void check_tail_policy(void) +{ + unsigned char input[6656],want[6656],buf[6656],iv[16],next[16],expected_iv[16]; + current=&cases[1];cipher=EVP_CIPHER_fetch(NULL,current->name,NULL);assert(cipher); + memset(input,0x49,sizeof input); + for(size_t tail=16;tail<=2560;tail+=16)for(int enc=0;enc<=1;enc++)for(fail_after=-1;fail_after<=1;fail_after++){ + size_t len=4096+tail;memset(iv,0,16);memcpy(expected_iv,iv,16); + software(enc,input,want,len,expected_iv);memcpy(buf,input,len);memcpy(next,iv,16);calls=opens=closes=0; + size_t n=ossl_sg2002_cipher(1,1,enc,key,16,next,16,buf,buf,len,3072,512); + size_t expected=fail_after==0?0:tail<512||fail_after==1?4096:len; + assert(n == expected); + assert(calls == (fail_after==0||tail<512?1:2)); + assert(opens==1&&closes==1); + software(enc,buf+n,buf+n,len-n,next); + assert(!memcmp(buf,want,len)&&!memcmp(next,expected_iv,16)); + } + EVP_CIPHER_free(cipher); + puts("PASS 960 CBC tail-policy/failure cases at every block from 4096+16 to 4096+2560"); +} + +static void check_non_aes_rejected(void) +{ + const unsigned int algorithms[]={0,SG2002_ALG_DES,SG2002_ALG_TDES,SG2002_ALG_SM4,SG2002_ALG_SHA1,SG2002_ALG_SHA256,UINT32_MAX}; + unsigned char input[4096],out[4096],iv[16]; + current=&cases[0];fail_after=-1; + for(size_t i=0;iname,NULL);assert(cipher); + for(unsigned tail=0;tail<3;tail++)for(size_t mi=0;miblock:tail==1?512:minimum,len=sizes[z];int off=offsets[o];if(current->mode!=2&&len%current->block)continue; + memset(iv,0xff,16);iv[0]=0xa5;iv[15]=0xfe; /* full CTR carry across low-word boundary */ + memcpy(input,original,sizeof input);memcpy(next,iv,16); + if(!enc)software(1,original+off,input+off,len,next); + memcpy(expected_iv,iv,16);software(enc,input+off,want,len,expected_iv); + memset(buf,0xa7,sizeof buf);memcpy(buf+off,input+off,len);memcpy(next,iv,16);calls=opens=closes=0; + size_t n=ossl_sg2002_cipher(current->alg,current->mode,enc,key,current->keylen,next,current->block,buf+off,buf+off,len,minimum,continuation); + assert(n<=len&&n%current->block==0); + if(lenmode)assert(!memcmp(next,expected_iv,current->block)); + for(int k=0;klength/b' in bad.stderr, bad.stderr + print('PASS negative control: relying on a full-width hardware counter is rejected') + broken=original.replace('(done ? continuation_minimum : minimum)', 'minimum') + assert broken != original + source.write_text(broken) + subprocess.run(['cc','-O1','-g','-Wall','-Wextra','-Werror', + '-fsanitize=address,undefined','-I'+str(d/'crypto'),'-I'+str(d/'include'), + str(Path(__file__).with_suffix('.c')),'-lcrypto','-o',str(binary)],check=True) + bad=subprocess.run([str(binary)],capture_output=True) + assert bad.returncode != 0 and b'n == expected' in bad.stderr, bad.stderr + print('PASS negative control: repeating the setup minimum on a tail is rejected') + broken=original.replace('algorithm != SG2002_ALG_AES', + 'algorithm < SG2002_ALG_AES || algorithm > SG2002_ALG_SM4') + assert broken != original + source.write_text(broken) + subprocess.run(['cc','-O1','-g','-Wall','-Wextra','-Werror', + '-fsanitize=address,undefined','-I'+str(d/'crypto'),'-I'+str(d/'include'), + str(Path(__file__).with_suffix('.c')),'-lcrypto','-o',str(binary)],check=True) + bad=subprocess.run([str(binary)],capture_output=True) + assert bad.returncode != 0 and b'alg==current->alg' in bad.stderr, bad.stderr + print('PASS negative control: accepting a non-AES algorithm is rejected') diff --git a/firmware/crypto/experimental/openssl-aes/verify-ctr.c b/firmware/crypto/experimental/openssl-aes/verify-ctr.c new file mode 100644 index 00000000..4dddb3c0 --- /dev/null +++ b/firmware/crypto/experimental/openssl-aes/verify-ctr.c @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include "sg2002_crypto.h" +#endif +static void check(int ok, const char *what) +{ + if (!ok) { fprintf(stderr,"FAIL %s\n",what); ERR_print_errors_fp(stderr); exit(1); } +} +static void reference(const EVP_CIPHER *ecb, const unsigned char *key, + const unsigned char initial[16], const unsigned char *in, unsigned char *out, + size_t len, unsigned int width) +{ + EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new(); + unsigned char counter[16], stream[16]; int n; + check(ctx!=NULL,"reference context"); memcpy(counter,initial,16); + check(EVP_EncryptInit_ex2(ctx,ecb,key,NULL,NULL),"reference key"); + check(EVP_CIPHER_CTX_set_padding(ctx,0),"reference padding"); + for (size_t pos=0;pos16-width;i--) if(++counter[i-1]) break; + } + EVP_CIPHER_CTX_free(ctx); +} +static void provider(const EVP_CIPHER *ctr, const unsigned char *key, + const unsigned char *iv,const unsigned char *in,unsigned char *out, + size_t len,int split,int enc) +{ + EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();int n,total=0,last; + const size_t steps[]={1,2047,3073,4095};size_t pos=0,step=0; + check(ctx&&EVP_CipherInit_ex2(ctx,ctr,key,iv,enc,NULL),"provider init"); + while(poslen-pos)count=len-pos; + check(EVP_CipherUpdate(ctx,out+total,&n,in+pos,(int)count),"provider update"); + total+=n;pos+=count; + } + check(EVP_CipherFinal_ex(ctx,out+total,&last)&&total+last==(int)len,"provider final"); + EVP_CIPHER_CTX_free(ctx); +} +int main(int argc,char **argv) +{ + static const size_t lengths[]={64,2048,4096,8193}; + unsigned char key[32],iv[16],in[8224],want[8224],got[8224],c32[4096],c64[4096]; + unsigned int cases=0;int raw_only=argc==2&&!strcmp(argv[1],"--raw-only"); + int native=raw_only||(argc==2&&!strcmp(argv[1],"--native")); + check(argc==1||native,"argument"); + for(unsigned i=0;i=0,"native device"); +#else + check(!native,"native requires Linux"); +#endif + for(unsigned int bits=128;bits<=256;bits+=64) { + char name[32];snprintf(name,sizeof name,"AES-%u-ECB",bits); + EVP_CIPHER *ecb=EVP_CIPHER_fetch(NULL,name,NULL); + snprintf(name,sizeof name,"AES-%u-CTR",bits); + EVP_CIPHER *ctr=EVP_CIPHER_fetch(NULL,name,NULL);check(ecb&&ctr,"cipher fetch"); + for(unsigned int carry=0;carry<3;carry++) { + for(unsigned i=0;i<16;i++)iv[i]=(unsigned char)(0x13+i); + memset(iv+(carry==0?12:carry==1?8:0),0xff,carry==0?4:carry==1?8:16); + iv[15]=0xfd; /* 32-bit carry, 64-bit carry, then testmgr's ff..fd. */ +#ifdef __linux__ + if(native) { + struct sg2002_crypto_request req={.version=SG2002_CRYPTO_ABI, + .algorithm=SG2002_ALG_AES,.mode=SG2002_MODE_CTR,.encrypt=1, + .key_length=bits/8,.length=4096,.output_length=4096, + .input=(uintptr_t)in,.output=(uintptr_t)got}; + memcpy(req.key,key,bits/8);memcpy(req.iv,iv,16); + reference(ecb,key,iv,in,want,4096,16); + reference(ecb,key,iv,in,c64,4096,8); + reference(ecb,key,iv,in,c32,4096,4); + check(ioctl(fd,SG2002_CRYPTO_RUN,&req)==0&&req.status==1&&req.output_length==4096,"raw DMA"); + int full=!memcmp(got,want,4096),low64=!memcmp(got,c64,4096),low32=!memcmp(got,c32,4096); + check(full||low64||low32,"unrecognized hardware counter behavior"); + printf("RAW AES-%u carry=%u matches128=%d matches64=%d matches32=%d\n",bits,carry,full,low64,low32); + fflush(stdout); + } +#endif + if(raw_only)continue; + for(size_t l=0;l=0)close(fd); +#endif + printf("PASS %u EVP CTR carry cases\n",cases); + return 0; +} diff --git a/firmware/crypto/experimental/openssl-aes/verify-gcm.c b/firmware/crypto/experimental/openssl-aes/verify-gcm.c new file mode 100644 index 00000000..3127362a --- /dev/null +++ b/firmware/crypto/experimental/openssl-aes/verify-gcm.c @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +#define main baseline_benchmark_main +#include "../openssl-c906/evp-bench.c" +#undef main +/* Test provider state, not just the standalone DMA helper. A copied context + * must remain valid after freeing and cleansing the original key schedule. */ +static int copy_stream(const EVP_CIPHER *cipher,const unsigned char *key, + const unsigned char *iv,int ivlen,const unsigned char *aad,int aadlen, + const unsigned char *input,unsigned char *out,int len,int cut,int enc, + unsigned char tag[16]) +{ + EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new(),*copy=EVP_CIPHER_CTX_new();int n,total,last; + ck(ctx&©,"GCM copy contexts"); + ck(EVP_CipherInit_ex2(ctx,cipher,NULL,NULL,enc,NULL),"GCM init"); + ck(EVP_CIPHER_CTX_ctrl(ctx,EVP_CTRL_AEAD_SET_IVLEN,ivlen,NULL),"GCM IV length"); + ck(EVP_CipherInit_ex2(ctx,NULL,key,iv,enc,NULL),"GCM key/IV"); + if(!enc)ck(EVP_CIPHER_CTX_ctrl(ctx,EVP_CTRL_AEAD_SET_TAG,16,tag),"GCM expected tag"); + ck(EVP_CipherUpdate(ctx,NULL,&n,aad,7),"GCM first AAD"); + ck(EVP_CipherUpdate(ctx,NULL,&n,aad+7,aadlen-7),"GCM remaining AAD"); + if(cut>len)cut=len; + ck(EVP_CipherUpdate(ctx,out,&n,input,cut)&&n==cut,"GCM first input");total=n; + ck(EVP_CIPHER_CTX_copy(copy,ctx),"GCM context copy"); + EVP_CIPHER_CTX_free(ctx); + ck(EVP_CipherUpdate(copy,out+total,&n,input+cut,len-cut),"GCM copied update");total+=n; + int ok=EVP_CipherFinal_ex(copy,out+total,&last); + if(ok) { + ck(total+last==len,"GCM output length"); + if(enc)ck(EVP_CIPHER_CTX_ctrl(copy,EVP_CTRL_AEAD_GET_TAG,16,tag),"GCM copied tag"); + } + EVP_CIPHER_CTX_free(copy);return ok; +} +static void seed(const EVP_CIPHER *cipher,const unsigned char *key, + const unsigned char *iv,int ivlen,const unsigned char *aad,int aadlen, + const unsigned char *in,unsigned char *out,int len,unsigned char tag[16]) +{ + EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();int n,last; + ck(ctx&&EVP_EncryptInit_ex2(ctx,cipher,NULL,NULL,NULL),"seed init"); + ck(EVP_CIPHER_CTX_ctrl(ctx,EVP_CTRL_AEAD_SET_IVLEN,ivlen,NULL),"seed IV length"); + ck(EVP_EncryptInit_ex2(ctx,NULL,key,iv,NULL),"seed key"); + ck(EVP_EncryptUpdate(ctx,NULL,&n,aad,aadlen),"seed AAD"); + ck(EVP_EncryptUpdate(ctx,out,&n,in,len)&&n==len,"seed input"); + ck(EVP_EncryptFinal_ex(ctx,out+n,&last)&&last==0,"seed final"); + ck(EVP_CIPHER_CTX_ctrl(ctx,EVP_CTRL_AEAD_GET_TAG,16,tag),"seed tag"); + EVP_CIPHER_CTX_free(ctx); +} +int main(void) +{ + static const int lengths[]={1025,2049,3073,4097,16384,16385}; + static const int cuts[]={1,1024,2048,3072,4096}; + unsigned char key[32],iv[16],aad[23],plain[16432],expected[16432],out[16432],tag[16],actual_tag[16]; + uint64_t fp=UINT64_C(1469598103934665603);unsigned cases=0; + correctness(); /* Independent published known-answer vectors and bad tags. */ + for(unsigned i=0;i +#include +#include +#include +#include +static void check(int ok,const char *message){if(!ok){fprintf(stderr,"FAIL: %s\n",message);exit(2);}} +static uint64_t fingerprint=UINT64_C(14695981039346656037); +static void add(const unsigned char *p,size_t n){while(n--){fingerprint^=*p++;fingerprint*=UINT64_C(1099511628211);}} +int main(int argc,char **argv) +{ + unsigned char key[32],iv[16],next[16],input[7200],output[7200];unsigned cases=0; + const char *modes[]={"ECB","CBC","CTR"};const int offsets[]={0,5,15}; + for(unsigned i=0;i<32;i++)key[i]=(unsigned char)(i*7+3); + for(unsigned i=0;i<16;i++)iv[i]=(unsigned char)(i*11+3); + for(unsigned k=128;k<=256;k+=64)for(unsigned mode=0;mode<3;mode++){ + char name[24];snprintf(name,sizeof name,"AES-%u-%s",k,modes[mode]); + EVP_CIPHER *cipher=EVP_CIPHER_fetch(NULL,name,NULL);EVP_CIPHER_CTX *ctx=EVP_CIPHER_CTX_new();check(cipher&&ctx,"fetch/context"); + for(size_t tail=16;tail<=3072;tail+=16)for(int enc=0;enc<=1;enc++)for(unsigned o=0;o<3;o++)for(int inplace=0;inplace<=1;inplace++){ + size_t len=4096+tail;int off=offsets[o],n,m; + for(size_t i=0;i +#include +#include +#include +#include +#include +#include +#define LIMIT 65536 +static void ck(int ok, const char *what) +{ + if (!ok) { fprintf(stderr, "FAIL %s\n", what); ERR_print_errors_fp(stderr); exit(2); } +} +static void init(EVP_CIPHER_CTX *c, EVP_CIPHER *cipher, const unsigned char *key, + const unsigned char *iv, int encrypt) +{ + ck(EVP_CipherInit_ex2(c, cipher, key, iv, encrypt, NULL) == 1, "init"); + ck(EVP_CIPHER_CTX_set_padding(c, 0) == 1, "padding"); +} +static void whole(EVP_CIPHER_CTX *c, EVP_CIPHER *cipher, const unsigned char *key, + const unsigned char *iv, int enc, unsigned char *out, + const unsigned char *in, int len) +{ + int n, m; + init(c, cipher, key, iv, enc); + ck(EVP_CipherUpdate(c, out, &n, in, len) == 1, "whole update"); + ck(EVP_CipherFinal_ex(c, out+n, &m) == 1 && n+m == len, "whole final"); +} +static void streamed(EVP_CIPHER_CTX *c, EVP_CIPHER_CTX *dup, EVP_CIPHER *cipher, + const unsigned char *key, const unsigned char *iv, int enc, + unsigned char *out, unsigned char *copy, + const unsigned char *in, int len) +{ + int first = len ? 1 : 0, n, m, total, copied, pos; + init(c, cipher, key, iv, enc); + ck(EVP_CipherUpdate(c, out, &n, in, first) == 1, "stream prefix"); + ck(EVP_CIPHER_CTX_copy(dup, c) == 1, "context copy"); + memcpy(copy, out, (size_t)n); total = copied = n; pos = first; + while (pos < len) { + int step = pos % 3 ? 4097 : 31; + if (step > len-pos) step = len-pos; + ck(EVP_CipherUpdate(c, out+total, &n, in+pos, step) == 1, "stream update"); + ck(EVP_CipherUpdate(dup, copy+copied, &m, in+pos, step) == 1, "copied update"); + total += n; copied += m; pos += step; + } + ck(EVP_CipherFinal_ex(c, out+total, &n) == 1, "stream final"); + ck(EVP_CipherFinal_ex(dup, copy+copied, &m) == 1, "copied final"); + ck(total+n == len && copied+m == len, "stream lengths"); + ck(!memcmp(out, copy, (size_t)len), "copied context mismatch"); +} +int main(void) +{ + static const char *names[] = { + "AES-128-ECB", "AES-128-CBC", "AES-128-CTR", + "AES-192-ECB", "AES-192-CBC", "AES-192-CTR", + "AES-256-ECB", "AES-256-CBC", "AES-256-CTR" + }; + static const int sizes[] = {0,1,7,8,15,16,17,255,256,1023,1024,1025,4095,4096,4097,8192,16384,65536}; + static const int offsets[] = {0,5,15}; + unsigned char key[32], iv[16], data[LIMIT+32], answer[LIMIT+32]; + unsigned char actual[LIMIT+64], copied[LIMIT+64], inplace[LIMIT+64]; + EVP_CIPHER_CTX *c=EVP_CIPHER_CTX_new(), *dup=EVP_CIPHER_CTX_new(); + OSSL_PROVIDER *def=OSSL_PROVIDER_load(NULL,"default"); + uint64_t fp=UINT64_C(1469598103934665603); unsigned cases=0; + ck(c && dup && def, "providers/contexts"); + for (size_t i=0; i[3],$Ts->[3],0x3FC + + # Index into table. ++#if defined(__riscv_xtheadmemidx) && !defined(FIPS_MODULE) ++ th.lrwu $Ds->[0],$TBL,$Ts->[0],0 ++ th.lrwu $Ds->[1],$TBL,$Ts->[1],0 ++ th.lrwu $Ds->[2],$TBL,$Ts->[2],0 ++ th.lrwu $Ds->[3],$TBL,$Ts->[3],0 ++#else + add $I0,$TBL,$Ts->[0] + lwu $Ds->[0],0($I0) + add $I0,$TBL,$Ts->[1] +@@ -160,10 +166,19 @@ + lwu $Ds->[2],0($I0) + add $I0,$TBL,$Ts->[3] + lwu $Ds->[3],0($I0) +- +-___ +- +- return $ret; ++#endif ++ ++___ ++ ++ my $fast = ''; ++ my $hi = $shamt + 7; ++ for my $i (0..3) { ++ $fast .= " th.extu ".$Ts->[$i].",".$Qs->[$i].",$hi,$shamt\n"; ++ } ++ for my $i (0..3) { ++ $fast .= " th.lrwu ".$Ds->[$i].",$TBL,".$Ts->[$i].",2\n"; ++ } ++ return "#if defined(__riscv_xtheadbb) && defined(__riscv_xtheadmemidx) && !defined(FIPS_MODULE)\n".$fast."#else\n".$ret."#endif\n"; + } + + # Identical to do_lookup(), but loads only a single byte into each destination +@@ -171,6 +186,7 @@ + sub do_lookup_byte { + my $ret = do_lookup(@_); + $ret =~ s/lwu/lbu/g; ++ $ret =~ s/th.lrwu/th.lrbu/g; + return $ret; + } + +@@ -198,6 +214,12 @@ + andi $Ts->[2],$Ts->[2],0xFF + andi $Ts->[3],$Ts->[3],0xFF + ++#if defined(__riscv_xtheadmemidx) && !defined(FIPS_MODULE) ++ th.lrbu $Ds->[0],$TBL,$Ts->[0],0 ++ th.lrbu $Ds->[1],$TBL,$Ts->[1],0 ++ th.lrbu $Ds->[2],$TBL,$Ts->[2],0 ++ th.lrbu $Ds->[3],$TBL,$Ts->[3],0 ++#else + add $I0,$TBL,$Ts->[0] + lbu $Ds->[0],0($I0) + add $I0,$TBL,$Ts->[1] +@@ -206,10 +228,20 @@ + lbu $Ds->[2],0($I0) + add $I0,$TBL,$Ts->[3] + lbu $Ds->[3],0($I0) +- +-___ +- +- return $ret; ++#endif ++ ++___ ++ ++ my $fast = ''; ++ for my $i (0..3) { ++ my $lo = $i * 8; ++ my $hi = $lo + 7; ++ $fast .= " th.extu ".$Ts->[$i].",".$Qs->[$i].",$hi,$lo\n"; ++ } ++ for my $i (0..3) { ++ $fast .= " th.lrbu ".$Ds->[$i].",$TBL,".$Ts->[$i].",0\n"; ++ } ++ return "#if defined(__riscv_xtheadbb) && defined(__riscv_xtheadmemidx) && !defined(FIPS_MODULE)\n".$fast."#else\n".$ret."#endif\n"; + } + + ################################################################################ +@@ -813,7 +845,7 @@ + sd $T0,0($KEYP) + sd $T2,8($KEYP) + +- # if bits == 128 ++ // if bits == 128 + # jump into loop + beq $BITS,$T8,1f + +@@ -821,7 +853,7 @@ + srli $T5,$T4,32 + sd $T4,16($KEYP) + +- # if bits == 192 ++ // if bits == 192 + # jump into loop + beq $BITS,$T9,2f + +--- a/crypto/aes/build.info ++++ b/crypto/aes/build.info +@@ -52,7 +52,7 @@ + # aes-c64xplus.s implements AES_ctr32_encrypt + $AESDEF_c64xplus=AES_ASM AES_CTR_ASM + +- $AESASM_riscv64=aes_cbc.c aes-riscv64.s aes-riscv64-zkn.s aes-riscv64-zvkb-zvkned.s aes-riscv64-zvkned.s aes-riscv64-zvbb-zvkg-zvkned.s ++ $AESASM_riscv64=aes_cbc.c aes-riscv64.S aes-riscv64-zkn.s aes-riscv64-zvkb-zvkned.s aes-riscv64-zvkned.s aes-riscv64-zvbb-zvkg-zvkned.s + $AESDEF_riscv64=AES_ASM + $AESASM_riscv32=aes_core.c aes_cbc.c aes-riscv32-zkn.s + +@@ -126,7 +126,7 @@ + GENERATE[aes-mips.S]=asm/aes-mips.pl + INCLUDE[aes-mips.o]=.. + +-GENERATE[aes-riscv64.s]=asm/aes-riscv64.pl ++GENERATE[aes-riscv64.S]=asm/aes-riscv64.pl + GENERATE[aes-riscv64-zkn.s]=asm/aes-riscv64-zkn.pl + GENERATE[aes-riscv32-zkn.s]=asm/aes-riscv32-zkn.pl + GENERATE[aes-riscv64-zvkb-zvkned.s]=asm/aes-riscv64-zvkb-zvkned.pl diff --git a/firmware/crypto/openssl-c906/0005-aes-dma.patch b/firmware/crypto/openssl-c906/0005-aes-dma.patch new file mode 100644 index 00000000..b7df4cf4 --- /dev/null +++ b/firmware/crypto/openssl-c906/0005-aes-dma.patch @@ -0,0 +1,282 @@ +--- /dev/null ++++ b/crypto/sg2002_cipher.c +@@ -0,0 +1,80 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++#include ++#include ++#include ++#include "crypto/sg2002.h" ++ ++/* Advance just the chosen suffix. GCM uses 32 bits; ordinary CTR uses 128. */ ++static void increment(unsigned char *iv, unsigned int bytes, unsigned int blocks) ++{ ++ for (unsigned int i = bytes; i != 0 && blocks != 0; --i) { ++ blocks += iv[i-1]; ++ iv[i-1] = (unsigned char)blocks; ++ blocks >>= 8; ++ } ++} ++ ++size_t ossl_sg2002_cipher(unsigned int algorithm, unsigned int mode, int enc, ++ const unsigned char *key, size_t keylen, unsigned char *iv, ++ unsigned int counter_bytes, const unsigned char *in, unsigned char *out, ++ size_t len, size_t minimum, size_t continuation_minimum) ++{ ++ struct sg2002_crypto_request r; ++ unsigned char saved_iv[16]; ++ const unsigned int block = 16; ++ size_t done = 0; ++ int fd; ++ ++ if (len < minimum || minimum == 0 || minimum > SG2002_CRYPTO_MAX ++ || continuation_minimum < block || continuation_minimum > minimum ++ || algorithm != SG2002_ALG_AES ++ || mode > SG2002_MODE_CTR || keylen > sizeof(r.key) ++ || (mode != SG2002_MODE_ECB && iv == NULL) ++ || (mode == SG2002_MODE_CTR && counter_bytes != 4 && counter_bytes != block)) ++ return 0; ++ fd = ossl_sg2002_begin(algorithm); ++ if (fd < 0) ++ return 0; ++ memset(&r, 0, sizeof(r)); ++ r.version = SG2002_CRYPTO_ABI; ++ r.algorithm = algorithm; ++ r.mode = mode; ++ r.encrypt = mode == SG2002_MODE_CTR ? 1 : enc != 0; ++ r.key_length = (unsigned int)keylen; ++ memcpy(r.key, key, keylen); ++ /* Only the first submission pays descriptor acquisition. A later RUN ++ * still has a fixed cost, so the caller supplies a separate tail floor. */ ++ while (len - done >= (done ? continuation_minimum : minimum)) { ++ size_t chunk = len - done; ++ if (chunk > SG2002_CRYPTO_MAX) ++ chunk = SG2002_CRYPTO_MAX; ++ chunk -= chunk % block; ++ if (mode == SG2002_MODE_CTR) { ++ uint32_t counter = (uint32_t)iv[block-4] << 24 ++ | (uint32_t)iv[block-3] << 16 | (uint32_t)iv[block-2] << 8 | iv[block-1]; ++ uint64_t remaining = (UINT64_C(1) << 32) - counter; ++ if (chunk / block > remaining) ++ chunk = (size_t)remaining * block; ++ } ++ if (chunk == 0) ++ break; ++ r.length = r.output_length = (unsigned int)chunk; ++ r.input = (uintptr_t)(in + done); ++ r.output = (uintptr_t)(out + done); ++ if (mode != SG2002_MODE_ECB) ++ memcpy(r.iv, iv, block); ++ if (mode == SG2002_MODE_CBC && !enc) ++ memcpy(saved_iv, in + done + chunk - block, block); ++ if (!ossl_sg2002_run(fd, &r)) ++ break; ++ if (mode == SG2002_MODE_CBC) ++ memcpy(iv, enc ? out + done + chunk - block : saved_iv, block); ++ else if (mode == SG2002_MODE_CTR) ++ increment(iv + block - counter_bytes, counter_bytes, (unsigned int)(chunk / block)); ++ done += chunk; ++ } ++ ossl_sg2002_end(fd); ++ OPENSSL_cleanse(&r, sizeof(r)); ++ OPENSSL_cleanse(saved_iv, sizeof(saved_iv)); ++ return done; ++} +--- /dev/null ++++ b/providers/implementations/ciphers/cipher_aes_sg2002.inc +@@ -0,0 +1,47 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++#include "crypto/sg2002.h" ++#include "cipher_aes_sg2002.h" ++static size_t aes_sg2002_prefix(PROV_CIPHER_CTX *ctx, unsigned int mode, ++ unsigned char *out, const unsigned char *in, size_t len) ++{ ++ PROV_AES_CTX *a = (PROV_AES_CTX *)ctx; ++ size_t minimum = mode == SG2002_MODE_CTR ++ ? SG2002_AES_CTR_MIN : SG2002_AES_BLOCK_MIN; ++ if (len < minimum) ++ return 0; ++ return ossl_sg2002_cipher(SG2002_ALG_AES, mode, ctx->enc, ++ a->plat.sg2002.key, ctx->keylen, ctx->iv, 16, in, out, len, ++ minimum, SG2002_AES_CONTINUE_MIN); ++} ++static int cipher_sg2002_aes_cbc(PROV_CIPHER_CTX *ctx, unsigned char *out, ++ const unsigned char *in, size_t len) ++{ ++ size_t n = aes_sg2002_prefix(ctx, SG2002_MODE_CBC, out, in, len); ++ return n == len || ossl_cipher_hw_generic_cbc(ctx, out + n, in + n, len - n); ++} ++static int cipher_sg2002_aes_ecb(PROV_CIPHER_CTX *ctx, unsigned char *out, ++ const unsigned char *in, size_t len) ++{ ++ size_t n = aes_sg2002_prefix(ctx, SG2002_MODE_ECB, out, in, len); ++ return n == len || ossl_cipher_hw_generic_ecb(ctx, out + n, in + n, len - n); ++} ++static int cipher_sg2002_aes_ctr(PROV_CIPHER_CTX *ctx, unsigned char *out, ++ const unsigned char *in, size_t len) ++{ ++ size_t n; ++ if (len == 0) return 1; ++ if (ctx->num >= 16) return ossl_cipher_hw_generic_ctr(ctx, out, in, len); ++ if (ctx->num != 0) { ++ n = 16 - ctx->num; ++ if (n > len) n = len; ++ if (!ossl_cipher_hw_generic_ctr(ctx, out, in, n)) return 0; ++ out += n; in += n; len -= n; ++ } ++ n = aes_sg2002_prefix(ctx, SG2002_MODE_CTR, out, in, len); ++ return n == len || ossl_cipher_hw_generic_ctr(ctx, out + n, in + n, len - n); ++} ++#define cipher_sg2002_aes_ofb128 ossl_cipher_hw_generic_ofb128 ++#define cipher_sg2002_aes_cfb128 ossl_cipher_hw_generic_cfb128 ++#define cipher_sg2002_aes_cfb1 ossl_cipher_hw_generic_cfb1 ++#define cipher_sg2002_aes_cfb8 ossl_cipher_hw_generic_cfb8 ++#define AES_GENERIC_CIPHER(mode) cipher_sg2002_aes_##mode +--- /dev/null ++++ b/providers/implementations/ciphers/cipher_aes_gcm_sg2002.inc +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++#include ++#include ++#include "crypto/sg2002.h" ++#include "cipher_aes_sg2002.h" ++static void aes_sg2002_gcm_ctr(const unsigned char *in, unsigned char *out, ++ size_t blocks, const void *key, const unsigned char ivec[16]) ++{ ++ const PROV_AES_GCM_CTX *a = (const PROV_AES_GCM_CTX *) ++ ((const unsigned char *)key - offsetof(PROV_AES_GCM_CTX, ks.ks)); ++ unsigned char iv[16], stream[16]; ++ size_t n = 0; ++ memcpy(iv, ivec, sizeof(iv)); ++ if (blocks >= SG2002_AES_CTR_MIN / 16) ++ n = ossl_sg2002_cipher(SG2002_ALG_AES, SG2002_MODE_CTR, 1, ++ a->plat.sg2002.key, a->base.keylen, iv, 4, in, out, blocks * 16, ++ SG2002_AES_CTR_MIN, SG2002_AES_CONTINUE_MIN); ++ in += n; out += n; blocks -= n / 16; ++ while (blocks-- != 0) { ++ AES_encrypt(iv, stream, key); ++ for (unsigned int j = 0; j < 16; ++j) out[j] = in[j] ^ stream[j]; ++ for (unsigned int j = 16; j > 12; --j) if (++iv[j-1]) break; ++ in += 16; out += 16; ++ } ++ OPENSSL_cleanse(stream, sizeof(stream)); ++ OPENSSL_cleanse(iv, sizeof(iv)); ++} +--- a/include/crypto/sg2002.h ++++ b/include/crypto/sg2002.h +@@ -1,9 +1,18 @@ + /* SPDX-License-Identifier: Apache-2.0 */ + #ifndef OSSL_NANOKVM_SG2002_TRANSPORT_H + #define OSSL_NANOKVM_SG2002_TRANSPORT_H ++#include + #include "crypto/sg2002_crypto.h" + /* A call owns its fd. No shared request, key, buffer or userspace mutex. */ + int ossl_sg2002_begin(unsigned int algorithm); + int ossl_sg2002_run(int fd, struct sg2002_crypto_request *request); + void ossl_sg2002_end(int fd); ++/* AES only: returns a whole-block prefix; advances IV only on success. ++ * minimum includes session setup; continuation_minimum includes only RUN. ++ * The caller resumes its original software operation over the remainder. ++ */ ++size_t ossl_sg2002_cipher(unsigned int algorithm, unsigned int mode, int enc, ++ const unsigned char *key, size_t keylen, unsigned char *iv, ++ unsigned int counter_bytes, const unsigned char *in, unsigned char *out, ++ size_t len, size_t minimum, size_t continuation_minimum); + #endif +--- a/crypto/build.info ++++ b/crypto/build.info +@@ -146,6 +146,5 @@ + +-# NanoKVM direct DMA access is private to the normal/legacy providers. ++# NanoKVM direct DMA access is private to the normal provider. + IF[{- $target{asm_arch} eq "riscv64" -}] +- SOURCE[../libcrypto]=sg2002.c +- SOURCE[../providers/liblegacy.a]=sg2002.c ++ SOURCE[../libcrypto]=sg2002.c sg2002_cipher.c + ENDIF +--- a/providers/implementations/ciphers/cipher_aes.h ++++ b/providers/implementations/ciphers/cipher_aes.h +@@ -21,6 +21,9 @@ + /* Platform specific data */ + union { + int dummy; ++#if defined(__riscv) && __riscv_xlen == 64 && !defined(FIPS_MODULE) ++ struct { unsigned char key[32]; } sg2002; ++#endif + #if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) + struct { + union { +--- a/providers/implementations/ciphers/cipher_aes_gcm.h ++++ b/providers/implementations/ciphers/cipher_aes_gcm.h +@@ -22,6 +22,9 @@ + /* Platform specific data */ + union { + int dummy; ++#if defined(__riscv) && __riscv_xlen == 64 && !defined(FIPS_MODULE) ++ struct { unsigned char key[32]; } sg2002; ++#endif + #if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) + struct { + union { +--- a/providers/implementations/ciphers/cipher_aes_hw.c ++++ b/providers/implementations/ciphers/cipher_aes_hw.c +@@ -122,15 +122,24 @@ + return 0; + } + ++#if defined(__riscv) && __riscv_xlen == 64 && !defined(FIPS_MODULE) ++ memcpy(adat->plat.sg2002.key, key, keylen); ++#endif + return 1; + } ++ ++#if defined(__riscv) && __riscv_xlen == 64 && !defined(FIPS_MODULE) ++#include "cipher_aes_sg2002.inc" ++#else ++#define AES_GENERIC_CIPHER(mode) ossl_cipher_hw_generic_##mode ++#endif + + IMPLEMENT_CIPHER_HW_COPYCTX(cipher_hw_aes_copyctx, PROV_AES_CTX) + + #define PROV_CIPHER_HW_aes_mode(mode) \ + static const PROV_CIPHER_HW aes_##mode = { \ + cipher_hw_aes_initkey, \ +- ossl_cipher_hw_generic_##mode, \ ++ AES_GENERIC_CIPHER(mode), \ + cipher_hw_aes_copyctx \ + }; \ + PROV_CIPHER_HW_declare(mode) \ +--- a/providers/implementations/ciphers/cipher_aes_gcm_hw.c ++++ b/providers/implementations/ciphers/cipher_aes_gcm_hw.c +@@ -16,6 +16,10 @@ + #include "internal/deprecated.h" + + #include "cipher_aes_gcm.h" ++ ++#if defined(__riscv) && __riscv_xlen == 64 && !defined(FIPS_MODULE) ++#include "cipher_aes_gcm_sg2002.inc" ++#endif + + static int aes_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, + size_t keylen) +@@ -55,6 +59,10 @@ + GCM_HW_SET_KEY_CTR_FN(ks, AES_set_encrypt_key, AES_encrypt, NULL); + #endif /* AES_CTR_ASM */ + } ++#if defined(__riscv) && __riscv_xlen == 64 && !defined(FIPS_MODULE) ++ memcpy(actx->plat.sg2002.key, key, keylen); ++ ctx->ctr = aes_sg2002_gcm_ctr; ++#endif + return 1; + } + +--- /dev/null ++++ b/providers/implementations/ciphers/cipher_aes_sg2002.h +@@ -0,0 +1,8 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++#ifndef OSSL_NANOKVM_AES_SG2002_H ++#define OSSL_NANOKVM_AES_SG2002_H ++/* Native C906 thresholds against the extract/scaled-load software path. */ ++#define SG2002_AES_BLOCK_MIN 3072 ++#define SG2002_AES_CTR_MIN 2048 ++#define SG2002_AES_CONTINUE_MIN 512 ++#endif diff --git a/firmware/crypto/openssl-c906/README.md b/firmware/crypto/openssl-c906/README.md index 191fcd78..b8d23399 100644 --- a/firmware/crypto/openssl-c906/README.md +++ b/firmware/crypto/openssl-c906/README.md @@ -68,3 +68,20 @@ no EVP result. It is compiled only for RISC-V64, outside the FIPS module. See chunks. OpenSSL owns streaming, padding, digest lengths and software fallback. The FIPS module is excluded. See `experimental/openssl-sha256` for qualification; the selected 4 KiB threshold and scalar adapter have native qualification. + +## AES + +`0005-aes-c906.patch` separately optimizes the existing scalar AES table lookups +with XTheadBb byte extraction and scaled XTheadMemIdx loads. The original +RISC-V/FIPS assembly remains the fallback. It changes neither tables nor key +schedules. + +`0005-aes-dma.patch` adds ECB/CBC/CTR and the AES-CTR primitive used by GCM. +OpenSSL retains streaming, padding, GHASH, authentication and TLS handling. +The shared block helper commits only completed chunks and supports separate +full-width and GCM counters. See `experimental/openssl-aes` for qualification. + +The AES PR is based directly on the SHA-256 patch. SHA-1, DES, TDES and SM4 +OpenSSL offloads are not included. The AES helper rejects other algorithm IDs, +and the legacy provider does not link the SG2002 transport. AES block-mode +verification is self-contained in `experimental/openssl-aes/verify.c`.