diff --git a/.changes/016-http-client-byte-level-methods.md b/.changes/016-http-client-byte-level-methods.md new file mode 100644 index 000000000..8cabf0336 --- /dev/null +++ b/.changes/016-http-client-byte-level-methods.md @@ -0,0 +1,12 @@ +--- +group: platform +component: Framework +impact: compatible +category: Added +--- + +## Summary + +`Http.Client` can now send PUT, HEAD, DELETE, POST and PATCH requests with a raw byte body and read the status code and headers of every response, including 4xx and 5xx. Use `Http.Client.sendSecure method request body` for HTTPS services and `Http.Client.Internal.sendRaw` for trusted loopback endpoints. `Http.getSecure` and `getRaw` keep working exactly as before. + +Existing applications do not need code changes. To verify locally, run `./dev test "Http.Client" nhcore-test-core` and confirm the `sendRaw` and `sendSecure` examples pass. diff --git a/.changes/017-s3-blob-store.md b/.changes/017-s3-blob-store.md new file mode 100644 index 000000000..a20f7344f --- /dev/null +++ b/.changes/017-s3-blob-store.md @@ -0,0 +1,12 @@ +--- +group: platform +component: Framework +impact: compatible +category: Added +--- + +## Summary + +File uploads can now be stored in any S3-compatible object store (AWS S3, Cloudflare R2, Backblaze B2, MinIO, RustFS) with the new `Service.FileUpload.BlobStore.S3` backend. Build it in `App.hs` with `S3.createBlobStore S3Config { endpoint, bucket, region, accessKeyId, secretAccessKey }`, reading the values from your own `Config.hs`, and pass the result to `Application.withFileUpload` exactly like the local backend. Applications on the local filesystem backend are not affected. + +To verify locally, run `./dev test "BlobStore.S3" nhcore-test-service`; the suite signs requests against the published AWS examples and exercises the backend against an in-process fake S3 server. diff --git a/.changes/021-crypto-sha256-hmac-hex.md b/.changes/021-crypto-sha256-hmac-hex.md new file mode 100644 index 000000000..055b5f7f2 --- /dev/null +++ b/.changes/021-crypto-sha256-hmac-hex.md @@ -0,0 +1,12 @@ +--- +group: platform +component: Framework +impact: compatible +category: Added +--- + +## Summary + +`Crypto` now offers the raw building blocks behind signatures: `Crypto.sha256` (32-byte digest), `Crypto.hmacSha256 key message` (raw HMAC that can be chained, any key length) and `Crypto.toHex` (lowercase hexadecimal). Use them when a protocol tells you exactly which bytes to hash or sign; keep using `Crypto.signWith` and `Crypto.verifyWith` for webhook secrets, which are unchanged. + +Existing applications do not need code changes. To verify locally, run `./dev test "Crypto" nhcore-test-core` and confirm the `sha256`, `hmacSha256` and `toHex` examples pass. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 236a529fe..a24e04692 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -190,8 +190,21 @@ jobs: with: name: dist-newstyle path: dist-newstyle + # Tee the direct per-example output: integration criteria whose locators + # live in nhcore-test-core are proven from this report by the criteria + # gate in test-service. Non-hidden filename: upload-artifact skips + # dotfiles by default. - name: Run nhcore-test-core - run: LOG_LEVEL=Warn nix develop --command cabal test nhcore-test-core + run: | + set -o pipefail + LOG_LEVEL=Warn nix develop --command cabal test nhcore-test-core \ + --test-show-details=direct | tee nhcore-test-core.report + - uses: actions/upload-artifact@v7 + with: + name: nhcore-test-core-report + path: nhcore-test-core.report + retention-days: 1 + if-no-files-found: error test-auth: needs: [build] @@ -234,7 +247,8 @@ jobs: run: LOG_LEVEL=Warn nix develop --command cabal test nhcore-test-integration test-service: - needs: [build] + # test-core supplies its report for the criteria gate below. + needs: [build, test-core] runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -277,12 +291,21 @@ jobs: set -o pipefail LOG_LEVEL=Warn nix develop --command cabal test nhcore-test-service \ --test-show-details=direct | tee .nhcore-test-service.report + - uses: actions/download-artifact@v8 + if: github.event_name == 'pull_request' + with: + name: nhcore-test-core-report + # One report across both suites: a missing core report fails `cat` + # (closed), never a silently service-only gate. - name: Prove integration criterion selectors and real fixtures if: github.event_name == 'pull_request' env: BASE_REF: ${{ github.base_ref }} POSTGRES_AVAILABLE: "true" - run: ./dev spec-check --criteria-runtime "origin/${BASE_REF}" --report .nhcore-test-service.report + run: | + set -euo pipefail + cat nhcore-test-core.report .nhcore-test-service.report > nhcore-criteria.report + ./dev spec-check --criteria-runtime "origin/${BASE_REF}" --report nhcore-criteria.report test-hurl: needs: [build] diff --git a/codemap/.doc-ratchet b/codemap/.doc-ratchet index 8340fb92e..89588a5ba 100644 --- a/codemap/.doc-ratchet +++ b/codemap/.doc-ratchet @@ -1,5 +1,5 @@ # doc-ratchet: counters may DECREASE only (CI-enforced by check.py). # Source-derived (platform-independent) — see generate.py. undocumented_modules: 212 -undocumented_exports: 560 -undocumented_doctest_modules: 315 +undocumented_exports: 545 +undocumented_doctest_modules: 314 diff --git a/codemap/api-hot.md b/codemap/api-hot.md index c2b8e1dd7..cc1c04410 100644 --- a/codemap/api-hot.md +++ b/codemap/api-hot.md @@ -10,25 +10,25 @@ codemap/signatures/ · type-directed search: ./dev api "" ## Task -- `yield :: value -> Task w value` +- `yield :: value -> Task w value` - `ignoreError :: Task err Unit -> Task w Unit` -- `mapError :: (err1 -> err2) -> Task err1 value -> Task err2 value` -- `throw :: err -> Task err w` +- `mapError :: (err1 -> err2) -> Task err1 value -> Task err2 value` +- `throw :: err -> Task err w` - `fromIO :: IO value -> Task w value` - `asResult :: Task err value -> Task err2 (Result err value)` -- `when :: Bool -> Task err Unit -> Task err Unit` +- `when :: Bool -> Task err Unit -> Task err Unit` - `forEach :: (element -> Task err Unit) -> Array element -> Task err Unit` - `fromFailableIO :: Exception exception => IO result -> Task exception result` - `map :: (input -> output) -> Task err input -> Task err output` ## Array -- `map :: (a -> b) -> Array a -> Array b` +- `map :: (a -> b) -> Array a -> Array b` - `>>> map sqrt (fromLinkedList [1,4,9] :: Array Float)` → `Array [1.0,2.0,3.0]` -- `fromLinkedList :: LinkedList a -> Array a` -- `empty :: Array a` +- `fromLinkedList :: LinkedList a -> Array a` +- `empty :: Array a` - `>>> empty :: Array Int` → `Array []` -- `toLinkedList :: Array a -> LinkedList a` +- `toLinkedList :: Array a -> LinkedList a` - `>>> toLinkedList (fromLinkedList [3,5,8] :: Array Int)` → `[3,5,8]` - `isEmpty :: Array a -> Bool` - `>>> isEmpty empty` → `True` … @@ -47,14 +47,14 @@ codemap/signatures/ · type-directed search: ./dev api "" - `fromLinkedList :: LinkedList Char -> Text` - `toLinkedList :: Text -> LinkedList Char` -- `toBytes :: Text -> Bytes` +- `toBytes :: Text -> Bytes` - `replace :: Text -> Text -> Text -> Text` +- `isEmpty :: Text -> Bool` +- `joinWith :: Text -> Array Text -> Text` +- `trim :: Text -> Text` - `fromBytes :: Bytes -> Text` -- `trim :: Text -> Text` -- `joinWith :: Text -> Array Text -> Text` -- `isEmpty :: Text -> Bool` -- `append :: Text -> Text -> Text` -- `toLower :: Text -> Text` +- `append :: Text -> Text -> Text` +- `toLower :: Text -> Text` ## Json @@ -116,21 +116,21 @@ codemap/signatures/ · type-directed search: ./dev api "" ## Bytes -- `unwrap :: Bytes -> ByteString` -- `fromLegacy :: ByteString -> Bytes` -- `toLazyLegacy :: Bytes -> LazyByteString` -- `length :: Bytes -> Int` +- `unwrap :: Bytes -> ByteString` +- `fromLegacy :: ByteString -> Bytes` +- `toLazyLegacy :: Bytes -> LazyByteString` +- `empty :: Bytes` +- `length :: Bytes -> Int` - `>>> [1, 2, 3] |> Bytes.pack |> Bytes.length` → `3` - `pack :: [Word8] -> Bytes` - `toBase64 :: Bytes -> Bytes` - `dropEnd :: Int -> Bytes -> Bytes` - `isSuffixOf :: Bytes -> Bytes -> Bool` - `splitOnce :: Bytes -> Bytes -> Maybe (Bytes, Bytes)` -- `fromLazyLegacy :: LazyByteString -> Bytes` ## Maybe -- `withDefault :: a -> Maybe a -> a` +- `withDefault :: a -> Maybe a -> a` - `map :: (a -> b) -> Maybe a -> Maybe b` - `getOrDie :: HasCallStack => Maybe a -> a` - `andThen :: (a -> Maybe b) -> Maybe a -> Maybe b` @@ -169,7 +169,7 @@ codemap/signatures/ · type-directed search: ./dev api "" ## LinkedList -- `map :: (a -> b) -> LinkedList a -> LinkedList b` +- `map :: (a -> b) -> LinkedList a -> LinkedList b` - `any :: (a -> Bool) -> LinkedList a -> Bool` - `filter :: (a -> Bool) -> LinkedList a -> LinkedList a` - `reverse :: LinkedList a -> LinkedList a` @@ -201,6 +201,19 @@ codemap/signatures/ · type-directed search: ./dev api "" - `race :: Show err => Task err a -> Task err b -> Task err (RaceWinner a b)` - `waitCatch :: Show err => AsyncTask err result -> Task err2 (Result Text result)` +## Http.Client + +- `request :: Request` +- `withUrl :: Text -> Request -> Request` +- `withTimeout :: Int -> Request -> Request` +- `addHeader :: Text -> Text -> Request -> Request` +- `get :: FromJSON response => Request -> Task Error (Response response)` +- `postForm :: FromJSON response => Request -> Array (Text, Text) -> Task Error (Response response)` +- `methodName :: Method -> Text` +- `patch :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response)` +- `post :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response)` +- `put :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response)` + ## Path - `toLinkedList :: Path -> LinkedList Char` @@ -237,6 +250,13 @@ codemap/signatures/ · type-directed search: ./dev api "" - `selectEventBatch :: Var Int64 -> Maybe RelativePosition -> Maybe ReadDirection -> Maybe (Array EntityName) -> Task PostgresStoreError (Session (Array PostgresEventRecord))` - `selectEventByGlobalPositionSession :: Int64 -> Session (Maybe PostgresEventRecord)` +## DateTime + +- `now :: Task w DateTime` +- `toEpochSeconds :: DateTime -> Int64` +- `fromEpochSeconds :: Int64 -> DateTime` +- `addSeconds :: Int64 -> DateTime -> DateTime` + ## Stream - `fromArray :: Array value -> Task error (Stream value)` @@ -247,26 +267,6 @@ codemap/signatures/ · type-directed search: ./dev api "" - `consumeMaybe :: (accumulator -> value -> Task Text accumulator) -> accumulator -> Stream value -> Task Text (Maybe accumulator)` - `end :: Stream value -> Task error Unit` -## DateTime - -- `now :: Task w DateTime` -- `toEpochSeconds :: DateTime -> Int64` -- `fromEpochSeconds :: Int64 -> DateTime` -- `addSeconds :: Int64 -> DateTime -> DateTime` - -## Http.Client - -- `request :: Request` -- `withUrl :: Text -> Request -> Request` -- `withTimeout :: Int -> Request -> Request` -- `get :: FromJSON response => Request -> Task Error (Response response)` -- `postForm :: FromJSON response => Request -> Array (Text, Text) -> Task Error (Response response)` -- `addHeader :: Text -> Text -> Request -> Request` -- `patch :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response)` -- `post :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response)` -- `put :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response)` -- `delete :: FromJSON response => Request -> Task Error (Response response)` - ## Service.Transport.Mcp.JsonRpc - `errorResponse :: Maybe Value -> JsonRpcError -> JsonRpcResponse` @@ -283,7 +283,7 @@ codemap/signatures/ · type-directed search: ./dev api "" - `fmap :: Functor f => (a -> b) -> f a -> f b` - `map :: Mappable mappable => (typeA -> typeB) -> mappable typeA -> mappable typeB` -*cut: 103 more modules (AtomicVar (26), Integration.Http (26), Decider (21), Result (21), Lock (20), …) — full surface: codemap/signatures/* +*cut: 106 more modules (AtomicVar (26), Integration.Http (26), Decider (21), Result (21), Lock (20), …) — full surface: codemap/signatures/* --- @@ -294,35 +294,35 @@ when WRITING TESTS; they are not feature-code frequency signal. ## Task -- `mapError :: (err1 -> err2) -> Task err1 value -> Task err2 value` -- `yield :: value -> Task w value` -- `asResult :: Task err value -> Task err2 (Result err value)` +- `mapError :: (err1 -> err2) -> Task err1 value -> Task err2 value` +- `yield :: value -> Task w value` +- `asResult :: Task err value -> Task err2 (Result err value)` - `mapArray :: (element -> Task err output) -> Array element -> Task err (Array output)` - `throw :: err -> Task err w` +- `fromIO :: IO value -> Task w value` - `andThen :: (input -> Task err output) -> Task err input -> Task err output` -- `fromIO :: IO value -> Task w value` - `map :: (input -> output) -> Task err input -> Task err output` ## Array - `length :: Array a -> Int` -- `empty :: Array a` -- `fromLinkedList :: LinkedList a -> Array a` -- `map :: (a -> b) -> Array a -> Array b` -- `get :: Int -> Array a -> Maybe a` +- `empty :: Array a` +- `fromLinkedList :: LinkedList a -> Array a` +- `map :: (a -> b) -> Array a -> Array b` +- `get :: Int -> Array a -> Maybe a` - `wrap :: a -> Array a` - `initialize :: Int -> (Int -> a) -> Array a` - `contains :: Eq value => value -> Array value -> Bool` ## Text -- `contains :: Text -> Text -> Bool` -- `toBytes :: Text -> Bytes` +- `contains :: Text -> Text -> Bool` +- `toBytes :: Text -> Bytes` - `fromLinkedList :: LinkedList Char -> Text` - `length :: Text -> Int` - `repeat :: Int -> Text -> Text` - `toLinkedList :: Text -> LinkedList Char` -- `startsWith :: Text -> Text -> Bool` +- `startsWith :: Text -> Text -> Bool` - `all :: (Char -> Bool) -> Text -> Bool` ## Parser @@ -360,13 +360,13 @@ when WRITING TESTS; they are not feature-code frequency signal. ## ConcurrentVar -- `containing :: value -> Task w (ConcurrentVar value)` -- `modify :: (value -> value) -> ConcurrentVar value -> Task w Unit` -- `peek :: ConcurrentVar value -> Task w value` +- `containing :: value -> Task w (ConcurrentVar value)` +- `modify :: (value -> value) -> ConcurrentVar value -> Task w Unit` +- `peek :: ConcurrentVar value -> Task w value` - `get :: ConcurrentVar value -> Task w value` - `set :: value -> ConcurrentVar value -> Task w ()` - `new :: forall value w. Task w (ConcurrentVar value)` -- `swap :: value -> ConcurrentVar value -> Task w value` +- `swap :: value -> ConcurrentVar value -> Task w value` - `modifyReturning :: (value -> Task Never (value, a)) -> ConcurrentVar value -> Task w a` ## Uuid @@ -377,4 +377,4 @@ when WRITING TESTS; they are not feature-code frequency signal. - `generateV5 :: Uuid -> Text -> Uuid` - `fromText :: Text -> Maybe Uuid` -*cut: 138 more modules (Map (257), Service.Application (250), Set (233), AsyncTask (223), Stream (212), …) — full surface: codemap/signatures/* +*cut: 139 more modules (Map (257), Service.Application (250), Set (233), AsyncTask (223), Stream (212), …) — full surface: codemap/signatures/* diff --git a/codemap/phrasebook.md b/codemap/phrasebook.md index 5710950a9..d2be50466 100644 --- a/codemap/phrasebook.md +++ b/codemap/phrasebook.md @@ -89,6 +89,18 @@ Array [0] [] ``` +### Crypto + +```haskell +>>> Bytes.pack [0, 15, 255] |> Crypto.toHex +"000fff" +``` + +```haskell +>>> Text.toBytes "abc" |> Crypto.sha256 |> Crypto.toHex +"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +``` + ### Float ```haskell @@ -194,6 +206,27 @@ Set (fromList [1,2,3]) "Cart" ``` +## file-upload + +### Service.FileUpload.BlobStore.S3 + +```haskell +>>> isDnsBucketName "my-bucket" +True +``` + +```haskell +>>> isDnsBucketName "My_Bucket" +False +``` + +### Service.FileUpload.BlobStore.S3.SigV4 + +```haskell +>>> Text.toBytes "" |> sha256Hex +"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +``` + ## postgres-infra ### Service.Infra.Postgres.SslMode @@ -237,4 +270,4 @@ True --- -*32 example sessions · modules with ZERO doctests: 315 (the documentation backlog — see codemap/.doc-ratchet)* +*37 example sessions · modules with ZERO doctests: 314 (the documentation backlog — see codemap/.doc-ratchet)* diff --git a/codemap/signatures/nhcore-core.txt b/codemap/signatures/nhcore-core.txt index acf6d8f47..9dcd65284 100644 --- a/codemap/signatures/nhcore-core.txt +++ b/codemap/signatures/nhcore-core.txt @@ -576,9 +576,12 @@ data HmacKey generateHmacKey :: Task err HmacKey hmacKeyFromBytes :: Bytes -> Result Text HmacKey hmacKeyFromText :: Text -> Result Text HmacKey +hmacSha256 :: Bytes -> Bytes -> Bytes instance GHC.Classes.Eq Crypto.HmacKey instance GHC.Show.Show Crypto.HmacKey +sha256 :: Bytes -> Bytes signWith :: HmacKey -> Bytes -> Text +toHex :: Bytes -> Text verifyWith :: HmacKey -> Text -> Bytes -> Bool module DateTime @@ -725,8 +728,14 @@ unchanged :: value -> value module Http module Http.Client +Delete :: Method Error :: Text -> Error +Get :: Method +Head :: Method InvalidUrl :: Text -> Error +Patch :: Method +Post :: Method +Put :: Method Request :: Maybe Text -> Map Text Text -> Maybe Int -> Int -> !Maybe Int -> Request Response :: Int -> Array (Text, Text) -> body -> Response body ResponseTooLarge :: Int -> Error @@ -739,18 +748,23 @@ ResponseTooLarge :: Int -> Error [timeoutSeconds] :: Request -> Maybe Int [url] :: Request -> Maybe Text addHeader :: Text -> Text -> Request -> Request +applyMethodAndBody :: Method -> Bytes -> Request -> Request checkContentLengthHeaderLimit :: Maybe Int -> [(CI ByteString, ByteString)] -> Maybe Int data Error +data Method data Request data Response body delete :: FromJSON response => Request -> Task Error (Response response) get :: FromJSON response => Request -> Task Error (Response response) getSecure :: Request -> Task Error (Response Bytes) instance Data.Default.Internal.Default Http.Client.Request +instance GHC.Classes.Eq Http.Client.Method instance GHC.Generics.Generic (Http.Client.Response body) instance GHC.Show.Show Http.Client.Error +instance GHC.Show.Show Http.Client.Method instance GHC.Show.Show Http.Client.Request instance GHC.Show.Show body => GHC.Show.Show (Http.Client.Response body) +methodName :: Method -> Text patch :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response) post :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response) postForm :: FromJSON response => Request -> Array (Text, Text) -> Task Error (Response response) @@ -758,6 +772,7 @@ postRaw :: FromJSON response => Request -> Text -> Text -> Task Error (Response put :: (FromJSON response, ToJSON requestBody) => Request -> requestBody -> Task Error (Response response) request :: Request secureTlsSupportedVersions :: [Version] +sendSecure :: Method -> Request -> Bytes -> Task Error (Response Bytes) withMaxResponseSize :: Int -> Request -> Request withRedirects :: Int -> Request -> Request withTimeout :: Int -> Request -> Request @@ -765,6 +780,7 @@ withUrl :: Text -> Request -> Request module Http.Client.Internal getRaw :: Request -> Task Error (Response Bytes) +sendRaw :: Method -> Request -> Bytes -> Task Error (Response Bytes) module IO andThen :: (a -> IO b) -> IO a -> IO b diff --git a/codemap/signatures/nhcore-service.txt b/codemap/signatures/nhcore-service.txt index fcdbe6904..3f65e22ec 100644 --- a/codemap/signatures/nhcore-service.txt +++ b/codemap/signatures/nhcore-service.txt @@ -884,6 +884,38 @@ instance GHC.Classes.Eq Service.FileUpload.BlobStore.Local.LocalBlobStoreConfig instance GHC.Generics.Generic Service.FileUpload.BlobStore.Local.LocalBlobStoreConfig instance GHC.Show.Show Service.FileUpload.BlobStore.Local.LocalBlobStoreConfig +module Service.FileUpload.BlobStore.S3 +S3Config :: Text -> Text -> Text -> Text -> Text -> S3Config +[accessKeyId] :: S3Config -> Text +[bucket] :: S3Config -> Text +[endpoint] :: S3Config -> Text +[region] :: S3Config -> Text +[secretAccessKey] :: S3Config -> Text +createBlobStore :: S3Config -> Task Text BlobStore +data S3Config + +module Service.FileUpload.BlobStore.S3.SigV4 +CanonicalRequest :: Text -> Text -> Array (Text, Text) -> Array (Text, Text) -> Text -> CanonicalRequest +Credentials :: Text -> Text -> Text -> Credentials +[accessKeyId] :: Credentials -> Text +[headers] :: CanonicalRequest -> Array (Text, Text) +[method] :: CanonicalRequest -> Text +[path] :: CanonicalRequest -> Text +[payloadHash] :: CanonicalRequest -> Text +[query] :: CanonicalRequest -> Array (Text, Text) +[region] :: Credentials -> Text +[secretAccessKey] :: Credentials -> Text +authorizationHeader :: Credentials -> DateTime -> CanonicalRequest -> Text +canonicalRequest :: CanonicalRequest -> Text +data CanonicalRequest +data Credentials +formatAmzDate :: DateTime -> Text +sha256Hex :: Bytes -> Text +signature :: Credentials -> DateTime -> CanonicalRequest -> Text +signedHeaders :: Array (Text, Text) -> Text +stringToSign :: Credentials -> DateTime -> CanonicalRequest -> Text +uriEncode :: Text -> Text + module Service.FileUpload.Core AdminPurge :: FileDeletionReason BlobKey :: Text -> BlobKey diff --git a/core/core/Crypto.hs b/core/core/Crypto.hs index 03fb6c83e..7e4667c76 100644 --- a/core/core/Crypto.hs +++ b/core/core/Crypto.hs @@ -45,6 +45,11 @@ module Crypto ( -- * Verification verifyWith, + + -- * Primitives + sha256, + hmacSha256, + toHex, ) where import Basics @@ -127,13 +132,10 @@ generateHmacKey = do -- |> Crypto.signWith key -- @ signWith :: HmacKey -> Bytes -> Text -signWith (HmacKey keyBytes) message = do - let messageBytes = Bytes.unwrap message - let hmacResult = HMAC.hmac keyBytes messageBytes :: HMAC.HMAC Hash.SHA256 - let signatureBytes = BA.convert hmacResult :: BS.ByteString - let encoded = Encoding.convertToBase Encoding.Base16 signatureBytes :: BS.ByteString - Bytes.fromLegacy encoded - |> Text.fromBytes +signWith (HmacKey keyBytes) message = + message + |> hmacSha256 (Bytes.fromLegacy keyBytes) + |> toHex -- | Verify an HMAC-SHA256 signature in constant time. @@ -152,3 +154,52 @@ verifyWith key signature message = do let expectedBytes = Text.toBytes expected |> Bytes.unwrap let providedBytes = Text.toBytes (Text.toLower signature) |> Bytes.unwrap BA.constEq providedBytes expectedBytes + + +-- | Raw SHA-256 digest (32 bytes) of the given bytes. +-- +-- This is the unkeyed hash. For message authentication use 'signWith' +-- or 'hmacSha256' instead. +-- +-- @ +-- payload +-- |> Crypto.sha256 +-- |> Crypto.toHex +-- @ +sha256 :: Bytes -> Bytes +sha256 message = do + let digest = Hash.hash (Bytes.unwrap message) :: Hash.Digest Hash.SHA256 + Bytes.fromLegacy (BA.convert digest :: BS.ByteString) + + +-- | Raw HMAC-SHA256 (32 bytes) of a message under an arbitrary key. +-- +-- Unlike 'signWith' this takes any key bytes and returns the raw MAC, so +-- it can be chained — protocols such as AWS Signature V4 derive a signing +-- key by feeding one HMAC result into the next. For a single webhook +-- signature with a fixed secret prefer 'signWith' and 'verifyWith', which +-- also enforce the 32-byte minimum key length and compare in constant +-- time. +-- +-- @ +-- message +-- |> Crypto.hmacSha256 key +-- @ +hmacSha256 :: Bytes -> Bytes -> Bytes +hmacSha256 key message = do + let mac = HMAC.hmac (Bytes.unwrap key) (Bytes.unwrap message) :: HMAC.HMAC Hash.SHA256 + Bytes.fromLegacy (BA.convert mac :: BS.ByteString) + + +-- | Lowercase hexadecimal text of the given bytes (two characters per byte). +-- +-- >>> Bytes.pack [0, 15, 255] |> Crypto.toHex +-- "000fff" +-- +-- >>> Text.toBytes "abc" |> Crypto.sha256 |> Crypto.toHex +-- "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +toHex :: Bytes -> Text +toHex bytes = do + let encoded = Encoding.convertToBase Encoding.Base16 (Bytes.unwrap bytes) :: BS.ByteString + Bytes.fromLegacy encoded + |> Text.fromBytes diff --git a/core/http/Http/Client.hs b/core/http/Http/Client.hs index 6cbcf1452..71b93830c 100644 --- a/core/http/Http/Client.hs +++ b/core/http/Http/Client.hs @@ -12,8 +12,12 @@ module Http.Client ( Response (..), -- * HTTP Methods + Method (..), + methodName, get, getSecure, + sendSecure, + applyMethodAndBody, secureTlsSupportedVersions, checkContentLengthHeaderLimit, -- ^ Enforces HTTPS (rejects http:// URLs). Use for all external API calls. @@ -119,10 +123,12 @@ instance Default Request where } +-- | An empty request: no URL, no headers, default timeout and limits. request :: Request request = def +-- | Set the request URL. withUrl :: Text -> Request -> Request withUrl url options = options @@ -144,6 +150,7 @@ withTimeout seconds options = panic [fmt|withTimeout requires positive seconds, got: #{seconds}|] +-- | Add one header; later additions win on duplicate names. addHeader :: Text -> Text -> Request -> Request addHeader key value options = let newHeaders = options.headers |> Map.set key value @@ -174,6 +181,7 @@ withRedirects count options = -- | Set the maximum response body size in bytes. -- Default is 10MB. Use this to override for specific endpoints. {-# INLINE withMaxResponseSize #-} +-- | Cap the response body in bytes; larger bodies fail with a size error. withMaxResponseSize :: GhcInt.Int -> Request -> Request withMaxResponseSize maxBytes options = case maxBytes > 0 of @@ -185,6 +193,24 @@ withMaxResponseSize maxBytes options = panic [fmt|withMaxResponseSize requires positive bytes, got: #{maxBytes}|] +-- | HTTP method for byte-level requests ('sendSecure', 'Http.Client.Internal.sendRaw'). +-- A closed set: protocol clients (object stores, webhooks) pick a verb from here +-- instead of spelling the wire token themselves. +data Method = Get | Head | Post | Put | Patch | Delete + deriving (Eq, Show) + + +-- | The wire token of a 'Method' ("GET", "HEAD", ...). +methodName :: Method -> Text +methodName method = case method of + Get -> "GET" + Head -> "HEAD" + Post -> "POST" + Put -> "PUT" + Patch -> "PATCH" + Delete -> "DELETE" + + data Error = Error Text | InvalidUrl Text -- ^ The URL does not use HTTPS. Use getSecure only with https:// URLs. @@ -285,6 +311,7 @@ requestHost :: Request -> Text requestHost options = options.url |> Maybe.map sanitizeUrlText |> Maybe.withDefault "" +-- | GET a JSON document from any URL (plain HTTP allowed; prefer getSecure). get :: (Json.FromJSON response) => Request -> @@ -317,6 +344,7 @@ getIO options = do secureTlsSupportedVersions :: [TLS.Version] secureTlsSupportedVersions = [TLS.TLS13, TLS.TLS12] +-- | Build the TLS 1.2+ connection manager shared by every secure request. secureTlsManager :: GhcIO.IO HttpClient.Manager secureTlsManager = do let tlsParams = @@ -331,14 +359,33 @@ secureTlsManager = do HttpClient.newManager managerSettings {-# NOINLINE cachedSecureTlsManager #-} +-- | The process-wide secure manager, built once. cachedSecureTlsManager :: HttpClient.Manager cachedSecureTlsManager = GhcUnsafe.unsafePerformIO secureTlsManager {-# INLINE getSecure #-} +-- | GET raw bytes over HTTPS only; returns the status code instead of throwing on 4xx/5xx. getSecure :: Request -> Task Error (Response Bytes) -getSecure options = do +getSecure options = sendSecure Get options Bytes.empty + + +-- | Performs a byte-level request with any 'Method', enforcing HTTPS and TLS 1.2+. +-- +-- Built for protocol clients (object stores, signed webhooks) that need the +-- status code and headers of every response, including 4xx/5xx: this never +-- throws on a non-2xx status, never follows redirects beyond 'maxRedirects', +-- and never decodes the body. The request body is sent as-is; set +-- @Content-Type@ with 'addHeader'. The response body is bounded by +-- 'maxResponseBytes' exactly like 'getSecure'. +-- Use 'Http.Client.Internal.sendRaw' only for trusted loopback endpoints. +sendSecure :: + Method -> + Request -> + Bytes -> + Task Error (Response Bytes) +sendSecure method options body = do let urlText = options.url |> Maybe.withDefault "" case Text.startsWith "https://" urlText of False -> @@ -346,30 +393,37 @@ getSecure options = do Task.throw (InvalidUrl (sanitizeUrlText urlText)) True -> do let host = requestHost options - Log.debug [fmt|HTTP GET (secure) #{host}|] |> Task.ignoreError - secureResult <- getSecureRawIO options |> Task.fromIO + let verb = methodName method + Log.debug [fmt|HTTP #{verb} (secure) #{host}|] |> Task.ignoreError + secureResult <- sendSecureRawIO method options body |> Task.fromIO case secureResult of Err error -> Task.throw error Ok response -> checkResponseSize options response --- | Internal IO action for secure raw GET request using TLS 1.2+ only. -getSecureRawIO :: Request -> GhcIO.IO (Result Error (Response Bytes)) -getSecureRawIO options = do +-- | Internal IO action for a secure byte-level request using TLS 1.2+ only. +sendSecureRawIO :: Method -> Request -> Bytes -> GhcIO.IO (Result Error (Response Bytes)) +sendSecureRawIO method options body = do httpResult <- case options.maxResponseBytes of Nothing -> GhcException.try @HttpClient.HttpException do - response <- getRawIO options + response <- sendRawIO method options body pure (Ok response) Just _ -> GhcException.try @HttpClient.HttpException do baseReq <- parseRequestUrl options let req = baseReq |> applyRequestOptions options + |> applyMethodAndBody method body |> setRequestIgnoreStatus rawResponse <- HttpClient.responseOpen req cachedSecureTlsManager - case checkContentLengthHeaderLimit options.maxResponseBytes (HttpClient.responseHeaders rawResponse) of + -- A HEAD response announces the body it does NOT send; only the + -- post-read check applies to it. + let declaredLimit = case method of + Head -> Nothing + _ -> checkContentLengthHeaderLimit options.maxResponseBytes (HttpClient.responseHeaders rawResponse) + case declaredLimit of Just maxBytes -> do HttpClient.responseClose rawResponse pure (Err (ResponseTooLarge maxBytes)) @@ -384,24 +438,37 @@ getSecureRawIO options = do GhcEither.Right responseResult -> pure responseResult --- | Internal IO action for secure raw GET request using TLS 1.2+ only. -getRawIO :: +-- | Internal IO action for an unbounded secure byte-level request using TLS 1.2+ only. +sendRawIO :: + Method -> Request -> + Bytes -> GhcIO.IO (Response Bytes) -getRawIO options = do +sendRawIO method options body = do baseReq <- parseRequestUrl options let req = baseReq |> applyRequestOptions options + |> applyMethodAndBody method body |> setRequestIgnoreStatus |> HttpSimple.setRequestManager cachedSecureTlsManager httpResponse <- HttpSimple.httpBS req pure (extractResponseBytes httpResponse) +-- | Set the wire method and the raw request body. Shared with 'Http.Client.Internal'. +applyMethodAndBody :: Method -> Bytes -> HttpClient.Request -> HttpClient.Request +applyMethodAndBody method body req = + req + |> HttpSimple.setRequestMethod (methodName method |> Text.convert) + |> HttpSimple.setRequestBodyLBS (Bytes.toLazyLegacy body) + + +-- | Read the whole body reader into memory, honouring the size limit. readBodyFully :: HttpClient.BodyReader -> GhcIO.IO ByteString readBodyFully reader = readBodyChunks reader GhcBSChar.empty +-- | Accumulate body chunks until EOF or the size limit. readBodyChunks :: HttpClient.BodyReader -> ByteString -> GhcIO.IO ByteString readBodyChunks reader accumulatedBody = do chunk <- HttpClient.brRead reader @@ -412,6 +479,7 @@ readBodyChunks reader accumulatedBody = do readBodyChunks reader nextBody +-- | Reject early when @Content-Length@ already exceeds the limit. checkContentLengthHeaderLimit :: Maybe GhcInt.Int -> [(CI.CI ByteString, ByteString)] -> Maybe GhcInt.Int checkContentLengthHeaderLimit maxResponseLimit responseHeaders = case maxResponseLimit of @@ -429,6 +497,7 @@ checkContentLengthHeaderLimit maxResponseLimit responseHeaders = False -> Nothing +-- | Parse a @Content-Length@ value, if well-formed. parseContentLengthHeader :: ByteString -> Maybe GhcInt.Int parseContentLengthHeader rawContentLength = case GhcBSChar.readInt rawContentLength of @@ -439,6 +508,7 @@ parseContentLengthHeader rawContentLength = False -> Nothing +-- | First value for a header name, if present. findHeaderValue :: CI.CI ByteString -> [(CI.CI ByteString, ByteString)] -> Maybe ByteString findHeaderValue headerName headers = case headers of @@ -449,6 +519,7 @@ findHeaderValue headerName headers = False -> findHeaderValue headerName remainingHeaders +-- | Drain an open response into bytes under the size limit. extractResponseBytesFromOpenResponse :: HttpClient.Response HttpClient.BodyReader -> ByteString -> Response Bytes extractResponseBytesFromOpenResponse rawResponse responseBody = Response { statusCode = rawResponse |> HttpClient.responseStatus |> HttpStatus.statusCode @@ -457,6 +528,7 @@ extractResponseBytesFromOpenResponse rawResponse responseBody = Response } +-- | Convert wire headers to @(name, value)@ text pairs. extractRawHeaders :: [(CI.CI ByteString, ByteString)] -> Array (Text, Text) extractRawHeaders rawHeaders = rawHeaders diff --git a/core/http/Http/Client/Internal.hs b/core/http/Http/Client/Internal.hs index 19fa029a8..e002a7a2c 100644 --- a/core/http/Http/Client/Internal.hs +++ b/core/http/Http/Client/Internal.hs @@ -3,6 +3,7 @@ module Http.Client.Internal -- | WARNING: Does NOT enforce HTTPS. Use 'Http.Client.getSecure' for external APIs. -- This module is for internal use only (e.g., localhost OAuth2 discovery per ADR-0018). getRaw, + sendRaw, ) where @@ -14,7 +15,7 @@ import Bytes qualified import Data.ByteString (ByteString) import Data.CaseInsensitive qualified as CI import Data.Either qualified as GhcEither -import Http.Client (Request (..), Response (..)) +import Http.Client (Method, Request (..), Response (..)) import Http.Client qualified as Http import Log qualified import Map qualified @@ -41,9 +42,24 @@ import ToText (toText) getRaw :: Http.Request -> Task Http.Error (Http.Response Bytes) -getRaw options = do - Log.debug "HTTP GET (raw/internal)" |> Task.ignoreError - response <- getRawInternalIO options +getRaw options = sendRaw Http.Get options Bytes.empty + + +-- | Performs a byte-level request with any 'Method' without enforcing HTTPS. +-- The plain-HTTP twin of 'Http.Client.sendSecure': same non-throwing status +-- handling, same response-size limit, same redirect/proxy defaults. +-- +-- WARNING: Does NOT enforce HTTPS. Only for trusted loopback endpoints +-- (local object-store fixtures, OAuth2 discovery per ADR-0018). +sendRaw :: + Method -> + Http.Request -> + Bytes -> + Task Http.Error (Http.Response Bytes) +sendRaw method options body = do + let verb = Http.methodName method + Log.debug [fmt|HTTP #{verb} (raw/internal)|] |> Task.ignoreError + response <- sendRawInternalIO method options body |> Task.fromFailableIO @HttpClient.HttpException |> Task.mapError sanitizeInternalHttpError case options.maxResponseBytes of @@ -54,15 +70,18 @@ getRaw options = do False -> Task.yield response --- | Internal IO action for raw GET request -getRawInternalIO :: +-- | Internal IO action for a raw byte-level request +sendRawInternalIO :: + Method -> Http.Request -> + Bytes -> GhcIO.IO (Http.Response Bytes) -getRawInternalIO options = do +sendRawInternalIO method options body = do baseReq <- parseRequestUrl options let req = baseReq |> applyRequestOptions options + |> Http.applyMethodAndBody method body |> setRequestIgnoreStatus httpResponse <- HttpSimple.httpBS req pure (extractResponseBytes httpResponse) diff --git a/core/nhcore.cabal b/core/nhcore.cabal index 4a0a2b444..f462f1eb5 100644 --- a/core/nhcore.cabal +++ b/core/nhcore.cabal @@ -280,6 +280,8 @@ library Service.FileUpload.Lifecycle Service.FileUpload.BlobStore Service.FileUpload.BlobStore.Local + Service.FileUpload.BlobStore.S3 + Service.FileUpload.BlobStore.S3.SigV4 Service.FileUpload.FileStateStore.Postgres Service.FileUpload.Routes Service.FileUpload.Resolver @@ -491,6 +493,9 @@ test-suite nhcore-test Service.FileUpload.CoreSpec Service.FileUpload.LifecycleSpec Service.FileUpload.BlobStore.LocalSpec + Service.FileUpload.BlobStore.S3Spec + Service.FileUpload.BlobStore.S3.FakeS3 + Service.FileUpload.BlobStore.S3.SigV4Spec Service.FileUpload.RoutesSpec Service.FileUpload.ResolverSpec Service.FileUpload.DownloadSpec @@ -597,6 +602,9 @@ test-suite nhcore-test-service Service.EventStore.Postgres.NotificationsSpec Service.EventStore.PostgresSpec Service.FileUpload.BlobStore.LocalSpec + Service.FileUpload.BlobStore.S3Spec + Service.FileUpload.BlobStore.S3.FakeS3 + Service.FileUpload.BlobStore.S3.SigV4Spec Service.FileUpload.ContentDedupSpec Service.FileUpload.CoreSpec Service.FileUpload.DownloadSpec diff --git a/core/service/Service/FileUpload/BlobStore.hs b/core/service/Service/FileUpload/BlobStore.hs index 284d67a48..1e447ed6a 100644 --- a/core/service/Service/FileUpload/BlobStore.hs +++ b/core/service/Service/FileUpload/BlobStore.hs @@ -39,7 +39,7 @@ instance Show BlobStoreError where -- ========================================================================== -- | Interface for blob storage backends --- Implementations: Local (filesystem), S3 (future), GCS (future) +-- Implementations: Local (filesystem), S3 (S3-compatible object stores), GCS (future) data BlobStore = BlobStore { store :: BlobKey -> Bytes -> Task BlobStoreError Unit -- ^ Store bytes under the given key (overwrites if exists) diff --git a/core/service/Service/FileUpload/BlobStore/S3.hs b/core/service/Service/FileUpload/BlobStore/S3.hs new file mode 100644 index 000000000..e7bf4ff62 --- /dev/null +++ b/core/service/Service/FileUpload/BlobStore/S3.hs @@ -0,0 +1,329 @@ +-- | S3-compatible blob store (AWS S3, Cloudflare R2, Backblaze B2, MinIO, +-- RustFS, ...). Path-style addressing, SigV4 request signing, one HTTP +-- request per operation through the Core HTTP client. +-- +-- Same shape as "Service.FileUpload.BlobStore.Local": the configuration record +-- owns everything the backend needs, and 'createBlobStore' is the only +-- constructor. The bucket must already exist. HTTPS is required unless the +-- endpoint is a literal loopback address (local fixtures and tests). +-- Redirects are never followed; nothing is retried, so a failed write has an +-- unknown outcome and surfaces as 'StorageError'. +-- +-- Azure Blob Storage speaks a different protocol (SharedKey) and would be a +-- sibling module with this same shape, not a variant of this one. +module Service.FileUpload.BlobStore.S3 ( + -- * Configuration + S3Config (..), + + -- * Constructor + createBlobStore, +) where + +import Array (Array) +import Array qualified +import Basics +import Bytes (Bytes) +import Bytes qualified +import Char qualified +import DateTime (DateTime) +import DateTime qualified +import Http.Client (Method (..), Request, Response) +import Http.Client qualified as Http +import Http.Client.Internal qualified as HttpInternal +import Maybe qualified +import Result (Result (..)) +import Service.FileUpload.BlobStore (BlobStore (..), BlobStoreError (..)) +import Service.FileUpload.BlobStore.S3.SigV4 (CanonicalRequest (..), Credentials (..)) +import Service.FileUpload.BlobStore.S3.SigV4 qualified as SigV4 +import Service.FileUpload.Core (BlobKey (..)) +import Task (Task) +import Task qualified +import Text (Text) +import Text qualified + + +-- ========================================================================== +-- Configuration +-- ========================================================================== + +-- | Configuration for an S3-compatible blob store. +-- +-- Read the credentials from the application's own configuration (e.g. the +-- app's @Config.hs@) and build this record in @App.hs@; the blob store never +-- reads the environment itself. No 'Show' instance: the secret must not reach +-- logs. +data S3Config = S3Config + { endpoint :: Text + -- ^ Scheme, host and optional port only, e.g. @https://s3.eu-central-1.amazonaws.com@ + -- or @http://127.0.0.1:9000@. No path, query, fragment or userinfo. + , bucket :: Text + -- ^ Existing bucket; DNS-style name (3-63 chars of @a-z 0-9 - .@). + , region :: Text + -- ^ SigV4 region, e.g. @us-east-1@ (R2, B2 and MinIO accept @auto@ or @us-east-1@). + , accessKeyId :: Text + , secretAccessKey :: Text + } + + +-- | Validated connection parameters derived once from 'S3Config'. +data S3Store = S3Store + { baseUrl :: Text + -- ^ Endpoint without trailing slash. + , hostHeader :: Text + -- ^ Host header value: host plus port when the port is not the scheme default. + , secure :: Bool + , bucketName :: Text + , credentials :: Credentials + } + + +-- ========================================================================== +-- Constructor +-- ========================================================================== + +-- | Create an S3 blob store. Fails on an invalid configuration; does not +-- contact the endpoint. +createBlobStore :: S3Config -> Task Text BlobStore +createBlobStore config = do + store <- validateConfig config + Task.yield + BlobStore + { store = storeImpl store + , retrieve = retrieveImpl store + , delete = deleteImpl store + , exists = existsImpl store + } + + +-- ========================================================================== +-- Validation +-- ========================================================================== + +-- | Check the endpoint scheme, bucket name and credentials once, before any request. +validateConfig :: S3Config -> Task Text S3Store +validateConfig config = do + Task.when + (Text.isEmpty config.bucket || Text.isEmpty config.region || Text.isEmpty config.accessKeyId || Text.isEmpty config.secretAccessKey) + (Task.throw "S3 configuration fields must not be empty") + Task.unless (isDnsBucketName config.bucket) (Task.throw "S3 bucket must be a DNS-style bucket name") + endpoint <- parseEndpoint config.endpoint + Task.yield + S3Store + { baseUrl = endpoint.url + , hostHeader = endpoint.hostHeader + , secure = endpoint.secure + , bucketName = config.bucket + , credentials = + Credentials + { accessKeyId = config.accessKeyId + , secretAccessKey = config.secretAccessKey + , region = config.region + } + } + + +-- | True for a bucket name that is valid in a path-style S3 URL: 3 to 63 +-- lowercase ASCII letters, digits, hyphens and dots, where every dot-separated +-- label is non-empty and starts and ends with a letter or digit. +-- +-- >>> isDnsBucketName "my-bucket.v2" +-- True +-- +-- >>> isDnsBucketName "My_Bucket" +-- False +-- +-- >>> isDnsBucketName "a..b" +-- False +isDnsBucketName :: Text -> Bool +isDnsBucketName name = do + let validChar char = (Char.isLower char && Char.toCode char < 128) || Char.isDigit char || char == '-' || char == '.' + let invalidLabel label = Text.isEmpty label || Text.startsWith "-" label || Text.endsWith "-" label + Text.length name >= 3 + && Text.length name <= 63 + && Text.all validChar name + && not (name |> Text.split "." |> Array.any invalidLabel) + + +data Endpoint = Endpoint + { url :: Text + , hostHeader :: Text + , secure :: Bool + } + + +-- | Accept @scheme://host[:port]@ only. HTTPS everywhere except literal loopback. +parseEndpoint :: Text -> Task Text Endpoint +parseEndpoint raw = do + let trimmed = raw |> Text.trim |> stripTrailingSlash + (isSecure, authority) <- + if Text.startsWith "https://" trimmed + then Task.yield (True, Text.dropLeft 8 trimmed) + else + if Text.startsWith "http://" trimmed + then Task.yield (False, Text.dropLeft 7 trimmed) + else Task.throw "S3 endpoint must start with https:// or http://" + Task.when + (Text.isEmpty authority || Text.any (\char -> char == '/' || char == '?' || char == '#' || char == '@' || char == ' ') authority) + (Task.throw "S3 endpoint must not contain credentials, path, query or fragment") + let host = hostOf authority + Task.unless (isSecure || isLoopback host) (Task.throw "S3 requires HTTPS outside literal loopback") + let defaultPort = if isSecure then ":443" else ":80" + let hostHeader = if Text.endsWith defaultPort authority then host else authority + Task.yield Endpoint {url = trimmed, hostHeader, secure = isSecure} + + +-- | Remove one trailing slash so paths join without doubling it. +stripTrailingSlash :: Text -> Text +stripTrailingSlash text = if Text.endsWith "/" text then Text.dropRight 1 text else text + + +-- | Host without the port; keeps IPv6 brackets ("[::1]:9000" -> "[::1]"). +hostOf :: Text -> Text +hostOf authority = + if Text.startsWith "[" authority + then do + let inside = Text.split "]" authority |> Array.first |> Maybe.withDefault authority + [fmt|#{inside}]|] + else Text.split ":" authority |> Array.first |> Maybe.withDefault authority + + +-- | True for literal loopback hosts, the only ones allowed over plain HTTP. +isLoopback :: Text -> Bool +isLoopback host = host == "127.0.0.1" || host == "localhost" || host == "[::1]" + + +-- ========================================================================== +-- Operations +-- ========================================================================== + +-- | 'BlobStore.store': PUT the bytes; any non-2xx is a storage error. +storeImpl :: S3Store -> BlobKey -> Bytes -> Task BlobStoreError Unit +storeImpl store blobKey bytes = do + response <- sendObject store Put blobKey bytes + Task.unless (response.statusCode == 200) (Task.throw (statusError response)) + + +-- | 'BlobStore.retrieve': GET; 404 is 'BlobNotFound', other non-2xx are errors. +retrieveImpl :: S3Store -> BlobKey -> Task BlobStoreError Bytes +retrieveImpl store blobKey = do + response <- sendObject store Get blobKey Bytes.empty + if response.statusCode == 200 + then Task.yield response.body + else + if response.statusCode == 404 + then Task.throw (NotFound blobKey) + else Task.throw (statusError response) + + +-- | Idempotent: S3 answers 204 whether or not the object existed; some +-- compatible stores answer 200 or 404 for a missing key. +deleteImpl :: S3Store -> BlobKey -> Task BlobStoreError Unit +deleteImpl store blobKey = do + response <- sendObject store Delete blobKey Bytes.empty + let gone = response.statusCode == 204 || response.statusCode == 200 || response.statusCode == 404 + Task.unless gone (Task.throw (statusError response)) + + +-- | 'BlobStore.exists': HEAD; 200 is True, 404 is False, anything else is an error. +existsImpl :: S3Store -> BlobKey -> Task BlobStoreError Bool +existsImpl store blobKey = do + response <- sendObject store Head blobKey Bytes.empty + if response.statusCode == 200 + then Task.yield True + else + if response.statusCode == 404 + then Task.yield False + else Task.throw (statusError response) + + +-- | The HTTP status is the only detail that leaves the backend: S3 error +-- bodies may echo the request, and 403 bodies describe the signature. +statusError :: Response Bytes -> BlobStoreError +statusError response = do + let status = response.statusCode + StorageError [fmt|S3 HTTP #{status}|] + + +-- ========================================================================== +-- Transport +-- ========================================================================== + +-- | One signed request against @/bucket/key@. Transport failures (DNS, TLS, +-- timeout, oversized body) become 'StorageError' without the URL. +sendObject :: S3Store -> Method -> BlobKey -> Bytes -> Task BlobStoreError (Response Bytes) +sendObject store method blobKey body = do + path <- case objectPath store blobKey of + Err err -> Task.throw err + Ok validPath -> Task.yield validPath + now <- DateTime.now + let request = signedRequest store method path body now + let send = if store.secure then Http.sendSecure else HttpInternal.sendRaw + send method request body + |> Task.mapError transportError + + +-- | Map a transport failure to 'StorageError' without leaking the signed request. +transportError :: Http.Error -> BlobStoreError +transportError err = case err of + Http.ResponseTooLarge limit -> StorageError [fmt|S3 response exceeded #{limit} bytes|] + Http.InvalidUrl _ -> StorageError "S3 endpoint rejected by the HTTP client" + Http.Error _ -> StorageError "S3 transport failure (write outcome may be unknown)" + + +-- | Build the request with the SigV4 headers. 'Request' redacts headers in +-- its 'Show' instance, so the Authorization value cannot leak through logs. +signedRequest :: S3Store -> Method -> Text -> Bytes -> DateTime -> Request +signedRequest store method path body now = do + let payloadHash = SigV4.sha256Hex body + let headers = + [ ("host", store.hostHeader) + , ("x-amz-content-sha256", payloadHash) + , ("x-amz-date", SigV4.formatAmzDate now) + ] + let canonical = + CanonicalRequest + { method = Http.methodName method + , path + , query = Array.empty + , headers + , payloadHash + } + let authorization = SigV4.authorizationHeader store.credentials now canonical + let baseUrl = store.baseUrl + Http.request + |> Http.withUrl [fmt|#{baseUrl}#{path}|] + |> Http.withTimeout 30 + |> withHeaders headers + |> Http.addHeader "Authorization" authorization + |> Http.withMaxResponseSize maxObjectBytes + + +-- | Hard ceiling on one GET body; matches the upload route's default order of +-- magnitude and protects the process from a hostile endpoint. +maxObjectBytes :: Int +maxObjectBytes = 256 * 1024 * 1024 + + +-- | Add every header pair to the request, in order. +withHeaders :: Array (Text, Text) -> Request -> Request +withHeaders headers request = + headers + |> Array.foldl (\(name, value) acc -> Http.addHeader name value acc) request + + +-- | @/bucket/segment/segment@ with every segment URI-encoded. Slashes are +-- kept as separators; dot segments are rejected so an intermediary cannot +-- normalize the path to a different key. +objectPath :: S3Store -> BlobKey -> Result BlobStoreError Text +objectPath store (BlobKey key) = do + let byteLength = key |> Text.toBytes |> Bytes.length + let segments = Text.split "/" key + if byteLength == 0 || byteLength > 1024 + then Err (InvalidBlobKey "S3 key must contain 1 to 1024 UTF-8 bytes") + else + if Array.any (\segment -> segment == "." || segment == ".." || Text.isEmpty segment) segments + then Err (InvalidBlobKey "S3 key must not contain empty or dot path segments") + else do + let encoded = segments |> Array.map SigV4.uriEncode |> Text.joinWith "/" + let bucket = store.bucketName + Ok [fmt|/#{bucket}/#{encoded}|] diff --git a/core/service/Service/FileUpload/BlobStore/S3/SigV4.hs b/core/service/Service/FileUpload/BlobStore/S3/SigV4.hs new file mode 100644 index 000000000..332da6f98 --- /dev/null +++ b/core/service/Service/FileUpload/BlobStore/S3/SigV4.hs @@ -0,0 +1,256 @@ +-- | AWS Signature Version 4 for the S3 service ("s3", single-chunk payloads). +-- +-- Pure functions only: the caller supplies the timestamp, so every step +-- (canonical request, string to sign, signature, Authorization header) is +-- checkable against the published AWS test vectors without a clock or a +-- network. Used by "Service.FileUpload.BlobStore.S3"; any SigV4-compatible +-- object store (AWS S3, Cloudflare R2, Backblaze B2, MinIO, RustFS) accepts +-- the result. +module Service.FileUpload.BlobStore.S3.SigV4 ( + -- * Inputs + Credentials (..), + CanonicalRequest (..), + + -- * Steps (exposed for test vectors) + canonicalRequest, + stringToSign, + signature, + signedHeaders, + authorizationHeader, + + -- * Encoding helpers + sha256Hex, + uriEncode, + formatAmzDate, +) where + +import Array (Array) +import Array qualified +import Basics +import Bytes (Bytes) +import Bytes qualified +import Char (Char) +import Char qualified +import Crypto qualified +import Data.Word (Word8) +import DateTime (DateTime) +import DateTime qualified +import LinkedList qualified +import Maybe qualified +import Text (Text) +import Text qualified + + +-- | Static access credentials. The secret never leaves this module unhashed. +data Credentials = Credentials + { accessKeyId :: Text, + secretAccessKey :: Text, + region :: Text + } + + +-- | The parts of an HTTP request that SigV4 covers. Headers must already +-- include @host@, @x-amz-date@ and @x-amz-content-sha256@; this module does +-- not add them, so the caller controls exactly what goes on the wire. +data CanonicalRequest = CanonicalRequest + { method :: Text, + -- | Absolute path, already URI-encoded per segment ("/bucket/a%20b"). + path :: Text, + -- | Query pairs, raw (unencoded) names and values. + query :: Array (Text, Text), + -- | Header names and values, raw. Names are lower-cased here. + headers :: Array (Text, Text), + -- | Lowercase hex SHA-256 of the payload. + payloadHash :: Text + } + + +-- | Step 1 of the AWS signing process. +canonicalRequest :: CanonicalRequest -> Text +canonicalRequest request = do + let canonicalQuery = + request.query + |> Array.map (\(name, value) -> (uriEncode name, uriEncode value)) + |> sortPairs + |> Array.map (\(name, value) -> [fmt|#{name}=#{value}|]) + |> Text.joinWith "&" + let canonicalHeaders = + normalizedHeaders request.headers + |> Array.map (\(name, value) -> [fmt|#{name}:#{value}\n|]) + |> Text.concat + let signed = signedHeaders request.headers + let method = request.method + let path = request.path + let payloadHash = request.payloadHash + [fmt|#{method}\n#{path}\n#{canonicalQuery}\n#{canonicalHeaders}\n#{signed}\n#{payloadHash}|] + + +-- | Semicolon-joined, sorted, lowercase header names. +signedHeaders :: Array (Text, Text) -> Text +signedHeaders headers = + normalizedHeaders headers + |> Array.map (\(name, _) -> name) + |> Text.joinWith ";" + + +-- | Step 2: the string to sign for the @s3@ service at the given instant. +stringToSign :: Credentials -> DateTime -> CanonicalRequest -> Text +stringToSign credentials instant request = do + let amzDate = formatAmzDate instant + let scope = credentialScope credentials instant + let hashedRequest = canonicalRequest request |> Text.toBytes |> sha256Hex + [fmt|AWS4-HMAC-SHA256\n#{amzDate}\n#{scope}\n#{hashedRequest}|] + + +-- | Step 3: lowercase hex signature over the string to sign. +signature :: Credentials -> DateTime -> CanonicalRequest -> Text +signature credentials instant request = do + let secret = credentials.secretAccessKey + let dateKey = hmac (Text.toBytes [fmt|AWS4#{secret}|]) (Text.toBytes (dateStamp instant)) + let regionKey = hmac dateKey (Text.toBytes credentials.region) + let serviceKey = hmac regionKey (Text.toBytes "s3") + let signingKey = hmac serviceKey (Text.toBytes "aws4_request") + hmac signingKey (stringToSign credentials instant request |> Text.toBytes) + |> hex + + +-- | Step 4: the complete @Authorization@ header value. +authorizationHeader :: Credentials -> DateTime -> CanonicalRequest -> Text +authorizationHeader credentials instant request = do + let scope = credentialScope credentials instant + let signed = signedHeaders request.headers + let sig = signature credentials instant request + let accessKey = credentials.accessKeyId + [fmt|AWS4-HMAC-SHA256 Credential=#{accessKey}/#{scope}, SignedHeaders=#{signed}, Signature=#{sig}|] + + +-- | @YYYYMMDD'T'HHMMSS'Z'@ in UTC, the @x-amz-date@ wire format. +formatAmzDate :: DateTime -> Text +formatAmzDate instant = do + let (year, month, day, hour, minute, second) = civilFromEpoch (DateTime.toEpochSeconds instant) + let two value = Text.fromInt value |> Text.padLeft 2 '0' + let yearText = Text.fromInt year + let monthText = two month + let dayText = two day + let hourText = two hour + let minuteText = two minute + let secondText = two second + [fmt|#{yearText}#{monthText}#{dayText}T#{hourText}#{minuteText}#{secondText}Z|] + + +-- | Lowercase hex SHA-256 of raw bytes (also the hash of the empty payload). +-- +-- >>> Text.toBytes "" |> sha256Hex +-- "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +sha256Hex :: Bytes -> Text +sha256Hex bytes = + bytes + |> Crypto.sha256 + |> Crypto.toHex + + +-- | AWS URI encoding: unreserved characters pass through, everything else +-- (including @/@) becomes uppercase percent escapes of its UTF-8 bytes. +uriEncode :: Text -> Text +uriEncode text = + text + |> Text.toArray + |> Array.map encodeChar + |> Text.concat + + +-- ========================================================================== +-- Internals +-- ========================================================================== + +-- | The @date/region/s3/aws4_request@ scope string. +credentialScope :: Credentials -> DateTime -> Text +credentialScope credentials instant = do + let date = dateStamp instant + let region = credentials.region + [fmt|#{date}/#{region}/s3/aws4_request|] + + +-- | UTC date as @YYYYMMDD@. +dateStamp :: DateTime -> Text +dateStamp instant = formatAmzDate instant |> Text.left 8 + + +-- | Lowercase names, trimmed values with internal whitespace collapsed, sorted by name. +normalizedHeaders :: Array (Text, Text) -> Array (Text, Text) +normalizedHeaders headers = + headers + |> Array.map (\(name, value) -> (Text.toLower name, collapseSpaces value)) + |> sortPairs + + +-- | Collapse runs of spaces to one, as canonical header values require. +collapseSpaces :: Text -> Text +collapseSpaces value = Text.words value |> Text.joinWith " " + + +-- | Sort pairs by key (byte order), the canonical header and query order. +sortPairs :: Array (Text, Text) -> Array (Text, Text) +sortPairs pairs = + pairs + |> Array.toLinkedList + |> LinkedList.sort + |> Array.fromLinkedList + + +-- | URI-encode one character per the SigV4 rules (unreserved characters pass through). +encodeChar :: Char -> Text +encodeChar char = + if isUnreserved char + then Text.fromChar char + else + Text.fromChar char + |> Text.toBytes + |> Bytes.unpack + |> LinkedList.map (\byte -> "%" |> Text.append (byteHex byte)) + |> Array.fromLinkedList + |> Text.concat + + +-- | RFC 3986 unreserved: letters, digits, @-@, @_@, @.@, @~@. +isUnreserved :: Char -> Bool +isUnreserved char = + (Char.isAlphaNum char && Char.toCode char < 128) || char == '-' || char == '_' || char == '.' || char == '~' + + +-- | Two uppercase hex digits for one byte, as percent-encoding requires. +byteHex :: Word8 -> Text +byteHex byte = do + let value = fromIntegral byte :: Int + let digit nibble = "0123456789ABCDEF" |> Text.toArray |> Array.get nibble |> Maybe.withDefault '0' + Text.fromArray [digit (value // 16), digit (modBy 16 value)] + + +-- | Lowercase hex via 'Crypto.toHex'. +hex :: Bytes -> Text +hex = Crypto.toHex + + +-- | Raw HMAC-SHA256 via 'Crypto.hmacSha256'; chained to derive the signing key. +hmac :: Bytes -> Bytes -> Bytes +hmac = Crypto.hmacSha256 + + +-- | Proleptic Gregorian civil time from Unix seconds (Howard Hinnant's +-- days-from-civil inverse). Avoids a wall-clock formatting dependency so the +-- signing steps stay pure and vector-testable. +civilFromEpoch :: Int64 -> (Int, Int, Int, Int, Int, Int) +civilFromEpoch epoch = do + let total = fromIntegral epoch :: Int + let days = total // 86400 + let secondsOfDay = modBy 86400 total + let z = days + 719468 + let era = (if z >= 0 then z else z - 146096) // 146097 + let dayOfEra = z - era * 146097 + let yearOfEra = (dayOfEra - dayOfEra // 1460 + dayOfEra // 36524 - dayOfEra // 146096) // 365 + let dayOfYear = dayOfEra - (365 * yearOfEra + yearOfEra // 4 - yearOfEra // 100) + let monthIndex = (5 * dayOfYear + 2) // 153 + let day = dayOfYear - (153 * monthIndex + 2) // 5 + 1 + let month = if monthIndex < 10 then monthIndex + 3 else monthIndex - 9 + let year = yearOfEra + era * 400 + (if month <= 2 then 1 else 0) + (year, month, day, secondsOfDay // 3600, modBy 60 (secondsOfDay // 60), modBy 60 secondsOfDay) diff --git a/core/test-service/Main.hs b/core/test-service/Main.hs index f00a9f639..563490c64 100644 --- a/core/test-service/Main.hs +++ b/core/test-service/Main.hs @@ -14,6 +14,8 @@ import Service.EventStore.Postgres.SubscriptionStoreSpec qualified import Service.EventStore.Postgres.NotificationsSpec qualified import Service.EventStore.PostgresSpec qualified import Service.FileUpload.BlobStore.LocalSpec qualified +import Service.FileUpload.BlobStore.S3.SigV4Spec qualified +import Service.FileUpload.BlobStore.S3Spec qualified import Service.FileUpload.ContentDedupSpec qualified import Service.FileUpload.CoreSpec qualified import Service.FileUpload.DownloadSpec qualified @@ -69,6 +71,8 @@ main = Hspec.hspec do Hspec.describe "Service.EventStore.Postgres" Service.EventStore.PostgresSpec.spec Hspec.describe "Service.EventStore.Postgres.Notifications" Service.EventStore.Postgres.NotificationsSpec.spec Hspec.describe "Service.FileUpload.BlobStore.Local" Service.FileUpload.BlobStore.LocalSpec.spec + Hspec.describe "Service.FileUpload.BlobStore.S3" Service.FileUpload.BlobStore.S3Spec.spec + Hspec.describe "Service.FileUpload.BlobStore.S3.SigV4" Service.FileUpload.BlobStore.S3.SigV4Spec.spec Hspec.describe "Service.FileUpload.ContentDedup" Service.FileUpload.ContentDedupSpec.spec Hspec.describe "Service.FileUpload.Core" Service.FileUpload.CoreSpec.spec Hspec.describe "Service.FileUpload.Download" Service.FileUpload.DownloadSpec.spec diff --git a/core/test/CryptoSpec.hs b/core/test/CryptoSpec.hs index c4168d131..f4c4635a0 100644 --- a/core/test/CryptoSpec.hs +++ b/core/test/CryptoSpec.hs @@ -1,5 +1,6 @@ module CryptoSpec where +import Bytes qualified import Core import Crypto (HmacKey) import Crypto qualified @@ -137,3 +138,43 @@ spec = parallel do let firstSignature = message |> Crypto.signWith firstKey let secondSignature = message |> Crypto.signWith secondKey firstSignature |> shouldNotBe secondSignature + + -- ========================================================================== + -- Primitives (known-answer vectors) + -- ========================================================================== + describe "sha256" do + it "hashes the empty input to the FIPS 180-4 vector" \_ -> do + let digest = Text.toBytes "" |> Crypto.sha256 |> Crypto.toHex + digest |> shouldBe "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + it "hashes abc to the FIPS 180-4 vector" \_ -> do + let digest = Text.toBytes "abc" |> Crypto.sha256 |> Crypto.toHex + digest |> shouldBe "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + + describe "hmacSha256" do + it "matches RFC 4231 test case 2 (key Jefe)" \_ -> do + let mac = + Text.toBytes "what do ya want for nothing?" + |> Crypto.hmacSha256 (Text.toBytes "Jefe") + |> Crypto.toHex + mac |> shouldBe "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843" + + it "accepts keys shorter than 32 bytes, unlike signWith" \_ -> do + let mac = Text.toBytes "m" |> Crypto.hmacSha256 (Text.toBytes "k") + Bytes.length mac |> shouldBe 32 + + it "agrees with signWith for a valid HmacKey" \_ -> do + let message = Text.toBytes "same message" + let viaSign = message |> Crypto.signWith knownKey + let viaRaw = + message + |> Crypto.hmacSha256 (Text.toBytes "0123456789abcdef0123456789abcdef") + |> Crypto.toHex + viaRaw |> shouldBe viaSign + + describe "toHex" do + it "encodes two lowercase characters per byte" \_ -> do + Crypto.toHex (Bytes.pack [0, 15, 16, 255]) |> shouldBe "000f10ff" + + it "encodes the empty input to the empty text" \_ -> do + Crypto.toHex (Text.toBytes "") |> shouldBe "" diff --git a/core/test/Http/ClientRawSpec.hs b/core/test/Http/ClientRawSpec.hs index 4f0dd2038..9ed9d57bc 100644 --- a/core/test/Http/ClientRawSpec.hs +++ b/core/test/Http/ClientRawSpec.hs @@ -7,6 +7,7 @@ import Core import Http.Client qualified as Http import Http.Client.Internal qualified as HttpInternal import Json qualified +import Maybe qualified import Task qualified import Test import Text qualified @@ -114,11 +115,116 @@ spec = do Err (Http.ResponseTooLarge _) -> fail "Unexpected ResponseTooLarge from Internal.getRaw" Ok _ -> fail "Expected error for unreachable host" + describe "Http.Client.Internal.sendRaw" do + it "sendRaw PUT delivers method, raw body and custom header" \_ -> do + withMockServer mockEchoApp \testPort -> do + response <- + Http.request + |> Http.withUrl [fmt|http://localhost:#{testPort}/bucket/key|] + |> Http.addHeader "X-Probe" "sigv4" + |> (\req -> HttpInternal.sendRaw Http.Put req (Text.toBytes "payload-bytes")) + |> Task.asResult + case response of + Ok resp -> do + resp.statusCode |> shouldBe 200 + headerValue "X-Echo-Method" resp |> shouldBe (Just "PUT") + headerValue "X-Echo-Probe" resp |> shouldBe (Just "sigv4") + resp.body |> Text.fromBytes |> shouldBe "payload-bytes" + Err err -> fail [fmt|Expected Ok, got Err: #{show err}|] + + it "sendRaw HEAD returns headers and an empty body without throwing" \_ -> do + withMockServer mockEchoApp \testPort -> do + response <- + Http.request + |> Http.withUrl [fmt|http://localhost:#{testPort}/bucket/key|] + |> (\req -> HttpInternal.sendRaw Http.Head req Bytes.empty) + |> Task.asResult + case response of + Ok resp -> do + resp.statusCode |> shouldBe 200 + headerValue "X-Echo-Method" resp |> shouldBe (Just "HEAD") + resp.body |> Bytes.length |> shouldBe 0 + Err err -> fail [fmt|Expected Ok, got Err: #{show err}|] + + it "sendRaw DELETE surfaces a 404 as a status code, not an error" \_ -> do + withMockServer (mockStatusApp GhcHTTP.status404) \testPort -> do + response <- + Http.request + |> Http.withUrl [fmt|http://localhost:#{testPort}/bucket/missing|] + |> (\req -> HttpInternal.sendRaw Http.Delete req Bytes.empty) + |> Task.asResult + case response of + Ok resp -> resp.statusCode |> shouldBe 404 + Err err -> fail [fmt|Expected Ok, got Err: #{show err}|] + + it "sendRaw PUT surfaces a 403 as a status code, not an error" \_ -> do + withMockServer (mockStatusApp GhcHTTP.status403) \testPort -> do + response <- + Http.request + |> Http.withUrl [fmt|http://localhost:#{testPort}/bucket/key|] + |> (\req -> HttpInternal.sendRaw Http.Put req (Text.toBytes "x")) + |> Task.asResult + case response of + Ok resp -> resp.statusCode |> shouldBe 403 + Err err -> fail [fmt|Expected Ok, got Err: #{show err}|] + + it "sendRaw enforces maxResponseBytes on the response body" \_ -> do + withMockServer mockEchoApp \testPort -> do + response <- + Http.request + |> Http.withUrl [fmt|http://localhost:#{testPort}/bucket/key|] + |> Http.withMaxResponseSize 4 + |> (\req -> HttpInternal.sendRaw Http.Put req (Text.toBytes "more-than-four")) + |> Task.asResult + case response of + Err (Http.ResponseTooLarge limit) -> limit |> shouldBe 4 + Err err -> fail [fmt|Expected ResponseTooLarge, got: #{show err}|] + Ok _ -> fail "Expected ResponseTooLarge, got Ok" + + describe "Http.Client.sendSecure" do + it "sendSecure rejects a plain http:// URL with InvalidUrl" \_ -> do + response <- + Http.request + |> Http.withUrl "http://localhost:59999/bucket/key?token=secret" + |> (\req -> Http.sendSecure Http.Put req Bytes.empty) + |> Task.asResult + case response of + Err (Http.InvalidUrl sanitized) -> Text.contains "secret" sanitized |> shouldBe False + Err err -> fail [fmt|Expected InvalidUrl, got: #{show err}|] + Ok _ -> fail "Expected InvalidUrl, got Ok" + + it "methodName spells every wire token" \_ -> do + [Http.Get, Http.Head, Http.Post, Http.Put, Http.Patch, Http.Delete] + |> Array.map Http.methodName + |> shouldBe ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"] + -- ============================================================================ -- Test Helpers -- ============================================================================ +-- | Run a WAI application on a free loopback port for the duration of the test. +withMockServer :: GhcWai.Application -> (Int -> Task Text Unit) -> Task Text Unit +withMockServer app runTest = do + testPort <- getFreePort + serverThread <- GhcConcurrent.forkIO (GhcWarp.run testPort app) |> Task.fromIO + GhcConcurrent.threadDelay 50000 |> Task.fromIO + let cleanup = GhcConcurrent.killThread serverThread |> Task.fromIO + runTest testPort |> Task.finally cleanup + + +-- | Look up a response header case-insensitively. +headerValue :: Text -> Http.Response Bytes -> Maybe Text +headerValue name response = + response.headers + |> Array.find (\(headerName, _) -> Text.toLower headerName == Text.toLower name) + |> Maybe.map (\(_, value) -> value) + + +-- ============================================================================ +-- Test Helpers (ports) +-- ============================================================================ + -- | Allocate a free port dynamically to avoid collisions in parallel test runs. -- Binds to port 0 (OS assigns free port), reads the assigned port, then closes the socket. getFreePort :: Task _ Int @@ -178,6 +284,34 @@ mock200App _request respond = do ) +-- | Mock app that echoes the request method, the X-Probe header and the raw body. +mockEchoApp :: GhcWai.Application +mockEchoApp request respond = do + body <- GhcWai.strictRequestBody request + let probeHeader = + GhcWai.requestHeaders request + |> Array.fromLinkedList + |> Array.find (\(name, _) -> name == "X-Probe") + let probe = case probeHeader of + Nothing -> "" + Just (_, value) -> value + respond + ( GhcWai.responseLBS + GhcHTTP.status200 + [ (GhcHTTP.hContentType, "application/octet-stream"), + ("X-Echo-Method", GhcWai.requestMethod request), + ("X-Echo-Probe", probe) + ] + body + ) + + +-- | Mock app that answers every request with the given status and an empty body. +mockStatusApp :: GhcHTTP.Status -> GhcWai.Application +mockStatusApp status _request respond = + respond (GhcWai.responseLBS status [] GhcLBS.empty) + + -- | Mock app that returns valid JSON mockJsonApp :: GhcWai.Application mockJsonApp _request respond = do diff --git a/core/test/Service/FileUpload/BlobStore/S3/FakeS3.hs b/core/test/Service/FileUpload/BlobStore/S3/FakeS3.hs new file mode 100644 index 000000000..0d8754b7f --- /dev/null +++ b/core/test/Service/FileUpload/BlobStore/S3/FakeS3.hs @@ -0,0 +1,214 @@ +-- | In-process fake of the S3 object API, just enough for the S3 BlobStore +-- facade: PUT / GET / HEAD / DELETE on @/bucket/key@ with ETag headers and +-- 200 / 204 / 404 / 403 answers. It speaks the S3 wire protocol and nothing +-- else. +-- +-- Boundary (deliberate, per maintainer request): this module depends only on +-- wai/warp and Core. It imports nothing from the FileUpload service or from +-- other test infrastructure, and only the S3 specs import it, so it can move +-- with the S3 backend if the package is split. +-- +-- The signature is not verified cryptographically here (SigV4Spec covers the +-- algorithm against the AWS vectors); the fake checks that every request +-- carries the SigV4 headers and the configured access key, and can be +-- switched to reject everything with 403. +module Service.FileUpload.BlobStore.S3.FakeS3 ( + FakeS3 (..), + Mode (..), + withFakeS3, + objects, +) where + +import Array (Array) +import Array qualified +import Basics +import Bytes (Bytes) +import Bytes qualified +import ConcurrentMap (ConcurrentMap) +import ConcurrentMap qualified +import ConcurrentVar (ConcurrentVar) +import ConcurrentVar qualified +import Maybe (Maybe (..)) +import Maybe qualified +import Task (Task) +import Task qualified +import Text (Text) +import Text qualified + +-- Transport only: WAI/Warp are the HTTP server core already depends on. +import Control.Concurrent qualified as GhcConcurrent +import Data.ByteString.Lazy qualified as GhcLBS +import Data.CaseInsensitive qualified as GhcCI +import Network.HTTP.Types qualified as GhcHTTP +import Network.Socket qualified as GhcSocket +import Network.Wai qualified as GhcWai +import Network.Wai.Handler.Warp qualified as GhcWarp + + +-- | How the fake answers signed requests. +data Mode + = Accept + -- ^ Normal S3 behaviour. + | Forbid + -- ^ Every request answers 403 (wrong credentials / policy denial). + deriving (Eq, Show) + + +data FakeS3 = FakeS3 + { port :: Int + , endpoint :: Text + -- ^ @http://127.0.0.1:PORT@ + , bucket :: Text + , accessKeyId :: Text + , mode :: ConcurrentVar Mode + , store :: ConcurrentMap Text Bytes + -- ^ Objects by full path (@/bucket/key@) as received on the wire. + , requests :: ConcurrentVar (Array (Text, Text)) + -- ^ (method, path) of every request seen, oldest first. + } + + +-- | Run a fake S3 on a free loopback port for the duration of the action. +withFakeS3 :: Text -> Text -> (FakeS3 -> Task Text Unit) -> Task Text Unit +withFakeS3 bucket accessKeyId action = do + port <- getFreePort + mode <- ConcurrentVar.containing Accept + store <- ConcurrentMap.new + requests <- ConcurrentVar.containing Array.empty + let fake = + FakeS3 + { port + , endpoint = [fmt|http://127.0.0.1:#{port}|] + , bucket + , accessKeyId + , mode + , store + , requests + } + serverThread <- GhcConcurrent.forkIO (GhcWarp.run port (application fake)) |> Task.fromIO + GhcConcurrent.threadDelay 50000 |> Task.fromIO + let cleanup = GhcConcurrent.killThread serverThread |> Task.fromIO + action fake |> Task.finally cleanup + + +-- | Current object paths, for assertions. +objects :: FakeS3 -> Task Text (Array Text) +objects fake = ConcurrentMap.keys fake.store + + +-- ========================================================================== +-- WAI application +-- ========================================================================== + +application :: FakeS3 -> GhcWai.Application +application fake request respond = do + let method = GhcWai.requestMethod request |> Bytes.fromLegacy |> Text.fromBytes + let path = GhcWai.rawPathInfo request |> Bytes.fromLegacy |> Text.fromBytes + body <- GhcWai.strictRequestBody request + let headers = + GhcWai.requestHeaders request + |> Array.fromLinkedList + |> Array.map (\(name, value) -> (headerNameText name, value |> Bytes.fromLegacy |> Text.fromBytes)) + response <- + handle fake method path headers (body |> Bytes.fromLazyLegacy) + |> Task.runOrPanic + respond response + + +headerNameText :: GhcHTTP.HeaderName -> Text +headerNameText name = GhcCI.original name |> Bytes.fromLegacy |> Text.fromBytes |> Text.toLower + + +handle :: FakeS3 -> Text -> Text -> Array (Text, Text) -> Bytes -> Task Text GhcWai.Response +handle fake method path headers body = do + ConcurrentVar.modify (Array.push (method, path)) fake.requests + mode <- ConcurrentVar.peek fake.mode + let bucket = fake.bucket + let bucketPrefix = [fmt|/#{bucket}/|] :: Text + if not (isSigned fake headers) + then Task.yield (xmlError GhcHTTP.status403 "AccessDenied") + else + if mode == Forbid + then Task.yield (xmlError GhcHTTP.status403 "AccessDenied") + else + if not (Text.startsWith bucketPrefix path) + then Task.yield (xmlError GhcHTTP.status404 "NoSuchBucket") + else dispatch fake method path body + + +-- | SigV4 header-based auth always carries these three; the Credential must +-- name the configured access key. +isSigned :: FakeS3 -> Array (Text, Text) -> Bool +isSigned fake headers = do + let header name = headers |> Array.find (\(key, _) -> key == name) |> Maybe.map (\(_, value) -> value) + let accessKey = fake.accessKeyId + let expectedCredential = [fmt|AWS4-HMAC-SHA256 Credential=#{accessKey}/|] :: Text + let present name = case header name of + Nothing -> False + Just _ -> True + let authorized = case header "authorization" of + Nothing -> False + Just value -> Text.startsWith expectedCredential value && Text.contains ", Signature=" value + authorized && present "x-amz-date" && present "x-amz-content-sha256" + + +dispatch :: FakeS3 -> Text -> Text -> Bytes -> Task Text GhcWai.Response +dispatch fake method path body = + case method of + "PUT" -> do + ConcurrentMap.set path body fake.store + Task.yield (plain GhcHTTP.status200 [("ETag", etagOf body)] Bytes.empty) + "GET" -> do + stored <- ConcurrentMap.get path fake.store + case stored of + Nothing -> Task.yield (xmlError GhcHTTP.status404 "NoSuchKey") + Just bytes -> Task.yield (plain GhcHTTP.status200 [("ETag", etagOf bytes)] bytes) + "HEAD" -> do + stored <- ConcurrentMap.get path fake.store + case stored of + Nothing -> Task.yield (plain GhcHTTP.status404 [] Bytes.empty) + Just bytes -> Task.yield (plain GhcHTTP.status200 [("ETag", etagOf bytes)] Bytes.empty) + "DELETE" -> do + ConcurrentMap.remove path fake.store + Task.yield (plain GhcHTTP.status204 [] Bytes.empty) + _ -> Task.yield (xmlError GhcHTTP.status405 "MethodNotAllowed") + + +-- | Opaque but deterministic per content, like a real single-part ETag. +etagOf :: Bytes -> Text +etagOf bytes = do + let size = Bytes.length bytes + let tag = bytes |> Bytes.toBase64 |> Text.fromBytes |> Text.left 16 + [fmt|"#{tag}-#{size}"|] + + +plain :: GhcHTTP.Status -> Array (Text, Text) -> Bytes -> GhcWai.Response +plain status headers body = + GhcWai.responseLBS + status + (headers |> Array.map (\(name, value) -> (name |> Text.toBytes |> Bytes.unwrap |> GhcCI.mk, value |> Text.toBytes |> Bytes.unwrap)) |> Array.toLinkedList) + (Bytes.toLazyLegacy body) + + +xmlError :: GhcHTTP.Status -> Text -> GhcWai.Response +xmlError status code = + GhcWai.responseLBS + status + [(GhcHTTP.hContentType, "application/xml")] + (GhcLBS.fromStrict ([fmt|#{code}|] |> Text.toBytes |> Bytes.unwrap)) + + +-- | Bind port 0 on loopback, read the assigned port, release it. +getFreePort :: Task Text Int +getFreePort = + Task.fromIO do + let hints = GhcSocket.defaultHints {GhcSocket.addrSocketType = GhcSocket.Stream} + addrInfos <- GhcSocket.getAddrInfo (Just hints) (Just "127.0.0.1") (Just "0") + case addrInfos of + [] -> pure 19900 -- HOOK-ALLOW: IO boundary of the fake server + (addr : _) -> do + sock <- GhcSocket.openSocket addr + GhcSocket.bind sock (GhcSocket.addrAddress addr) + port <- GhcSocket.socketPort sock + GhcSocket.close sock + pure (fromIntegral port) -- HOOK-ALLOW: IO boundary of the fake server diff --git a/core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs b/core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs new file mode 100644 index 000000000..5a50ca898 --- /dev/null +++ b/core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs @@ -0,0 +1,132 @@ +-- | SigV4 against the worked examples in the AWS S3 API reference +-- ("Authenticating Requests: Using the Authorization Header", examples +-- GET Object, PUT Object, GET Bucket Lifecycle, GET Bucket (list objects)). +-- All examples use access key AKIAIOSFODNN7EXAMPLE, the documented example +-- secret, region us-east-1 and the instant 2013-05-24T00:00:00Z. +module Service.FileUpload.BlobStore.S3.SigV4Spec where + +import Bytes qualified +import Core +import DateTime qualified +import Service.FileUpload.BlobStore.S3.SigV4 (CanonicalRequest (..), Credentials (..)) +import Service.FileUpload.BlobStore.S3.SigV4 qualified as SigV4 +import Test +import Text qualified + + +spec :: Spec Unit +spec = do + describe "Service.FileUpload.BlobStore.S3.SigV4" do + describe "AWS reference vectors" do + it "hashes the empty payload to the documented SHA-256" \_ -> do + SigV4.sha256Hex Bytes.empty |> shouldBe emptyPayloadHash + + it "formats x-amz-date as YYYYMMDD'T'HHMMSS'Z'" \_ -> do + SigV4.formatAmzDate exampleInstant |> shouldBe "20130524T000000Z" + + it "GET Object: canonical request hash, string to sign and signature match AWS" \_ -> do + let request = + CanonicalRequest + { method = "GET" + , path = "/test.txt" + , query = [] + , headers = + [ ("Host", "examplebucket.s3.amazonaws.com") + , ("Range", "bytes=0-9") + , ("x-amz-content-sha256", emptyPayloadHash) + , ("x-amz-date", "20130524T000000Z") + ] + , payloadHash = emptyPayloadHash + } + SigV4.canonicalRequest request |> Text.toBytes |> SigV4.sha256Hex + |> shouldBe "7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972" + SigV4.stringToSign exampleCredentials exampleInstant request + |> shouldBe "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972" + SigV4.signature exampleCredentials exampleInstant request + |> shouldBe "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + SigV4.authorizationHeader exampleCredentials exampleInstant request + |> shouldBe "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + + it "PUT Object: signs a non-empty payload with extra headers" \_ -> do + let payloadHash = "Welcome to Amazon S3." |> Text.toBytes |> SigV4.sha256Hex + payloadHash |> shouldBe "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" + let request = + CanonicalRequest + { method = "PUT" + , path = "/test%24file.text" + , query = [] + , headers = + [ ("Date", "Fri, 24 May 2013 00:00:00 GMT") + , ("Host", "examplebucket.s3.amazonaws.com") + , ("x-amz-content-sha256", payloadHash) + , ("x-amz-date", "20130524T000000Z") + , ("x-amz-storage-class", "REDUCED_REDUNDANCY") + ] + , payloadHash + } + SigV4.canonicalRequest request |> Text.toBytes |> SigV4.sha256Hex + |> shouldBe "9e0e90d9c76de8fa5b200d8c849cd5b8dc7a3be3951ddb7f6a76b4158342019d" + SigV4.signature exampleCredentials exampleInstant request + |> shouldBe "98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd" + + it "GET Bucket Lifecycle: encodes a valueless query parameter" \_ -> do + let request = bucketRequest [("lifecycle", "")] + SigV4.canonicalRequest request |> Text.toBytes |> SigV4.sha256Hex + |> shouldBe "9766c798316ff2757b517bc739a67f6213b4ab36dd5da2f94eaebf79c77395ca" + SigV4.signature exampleCredentials exampleInstant request + |> shouldBe "fea454ca298b7da1c68078a5d1bdbfbbe0d65c699e0f91ac7a200a0136783543" + + it "GET Bucket: sorts query parameters by name" \_ -> do + let request = bucketRequest [("prefix", "J"), ("max-keys", "2")] + SigV4.canonicalRequest request |> Text.toBytes |> SigV4.sha256Hex + |> shouldBe "df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7" + SigV4.signature exampleCredentials exampleInstant request + |> shouldBe "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" + + describe "uriEncode" do + it "keeps unreserved characters and percent-encodes the rest in uppercase" \_ -> do + SigV4.uriEncode "AZaz09-_.~" |> shouldBe "AZaz09-_.~" + SigV4.uriEncode "a b/c$d" |> shouldBe "a%20b%2Fc%24d" + SigV4.uriEncode "\228" |> shouldBe "%C3%A4" + + describe "signedHeaders" do + it "lower-cases and sorts header names" \_ -> do + SigV4.signedHeaders [("X-Amz-Date", "a"), ("Host", "b"), ("Content-Type", "c")] + |> shouldBe "content-type;host;x-amz-date" + + +-- ========================================================================== +-- Fixtures +-- ========================================================================== + +emptyPayloadHash :: Text +emptyPayloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + +exampleCredentials :: Credentials +exampleCredentials = + Credentials + { accessKeyId = "AKIAIOSFODNN7EXAMPLE" + , secretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + , region = "us-east-1" + } + + +-- | 2013-05-24T00:00:00Z +exampleInstant :: DateTime +exampleInstant = DateTime.fromEpochSeconds 1369353600 + + +bucketRequest :: Array (Text, Text) -> CanonicalRequest +bucketRequest query = + CanonicalRequest + { method = "GET" + , path = "/" + , query + , headers = + [ ("Host", "examplebucket.s3.amazonaws.com") + , ("x-amz-content-sha256", emptyPayloadHash) + , ("x-amz-date", "20130524T000000Z") + ] + , payloadHash = emptyPayloadHash + } diff --git a/core/test/Service/FileUpload/BlobStore/S3Spec.hs b/core/test/Service/FileUpload/BlobStore/S3Spec.hs new file mode 100644 index 000000000..d5e3dea7e --- /dev/null +++ b/core/test/Service/FileUpload/BlobStore/S3Spec.hs @@ -0,0 +1,195 @@ +module Service.FileUpload.BlobStore.S3Spec where + +import Array qualified +import Bytes qualified +import ConcurrentVar qualified +import Core +import Result qualified +import Service.FileUpload.BlobStore (BlobStore (..), BlobStoreError (..)) +import Service.FileUpload.BlobStore.S3 (S3Config (..), createBlobStore) +import Service.FileUpload.BlobStore.S3.FakeS3 (FakeS3 (..), Mode (..), withFakeS3) +import Service.FileUpload.BlobStore.S3.FakeS3 qualified as FakeS3 +import Service.FileUpload.Core (BlobKey (..)) +import Task qualified +import Test +import Text qualified + + +spec :: Spec Unit +spec = do + describe "Service.FileUpload.BlobStore.S3" do + describe "Basic Operations" do + it "store and retrieve roundtrips bytes through signed PUT and GET" \_ -> do + withS3BlobStore \fake blobStore -> do + let blobKey = BlobKey "test-blob-1" + let content = "Hello, S3!" |> Text.toBytes + blobStore.store blobKey content |> orFail + retrieved <- blobStore.retrieve blobKey |> orFail + retrieved |> shouldBe content + seen <- ConcurrentVar.peek fake.requests + seen |> shouldBe [("PUT", "/test-bucket/test-blob-1"), ("GET", "/test-bucket/test-blob-1")] + + it "store overwrites an existing blob" \_ -> do + withS3BlobStore \_ blobStore -> do + let blobKey = BlobKey "overwrite-me" + blobStore.store blobKey ("first" |> Text.toBytes) |> orFail + blobStore.store blobKey ("second" |> Text.toBytes) |> orFail + retrieved <- blobStore.retrieve blobKey |> orFail + retrieved |> shouldBe ("second" |> Text.toBytes) + + it "exists answers True after store and False for an unknown key" \_ -> do + withS3BlobStore \fake blobStore -> do + let blobKey = BlobKey "present" + blobStore.store blobKey ("x" |> Text.toBytes) |> orFail + present <- blobStore.exists blobKey |> orFail + present |> shouldBe True + missing <- blobStore.exists (BlobKey "absent") |> orFail + missing |> shouldBe False + seen <- ConcurrentVar.peek fake.requests + seen |> Array.get 1 |> shouldBe (Just ("HEAD", "/test-bucket/present")) + + it "delete removes the blob and is idempotent" \_ -> do + withS3BlobStore \fake blobStore -> do + let blobKey = BlobKey "delete-me" + blobStore.store blobKey ("bye" |> Text.toBytes) |> orFail + blobStore.delete blobKey |> orFail + blobStore.delete blobKey |> orFail + remaining <- FakeS3.objects fake + remaining |> shouldBe [] + + it "stores and retrieves an empty blob" \_ -> do + withS3BlobStore \_ blobStore -> do + let blobKey = BlobKey "empty" + blobStore.store blobKey Bytes.empty |> orFail + retrieved <- blobStore.retrieve blobKey |> orFail + retrieved |> shouldBe Bytes.empty + + describe "Error Mapping" do + it "retrieve of a missing key is NotFound" \_ -> do + withS3BlobStore \_ blobStore -> do + result <- blobStore.retrieve (BlobKey "nope") |> Task.asResult + result |> shouldBe (Err (NotFound (BlobKey "nope"))) + + it "a 403 from the store is a StorageError carrying only the status" \_ -> do + withS3BlobStore \fake blobStore -> do + ConcurrentVar.swap Forbid fake.mode |> discard + result <- blobStore.store (BlobKey "denied") ("x" |> Text.toBytes) |> Task.asResult + result |> shouldBe (Err (StorageError "S3 HTTP 403")) + existsResult <- blobStore.exists (BlobKey "denied") |> Task.asResult + existsResult |> shouldBe (Err (StorageError "S3 HTTP 403")) + + it "an unreachable endpoint is a StorageError without the URL" \_ -> do + blobStore <- + createBlobStore (validConfig "http://127.0.0.1:9") + |> Task.mapError (\e -> [fmt|createBlobStore failed: #{e}|]) + result <- blobStore.exists (BlobKey "any") |> Task.asResult + case result of + Err (StorageError message) -> Text.contains "127.0.0.1" message |> shouldBe False + other -> fail [fmt|expected StorageError, got #{show other}|] + + describe "Key Validation" do + it "encodes each key segment and keeps slashes as separators" \_ -> do + withS3BlobStore \fake blobStore -> do + blobStore.store (BlobKey "owner/a b$c") ("x" |> Text.toBytes) |> orFail + seen <- ConcurrentVar.peek fake.requests + seen |> shouldBe [("PUT", "/test-bucket/owner/a%20b%24c")] + + it "rejects dot segments and empty keys without sending a request" \_ -> do + withS3BlobStore \fake blobStore -> do + dotResult <- blobStore.store (BlobKey "a/../b") Bytes.empty |> Task.asResult + dotResult |> shouldBe (Err (InvalidBlobKey "S3 key must not contain empty or dot path segments")) + emptyResult <- blobStore.exists (BlobKey "") |> Task.asResult + emptyResult |> shouldBe (Err (InvalidBlobKey "S3 key must contain 1 to 1024 UTF-8 bytes")) + seen <- ConcurrentVar.peek fake.requests + seen |> shouldBe [] + + describe "Configuration" do + it "rejects an http endpoint that is not loopback" \_ -> do + result <- createBlobStore (validConfig "http://s3.example.com") |> Task.asResult + result |> Result.map (\_ -> ()) |> shouldBe (Err "S3 requires HTTPS outside literal loopback") + + it "rejects an endpoint carrying a path, query or credentials" \_ -> do + withPath <- createBlobStore (validConfig "https://s3.example.com/bucket") |> Task.asResult + withPath |> Result.map (\_ -> ()) |> shouldBe (Err "S3 endpoint must not contain credentials, path, query or fragment") + withUser <- createBlobStore (validConfig "https://user:pw@s3.example.com") |> Task.asResult + withUser |> Result.map (\_ -> ()) |> shouldBe (Err "S3 endpoint must not contain credentials, path, query or fragment") + + it "rejects empty fields and non-DNS bucket names" \_ -> do + emptyKey <- createBlobStore (configWith "" "test-bucket") |> Task.asResult + emptyKey |> Result.map (\_ -> ()) |> shouldBe (Err "S3 configuration fields must not be empty") + badBucket <- createBlobStore (configWith "AKIAIOSFODNN7EXAMPLE" "Bad_Bucket") |> Task.asResult + badBucket |> Result.map (\_ -> ()) |> shouldBe (Err "S3 bucket must be a DNS-style bucket name") + + it "rejects bucket names with an empty label or a hyphen at a label edge" \_ -> do + let invalidNames = ["a..b", ".ab", "ab.", "-ab", "ab-", "a.-b", "a-.b"] + outcomes <- invalidNames |> Task.mapArray bucketOutcome + outcomes |> shouldBe (invalidNames |> Array.map (\name -> (name, Err "S3 bucket must be a DNS-style bucket name"))) + + it "accepts dotted and hyphenated bucket names" \_ -> do + let validNames = ["my-bucket", "a-b.c-d", "logs.2026.v2", "0a.b9"] + outcomes <- validNames |> Task.mapArray bucketOutcome + outcomes |> shouldBe (validNames |> Array.map (\name -> (name, Ok ()))) + + it "enforces the 3 to 63 character bucket name bounds" \_ -> do + outcomes <- ["abc", Text.repeat 63 "a", "ab", Text.repeat 64 "a"] |> Task.mapArray bucketOutcome + outcomes + |> Array.map (\(_, outcome) -> outcome) + |> shouldBe [Ok (), Ok (), Err "S3 bucket must be a DNS-style bucket name", Err "S3 bucket must be a DNS-style bucket name"] + + it "accepts an https endpoint without contacting it" \_ -> do + result <- createBlobStore (validConfig "https://s3.example.com") |> Task.asResult + result |> Result.map (\_ -> ()) |> shouldBe (Ok ()) + + it "accepts loopback over plain http, including bracketed IPv6" \_ -> do + ipv4 <- createBlobStore (validConfig "http://127.0.0.1:9000/") |> Task.asResult + ipv4 |> Result.map (\_ -> ()) |> shouldBe (Ok ()) + ipv6 <- createBlobStore (validConfig "http://[::1]:9000") |> Task.asResult + ipv6 |> Result.map (\_ -> ()) |> shouldBe (Ok ()) + + +-- ========================================================================== +-- Test Infrastructure +-- ========================================================================== + +validConfig :: Text -> S3Config +validConfig endpoint = + S3Config + { endpoint + , bucket = "test-bucket" + , region = "us-east-1" + , accessKeyId = "AKIAIOSFODNN7EXAMPLE" + , secretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + + +configWith :: Text -> Text -> S3Config +configWith accessKeyId bucket = + S3Config + { endpoint = "https://s3.example.com" + , bucket + , region = "us-east-1" + , accessKeyId + , secretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + + +-- | Validation outcome of a config that differs from a valid one only in its bucket. +bucketOutcome :: Text -> Task Text (Text, Result Text Unit) +bucketOutcome bucket = do + outcome <- + createBlobStore (configWith (validConfig "https://s3.example.com").accessKeyId bucket) + |> Task.asResult + Task.yield (bucket, outcome |> Result.map (\_ -> ())) + + +withS3BlobStore :: (FakeS3 -> BlobStore -> Task Text Unit) -> Task Text Unit +withS3BlobStore action = + withFakeS3 "test-bucket" "AKIAIOSFODNN7EXAMPLE" \fake -> do + blobStore <- + createBlobStore (validConfig fake.endpoint) + |> Task.mapError (\e -> [fmt|createBlobStore failed: #{e}|]) + action fake blobStore + + +orFail :: Task BlobStoreError value -> Task Text value +orFail task = task |> Task.mapError (\e -> [fmt|blob store failed: #{show e}|]) diff --git a/docs/changes/016-http-client-byte-level-methods.md b/docs/changes/016-http-client-byte-level-methods.md new file mode 100644 index 000000000..fcb33518e --- /dev/null +++ b/docs/changes/016-http-client-byte-level-methods.md @@ -0,0 +1,84 @@ +# Change 016: Add byte-level HTTP methods (PUT/HEAD/DELETE…) to Http.Client + +Protocol clients inside nhcore need to talk to object stores and similar REST +services: send a raw byte body with PUT, probe with HEAD, DELETE, read the +status code and headers (e.g. `ETag`) of every response including 4xx, and +never follow redirects. Today `Http.Client` offers byte-level, non-throwing +responses only for GET (`getSecure`, `Http.Client.Internal.getRaw`); the other +verbs decode JSON and throw on non-2xx. This change generalizes the existing +GET path into one method-agnostic primitive so the upcoming S3 `BlobStore` +(change 017) can use the Core HTTP client instead of raw `http-client`, as the +maintainer requested ("extend `Http.Client`" rather than adding a transport). + +```yaml spec +issue: adhoc:http-client-byte-level-methods +kind: feature +touches: [http-client] +breaking: false +new-dependency: false +new-capability: false +new-extension-point: false +``` + +## Contract delta + +`getSecure` and `getRaw` keep their signatures and become `Get` specializations +of the new functions. `sendSecure` enforces `https://` exactly like +`getSecure`; `sendRaw` is the loopback-only twin exported from +`Http.Client.Internal`, mirroring the existing `getRaw` split. Both reuse the +existing request options (headers, timeout, `maxRedirects = 0` default, proxy +disabled, `maxResponseBytes` limit) and the pinned TLS 1.2+ manager. + +```diff signatures ++ Http.Client: data Method ++ Http.Client: Get :: Method ++ Http.Client: Head :: Method ++ Http.Client: Post :: Method ++ Http.Client: Put :: Method ++ Http.Client: Patch :: Method ++ Http.Client: Delete :: Method ++ Http.Client: methodName :: Method -> Text ++ Http.Client: sendSecure :: Method -> Request -> Bytes -> Task Error (Response Bytes) ++ Http.Client.Internal: sendRaw :: Method -> Request -> Bytes -> Task Error (Response Bytes) +``` + +## Criteria + +| ID | Behavior | Proving test | Level | Boundary | +|----|----------|--------------|-------|----------| +| C1 | A PUT delivers the wire method, the raw body and caller headers unchanged | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#sendRaw PUT delivers method, raw body and custom header` | integration | http:real | +| C2 | HEAD returns status and headers with an empty body and does not throw | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#sendRaw HEAD returns headers and an empty body without throwing` | integration | http:real | +| C3 | Non-2xx statuses (404, 403) are returned as `statusCode`, never as `Err` | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#sendRaw DELETE surfaces a 404 as a status code, not an error`
`hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#sendRaw PUT surfaces a 403 as a status code, not an error` | integration | http:real | +| C4 | The response-size limit applies to every method | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#sendRaw enforces maxResponseBytes on the response body` | integration | http:real | +| C5 | `sendSecure` rejects plain `http://` with a sanitized `InvalidUrl` | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#sendSecure rejects a plain http:// URL with InvalidUrl` | unit | none | +| C6 | `methodName` spells every wire token of the closed `Method` set | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#methodName spells every wire token` | unit | none | +| C7 | Existing GET behavior is unchanged: `getRaw` returns a non-2xx status as `statusCode` | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#returns Ok with statusCode 401 on 401 response` | integration | http:real | +| C8 | Existing GET behavior is unchanged: `getRaw` reports a refused connection as `Err` | `hspec:nhcore-test-core:core/test/Http/ClientRawSpec.hs#returns Err on network error` | unit | none | + +C1–C4 and C7 send real HTTP over loopback to an in-process Warp server +started by `core/test/Http/ClientRawSpec.hs` (free port, no external +service), so a broken wire path fails them: they are `integration` / +`http:real`, and `docs/changes/test-surfaces.json` attests each exact selector +with its `Http.Client.Internal.sendRaw` or `Http.Client.Internal.getRaw` +fixture group. C5 and C6 stay `unit` / `none`: the scheme check fails before +any connection and `methodName` is pure. C8 also stays `unit` / `none`: it +only proves that a connection attempt to a closed loopback port becomes `Err`, +and no fixture server ever answers it. + +## User impact + +None breaking. `getSecure`/`getRaw` behave as before (GET, same errors, same +limits). New: `Http.Client.sendSecure` for HTTPS endpoints and +`Http.Client.Internal.sendRaw` for trusted loopback endpoints; both return the +status code instead of throwing on 4xx/5xx and never decode the body. Jess does +not need to change anything; integration authors gain PUT/HEAD/DELETE with raw +bodies without importing `Network.HTTP.Client`. + +## ADR + +Not required — no trigger (breaking / new-dependency / new-capability / +new-extension-point all false). Design note: the maintainer rule "if we can do +it ourselves without much complexity, do it ourselves; if too much, use a +Hackage package; for this one extend `Http.Client`" is satisfied by ≈90 lines +that reuse the existing `getSecure` machinery (`applyRequestOptions`, +`checkContentLengthHeaderLimit`, `readBodyFully`, `cachedSecureTlsManager`). diff --git a/docs/changes/017-s3-blob-store.md b/docs/changes/017-s3-blob-store.md new file mode 100644 index 000000000..94861f439 --- /dev/null +++ b/docs/changes/017-s3-blob-store.md @@ -0,0 +1,173 @@ +# Change 017: Add an S3-compatible BlobStore backend + +Applications that deploy outside a single machine cannot keep uploads in +`BlobStore/Local.hs` (ephemeral disks, several replicas). Issue #370 asks for +an S3 backend. Per the maintainer's review of the first attempt: the backend +must be a record "like `BlobStore/Local.hs`, configuration managed completely +by the blob store module", AWS-agnostic, built on the Core `Http.Client` +rather than a raw HTTP library, with credentials supplied by the application's +own `Config.hs`/`App.hs`, and tested against a mock HTTP server rather than a +real S3 service. This change delivers exactly that facade: `S3Config` plus +`createBlobStore`, the four `BlobStore` operations, hand-written SigV4 (no new +dependency), one request per operation, no retries, no redirects. + +```yaml spec +issue: issue#370 +kind: feature +touches: [file-upload] +breaking: false +new-dependency: false +new-capability: false +new-extension-point: false +``` + +## Contract delta + +Depends on change 016 (`Http.Client.sendSecure` / `Http.Client.Internal.sendRaw`) +and change 021 (`Crypto.sha256`, `Crypto.hmacSha256`, `Crypto.toHex`); this PR +is stacked on them, so `SigV4` imports nothing from `crypton` or +`Data.ByteArray` and never unwraps `Bytes`. `SigV4` is exported as its own module so the signing +steps are unit-testable against the AWS reference vectors; applications are +expected to use only `S3Config` and `createBlobStore`. + +```diff signatures ++ Service.FileUpload.BlobStore.S3: data S3Config ++ Service.FileUpload.BlobStore.S3: S3Config :: Text -> Text -> Text -> Text -> Text -> S3Config ++ Service.FileUpload.BlobStore.S3: [endpoint] :: S3Config -> Text ++ Service.FileUpload.BlobStore.S3: [bucket] :: S3Config -> Text ++ Service.FileUpload.BlobStore.S3: [region] :: S3Config -> Text ++ Service.FileUpload.BlobStore.S3: [accessKeyId] :: S3Config -> Text ++ Service.FileUpload.BlobStore.S3: [secretAccessKey] :: S3Config -> Text ++ Service.FileUpload.BlobStore.S3: createBlobStore :: S3Config -> Task Text BlobStore ++ Service.FileUpload.BlobStore.S3.SigV4: data Credentials ++ Service.FileUpload.BlobStore.S3.SigV4: Credentials :: Text -> Text -> Text -> Credentials ++ Service.FileUpload.BlobStore.S3.SigV4: [accessKeyId] :: Credentials -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: [secretAccessKey] :: Credentials -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: [region] :: Credentials -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: data CanonicalRequest ++ Service.FileUpload.BlobStore.S3.SigV4: CanonicalRequest :: Text -> Text -> Array (Text, Text) -> Array (Text, Text) -> Text -> CanonicalRequest ++ Service.FileUpload.BlobStore.S3.SigV4: [method] :: CanonicalRequest -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: [path] :: CanonicalRequest -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: [query] :: CanonicalRequest -> Array (Text, Text) ++ Service.FileUpload.BlobStore.S3.SigV4: [headers] :: CanonicalRequest -> Array (Text, Text) ++ Service.FileUpload.BlobStore.S3.SigV4: [payloadHash] :: CanonicalRequest -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: canonicalRequest :: CanonicalRequest -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: stringToSign :: Credentials -> DateTime -> CanonicalRequest -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: signature :: Credentials -> DateTime -> CanonicalRequest -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: signedHeaders :: Array (Text, Text) -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: authorizationHeader :: Credentials -> DateTime -> CanonicalRequest -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: sha256Hex :: Bytes -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: uriEncode :: Text -> Text ++ Service.FileUpload.BlobStore.S3.SigV4: formatAmzDate :: DateTime -> Text +``` + +### Decisions carried from the review + +- **Shape.** Same as `Local`: a config record and one constructor. No extra + operations outside the `BlobStore` facade. Conditional writes + (`If-None-Match`/`If-Match`) are **not** included: upstream dedup already + guarantees "store derived content once" through the content-hash lookup in + `Service.FileUpload.Web`, so the use case that motivated them is covered. +- **Provider-agnostic.** SigV4 + path-style addressing is the protocol shared + by AWS S3, Cloudflare R2, Backblaze B2, MinIO and RustFS. Azure Blob Storage + uses a different protocol (SharedKey REST) and is out of scope; it would be a + sibling module `BlobStore/Azure.hs` with the same `Config`/`createBlobStore` + shape. +- **Transport.** `Http.Client.sendSecure` (HTTPS, pinned TLS 1.2+, no + redirects, no proxy, bounded response) for every endpoint; + `Http.Client.Internal.sendRaw` only when the endpoint is a literal loopback + address (local fixtures). The module imports nothing from `Network.HTTP.*`. +- **Credentials.** Read by the application (its `Config.hs`), passed in + `S3Config` from `App.hs`. The backend never reads the environment. + `S3Config` has no `Show` instance; `Http.Client.Request` redacts headers, so + the `Authorization` value cannot reach logs. Per-file authorization is an + application concern (the app's own grant/ownership aggregate) and needs no + core convention or RFC; `Integration.Http.Auth` (JWT/OAuth) is unrelated. +- **Errors.** 404 on GET -> `NotFound`; 404 on HEAD -> `exists = False`; DELETE + accepts 204/200/404 (idempotent). Any other status -> `StorageError "S3 HTTP + "` (status only; S3 error bodies may echo the request). Transport + failures -> `StorageError` without the URL. +- **Testing.** No Docker, no live endpoint in CI. `SigV4Spec` reproduces the + four worked examples of the AWS S3 API reference (GET Object, PUT Object, + GET Bucket Lifecycle, GET Bucket) step by step. `S3Spec` drives the real + facade against `FakeS3`, an in-process WAI/Warp fake of the S3 object API on + a free loopback port. Verified before writing it: no reusable in-process HTTP + stub exists in `core/test*` today (only inline `GhcWarp.run` calls in + `Http/ClientRawSpec.hs`, `Http/ClientSpec.hs`, + `Auth/OAuth2/TokenRefreshSpec.hs`). **Boundary of `FakeS3`** (maintainer + request, so a future package split keeps it with the S3 backend): it lives + under `core/test/Service/FileUpload/BlobStore/S3/`, speaks only the S3 wire + protocol (PUT/GET/HEAD/DELETE, ETag, 200/204/404/403), depends only on + wai/warp/http-types/network (already in `common_cfg`) and Core, imports + nothing from the FileUpload service or other test infrastructure, and is + imported only by `S3Spec`. A run against a real S3-compatible server stays a + manual recipe (below), never a CI gate. + +### Manual check against a real S3-compatible server (not a gate) + +Disposable RustFS container on loopback, throw-away credentials, then point an +application (or a `cabal repl` session) at it: + +```sh +export RUSTFS_ACCESS_KEY=$(openssl rand -hex 12) RUSTFS_SECRET_KEY=$(openssl rand -hex 32) +docker run -d --name s3-manual --cap-drop ALL --security-opt no-new-privileges \ + -p 127.0.0.1::9000 --tmpfs /data --tmpfs /logs \ + -e RUSTFS_ACCESS_KEY -e RUSTFS_SECRET_KEY -e RUSTFS_CONSOLE_ENABLE=false \ + rustfs/rustfs:1.0.1 +ENDPOINT="http://$(docker port s3-manual 9000/tcp)" +# create the bucket once (any S3 client, e.g. mc or aws cli), then in ghci: +# S3.createBlobStore S3Config {endpoint = ENDPOINT, bucket = "manual", region = "us-east-1", +# accessKeyId = RUSTFS_ACCESS_KEY, secretAccessKey = RUSTFS_SECRET_KEY} +# and exercise store / retrieve / exists / delete. +docker rm -f s3-manual +``` + +The same steps work against MinIO (`minio/minio`), Cloudflare R2, Backblaze +B2 and AWS S3 with an HTTPS endpoint. + +## Criteria + +| ID | Behavior | Proving test | Level | Boundary | +|----|----------|--------------|-------|----------| +| C1 | SigV4 reproduces the AWS reference vectors (canonical request hash, string to sign, signature, Authorization header) for GET Object, PUT Object, GET Bucket Lifecycle and GET Bucket | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#GET Object: canonical request hash, string to sign and signature match AWS`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#PUT Object: signs a non-empty payload with extra headers`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#GET Bucket Lifecycle: encodes a valueless query parameter`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#GET Bucket: sorts query parameters by name` | unit | none | +| C2 | SigV4 helpers: empty-payload hash, `x-amz-date` format, AWS URI encoding, sorted lowercase signed headers | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#hashes the empty payload to the documented SHA-256`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#formats x-amz-date as YYYYMMDD'T'HHMMSS'Z'`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#keeps unreserved characters and percent-encodes the rest in uppercase`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3/SigV4Spec.hs#lower-cases and sorts header names` | unit | none | +| C3 | `store`/`retrieve` round-trip through signed PUT and GET against the fake, overwrite semantics, empty blob | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#store and retrieve roundtrips bytes through signed PUT and GET`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#store overwrites an existing blob`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#stores and retrieves an empty blob` | integration | http:real | +| C4 | `exists` via HEAD (200 -> True, 404 -> False); `delete` idempotent | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#exists answers True after store and False for an unknown key`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#delete removes the blob and is idempotent` | integration | http:real | +| C5 | Error mapping from fake responses: 404 -> `NotFound`, 403 -> `StorageError` with status only | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#retrieve of a missing key is NotFound`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#a 403 from the store is a StorageError carrying only the status` | integration | http:real | +| C6 | Keys: each segment URI-encoded with slashes kept; empty keys and dot segments rejected before any request | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#encodes each key segment and keeps slashes as separators`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#rejects dot segments and empty keys without sending a request` | integration | http:real | +| C7 | Configuration: HTTPS required except literal loopback (IPv4/IPv6), no path/query/userinfo in the endpoint, non-empty fields, DNS-style bucket of 3 to 63 lowercase letters, digits, hyphens and dots whose dot-separated labels are non-empty and start and end with a letter or digit; an invalid config is rejected by the constructor, so no store exists to send a request; a valid config does not contact the endpoint | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#rejects an http endpoint that is not loopback`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#rejects an endpoint carrying a path, query or credentials`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#rejects empty fields and non-DNS bucket names`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#rejects bucket names with an empty label or a hyphen at a label edge`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#accepts dotted and hyphenated bucket names`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#enforces the 3 to 63 character bucket name bounds`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#accepts an https endpoint without contacting it`
`hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#accepts loopback over plain http, including bracketed IPv6` | unit | none | +| C8 | An unreachable endpoint maps to `StorageError` without the URL | `hspec:nhcore-test-service:core/test/Service/FileUpload/BlobStore/S3Spec.hs#an unreachable endpoint is a StorageError without the URL` | unit | none | + +C3–C6 drive the real facade over loopback HTTP against `FakeS3`, the +in-process Warp listener started by `withFakeS3`, so they are `integration` / +`http:real`; `docs/changes/test-surfaces.json` attests each exact selector with +its `Service.FileUpload.BlobStore.S3` fixture group. C6's rejection example +uses the same fixture to prove that zero requests reach the wire. C1, C2 and +C7 stay `unit` / `none`: the signer is pure, and configuration validation +fails or succeeds inside `createBlobStore` without any network I/O. C8 also +stays `unit` / `none`: it only proves that a connection attempt to a closed +loopback port becomes a sanitized `StorageError`, and no fixture server ever +answers it. + +`file-upload` is a security-sensitive capability: after Gate 1, +`./dev spec-check --plan` routes this change to the local-only security design +review (ADR-0069) before implementation is finalized. + +## User impact + +None breaking. New opt-in backend: in `App.hs` build +`S3.createBlobStore S3Config {endpoint, bucket, region, accessKeyId, secretAccessKey}` +from values your own `Config.hs` reads, and pass the resulting `BlobStore` to +`Application.withFileUpload` exactly as with `Local.createBlobStore`. Existing +applications on the local backend are unaffected. Limits: single-request +uploads (no multipart), objects are read fully into memory (bounded at 256 MiB +per GET), the bucket must already exist, no retries (a failed write may or may +not have landed; the error says so). + +## ADR + +Not required — no trigger (breaking / new-dependency / new-capability / +new-extension-point all false). The SigV4 implementation reuses `crypton` and +`memory`, which `nhcore` already depends on for `Crypto`. Design rationale +(provider scope, transport choice, testing boundary) is recorded above so the +maintainer approves it at Gate 1 together with the contract. diff --git a/docs/changes/021-crypto-sha256-hmac-hex.md b/docs/changes/021-crypto-sha256-hmac-hex.md new file mode 100644 index 000000000..0fe8df255 --- /dev/null +++ b/docs/changes/021-crypto-sha256-hmac-hex.md @@ -0,0 +1,59 @@ +# Change 021: Add raw SHA-256, HMAC-SHA256 and hex primitives to Crypto + +Code inside nhcore that speaks a signed protocol needs the plain building +blocks: an unkeyed SHA-256 digest, a raw HMAC-SHA256 that can be chained +(one MAC result becomes the next key), and lowercase hex encoding. Today +`Crypto` exposes only the webhook-shaped `signWith`/`verifyWith` (hex text, +32-byte minimum `HmacKey`), so modules such as the S3 `BlobStore` signer +(change 017) and `FileUpload.Web`'s content hash import `crypton` and +`Data.ByteArray` directly and unwrap `Bytes`. The maintainer asked on #748 +to add the missing primitive to Core instead of unwrapping. This change does +that; `signWith` is re-expressed on top of the new functions so there is one +implementation. + +```yaml spec +issue: adhoc:crypto-sha256-hmac-hex +kind: feature +touches: [core-primitives] +breaking: false +new-dependency: false +new-capability: false +new-extension-point: false +``` + +## Contract delta + +Three pure, data-last functions on `Bytes`. `hmacSha256` deliberately takes an +arbitrary key (no 32-byte policy) because key-derivation chains need short +intermediate keys; `signWith`/`verifyWith` keep the policy and the +constant-time comparison and remain the right choice for a single webhook +secret. + +```diff signatures ++ Crypto: sha256 :: Bytes -> Bytes ++ Crypto: hmacSha256 :: Bytes -> Bytes -> Bytes ++ Crypto: toHex :: Bytes -> Text +``` + +## Criteria + +| ID | Criterion | Test | Level | Boundary | +|---|---|---|---|---| +| C1 | `sha256` matches the FIPS 180-4 vectors for `""` and `"abc"` | `hspec:nhcore-test-core:core/test/CryptoSpec.hs#hashes the empty input to the FIPS 180-4 vector`
`hspec:nhcore-test-core:core/test/CryptoSpec.hs#hashes abc to the FIPS 180-4 vector` | unit | none | +| C2 | `hmacSha256` matches RFC 4231 test case 2 | `hspec:nhcore-test-core:core/test/CryptoSpec.hs#matches RFC 4231 test case 2 (key Jefe)` | unit | none | +| C3 | `hmacSha256` accepts keys shorter than 32 bytes | `hspec:nhcore-test-core:core/test/CryptoSpec.hs#accepts keys shorter than 32 bytes, unlike signWith` | unit | none | +| C4 | `signWith` is unchanged: it equals `hmacSha256` + `toHex` for a valid key | `hspec:nhcore-test-core:core/test/CryptoSpec.hs#agrees with signWith for a valid HmacKey` | unit | none | +| C5 | `toHex` is lowercase, two characters per byte, empty for empty | `hspec:nhcore-test-core:core/test/CryptoSpec.hs#encodes two lowercase characters per byte`
`hspec:nhcore-test-core:core/test/CryptoSpec.hs#encodes the empty input to the empty text` | unit | none | + +## User impact + +None breaking. `signWith` and `verifyWith` produce exactly the same output as +before (C4). New: `Crypto.sha256`, `Crypto.hmacSha256`, `Crypto.toHex` for +protocol code that needs the raw primitives. Jess does not need to change +anything. + +## ADR + +Not required — no trigger. Follow-up (not in this change): +`Service.FileUpload.Web.computeContentHash` can be rewritten on +`Crypto.sha256 |> Crypto.toHex` and drop its own `crypton` imports. diff --git a/docs/changes/test-surfaces.json b/docs/changes/test-surfaces.json index 32fa31a81..b07ca77ca 100644 --- a/docs/changes/test-surfaces.json +++ b/docs/changes/test-surfaces.json @@ -208,6 +208,141 @@ "boundary": "postgres:real", "expected_examples": 1, "fixture": "Service.QueryObjectStore.Postgres" + }, + { + "kind": "hspec", + "suite": "nhcore-test-core", + "path": "core/test/Http/ClientRawSpec.hs", + "match": "sendRaw PUT delivers method, raw body and custom header", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Http.Client.Internal.sendRaw" + }, + { + "kind": "hspec", + "suite": "nhcore-test-core", + "path": "core/test/Http/ClientRawSpec.hs", + "match": "sendRaw HEAD returns headers and an empty body without throwing", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Http.Client.Internal.sendRaw" + }, + { + "kind": "hspec", + "suite": "nhcore-test-core", + "path": "core/test/Http/ClientRawSpec.hs", + "match": "sendRaw DELETE surfaces a 404 as a status code, not an error", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Http.Client.Internal.sendRaw" + }, + { + "kind": "hspec", + "suite": "nhcore-test-core", + "path": "core/test/Http/ClientRawSpec.hs", + "match": "sendRaw PUT surfaces a 403 as a status code, not an error", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Http.Client.Internal.sendRaw" + }, + { + "kind": "hspec", + "suite": "nhcore-test-core", + "path": "core/test/Http/ClientRawSpec.hs", + "match": "sendRaw enforces maxResponseBytes on the response body", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Http.Client.Internal.sendRaw" + }, + { + "kind": "hspec", + "suite": "nhcore-test-core", + "path": "core/test/Http/ClientRawSpec.hs", + "match": "returns Ok with statusCode 401 on 401 response", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Http.Client.Internal.getRaw" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "store and retrieve roundtrips bytes through signed PUT and GET", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "store overwrites an existing blob", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "stores and retrieves an empty blob", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "exists answers True after store and False for an unknown key", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "delete removes the blob and is idempotent", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "retrieve of a missing key is NotFound", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "a 403 from the store is a StorageError carrying only the status", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "encodes each key segment and keeps slashes as separators", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" + }, + { + "kind": "hspec", + "suite": "nhcore-test-service", + "path": "core/test/Service/FileUpload/BlobStore/S3Spec.hs", + "match": "rejects dot segments and empty keys without sending a request", + "boundary": "http:real", + "expected_examples": 1, + "fixture": "Service.FileUpload.BlobStore.S3" } ] } diff --git a/scripts/workflow-check b/scripts/workflow-check index 7d35bcd76..3b45157ed 100755 --- a/scripts/workflow-check +++ b/scripts/workflow-check @@ -44,6 +44,14 @@ Two contracts, both learned from real breakage: regression to a main/integration-only filter, or a dropped neo/** route, fails loudly. + 6. Criteria-runtime report (test.yml): integration criteria may name + locators in nhcore-test-core OR nhcore-test-service, but the + `--criteria-runtime --report` gate runs once, in test-service. test-core + must tee its direct output to a NON-hidden report under pipefail and + upload it fail-closed; test-service must need test-core, download that + artifact, and feed the gate the concatenation of both reports. A + service-only report silently fails every core locator. + Run: ./dev workflow-check CI: .github/workflows/checks.yml ./dev workflow-check --self-test (doctor/CI) """ @@ -429,6 +437,48 @@ def check_required_gate(name, text): return errs +CORE_REPORT = "nhcore-test-core.report" +CORE_REPORT_ARTIFACT = "nhcore-test-core-report" +SERVICE_REPORT = ".nhcore-test-service.report" + + +def check_criteria_report(name, text): + """Contract 6: the criteria-runtime gate reads core AND service reports.""" + errs = [] + E = errs.append + jobs = job_blocks(strip_comments(text)) + core, service = jobs.get("test-core", ""), jobs.get("test-service", "") + if is_hidden_path(CORE_REPORT): + E(f"{name}: core report `{CORE_REPORT}` is hidden; upload would be empty") + run_core = [s for s in _steps_of(core) if "cabal test nhcore-test-core" in s] + if not run_core or "set -o pipefail" not in run_core[0] \ + or f"tee {CORE_REPORT}" not in run_core[0] \ + or "--test-show-details=direct" not in run_core[0]: + E(f"{name}: test-core must tee direct output to `{CORE_REPORT}` under " + "`set -o pipefail`") + uploads = [s for s in _steps_of(core) if UPLOAD_RE.search(s) + and f"name: {CORE_REPORT_ARTIFACT}" in s] + if not uploads or f"path: {CORE_REPORT}" not in uploads[0] \ + or "if-no-files-found: error" not in uploads[0]: + E(f"{name}: test-core must upload `{CORE_REPORT}` as " + f"`{CORE_REPORT_ARTIFACT}` with `if-no-files-found: error`") + needs = re.search(r"^ needs:\s*\[([^\]]*)\]", service, re.M) + if not needs or "test-core" not in [n.strip() for n in needs.group(1).split(",")]: + E(f"{name}: test-service must need test-core to read its report") + if not any("actions/download-artifact" in s and f"name: {CORE_REPORT_ARTIFACT}" in s + for s in _steps_of(service)): + E(f"{name}: test-service must download `{CORE_REPORT_ARTIFACT}`") + gate = [s for s in _steps_of(service) if "--criteria-runtime" in s] + combined = re.search( + rf"cat {re.escape(CORE_REPORT)} {re.escape(SERVICE_REPORT)} > (\S+)", + gate[0] if gate else "") + if not gate or not combined or "set -euo pipefail" not in gate[0] \ + or f"--report {combined.group(1)}" not in gate[0]: + E(f"{name}: the criteria-runtime gate must read one report concatenated " + f"from `{CORE_REPORT}` and `{SERVICE_REPORT}` under `set -euo pipefail`") + return errs + + def check_component_gate(name, text): """neo-ci.yml is the Neo CLI component gate (contract 4). It must run on ARBITRARY PR bases (no base-branch allowlist, so stacked layers on any branch @@ -1019,6 +1069,7 @@ def run(root=WORKFLOWS): errs += check_codemap_regen(wf.name, text) if wf.name == REQUIRED_GATE: errs += check_required_gate(wf.name, text) + errs += check_criteria_report(wf.name, text) if wf.name == COMPONENT_GATE: errs += check_component_gate(wf.name, text) errs += check_consumer_contract(wf.name, text) @@ -1672,6 +1723,48 @@ def self_test(): good_slo.replace(" - uses: actions/checkout@v7.0.1\n", " - uses: actions/checkout@v7.0.1\n - uses: DeterminateSystems/determinate-nix-action@v3.21.7\n")) + good_report = ( + "jobs:\n" + " test-core:\n needs: [build]\n steps:\n" + " - name: Run nhcore-test-core\n run: |\n" + " set -o pipefail\n" + " cabal test nhcore-test-core --test-show-details=direct" + " | tee nhcore-test-core.report\n" + " - uses: actions/upload-artifact@v7\n with:\n" + " name: nhcore-test-core-report\n" + " path: nhcore-test-core.report\n" + " if-no-files-found: error\n" + " test-service:\n needs: [build, test-core]\n steps:\n" + " - uses: actions/download-artifact@v8\n with:\n" + " name: nhcore-test-core-report\n" + " - name: gate\n run: |\n set -euo pipefail\n" + " cat nhcore-test-core.report .nhcore-test-service.report > all.report\n" + " ./dev spec-check --criteria-runtime base --report all.report\n" + ) + + def cr_bad(name_, mutated): + check(name_, bool(check_criteria_report("test.yml", mutated))) + + check("well-wired criteria report passes", + not check_criteria_report("test.yml", good_report)) + cr_bad("service-only criteria report is flagged", + good_report.replace("--report all.report", "--report .nhcore-test-service.report")) + cr_bad("core report omitted from the concatenation is flagged", + good_report.replace("cat nhcore-test-core.report ", "cat ")) + cr_bad("test-core without a tee'd report is flagged", + good_report.replace(" | tee nhcore-test-core.report", "")) + cr_bad("test-core report without pipefail is flagged", + good_report.replace(" set -o pipefail\n", "")) + cr_bad("core report upload that tolerates a missing file is flagged", + good_report.replace(" if-no-files-found: error\n", "")) + cr_bad("test-service not needing test-core is flagged", + good_report.replace("needs: [build, test-core]", "needs: [build]")) + cr_bad("test-service not downloading the core report is flagged", + good_report.replace("actions/download-artifact@v8", "actions/cache@v6")) + cr_bad("commented-out core report download is flagged", + good_report.replace(" - uses: actions/download-artifact@v8\n", + " # - uses: actions/download-artifact@v8\n" + " - uses: actions/cache@v6\n")) print("workflow-check: self-test", "OK" if ok else "FAILED") return 0 if ok else 1