Upgrade awscrt to 0.35.0 (#10461)

Author: ashovlin

.changes/next-release/enhancement-awscrt-crt.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "awscrt",
+  "description": "Update awscrt version requirement to ``0.35.0``. This adds support for direct IO on downloads (GET requests) when the ``direct_io`` S3 configuration value is enabled."
+}

awscli/topics/s3-config.rst

@@ -348,6 +348,10 @@ files to and from S3. Valid choices are:
     * ``p5.48xlarge``
     * ``trn1n.32xlarge``
     * ``trn1.32xlarge``
+    * ``p5e.48xlarge``
+    * ``p5en.48xlarge``
+    * ``p6-b200.48xlarge``
+    * ``p6-b300.48xlarge``
 
   * There are no other running processes of the AWS CLI using the CRT S3 transfer
     client. To force multiple concurrently running processes of the AWS CLI to use
@@ -456,8 +460,13 @@ direct_io
 **Default** - ``false``
 
 If set to ``true``, the CRT client will enable direct IO to bypass the OS
-cache when sending PUT requests. Enabling direct IO may be useful in cases
-where the disk IO outperforms the kernel cache.
+cache for both uploads (PUT requests) and downloads (GET requests). Enabling
+direct IO may be useful in cases where the disk IO outperforms the kernel
+cache.
+
+Direct IO is a best-effort optimization. When its preconditions are not met
+(for example, when a part is not page-aligned), the CRT client transparently
+falls back to buffered IO and logs a warning rather than failing the transfer.
 
 Experimental Configuration Values
 =================================
@@ -469,3 +478,44 @@ Experimental Configuration Values
    guaranteed between versions of the AWS CLI.
 
 There are currently no experimental configuration values.
+
+CRT Environment Variables
+=========================
+
+When the ``preferred_transfer_client`` configuration value is set to or
+resolves to ``crt``, the following environment variables can be used to tune
+the behavior of the underlying CRT S3 client. These variables are provided by
+the CRT itself and are only applied when the ``crt`` transfer client is in use;
+the ``classic`` transfer client ignores them.
+
+AWS_CRT_S3_MEMORY_LIMIT_IN_GIB
+------------------------------
+Sets a memory cap, in gigabytes (GiB), for the CRT client's buffer pool, which
+manages memory for concurrent transfers. The value must be a positive integer
+representing GiB. An invalid value will raise ``AWS_ERROR_INVALID_ARGUMENT``.
+
+When this variable is unset, the CRT client selects a default memory limit
+based on the target throughput.
+
+For example, to cap the CRT client's buffer pool at 8 GiB before running a
+transfer::
+
+    $ export AWS_CRT_S3_MEMORY_LIMIT_IN_GIB=8
+    $ aws s3 cp large-file s3://amzn-s3-demo-bucket/large-file
+
+AWS_CRT_S3_MAX_PARTS_PENDING_READ
+---------------------------------
+**Default** - ``5``
+
+Controls the maximum number of parts that can be pending read from the input
+stream during an individual multipart upload. Increasing this value may improve
+throughput for large files when disk read speed benefits from reading more
+parts in parallel. The value must be a positive integer;
+a zero or otherwise invalid value is ignored with a warning and the default is
+used. This only affects multipart uploads.
+
+For example, to allow up to 20 parts to be pending read during a multipart
+upload::
+
+    $ export AWS_CRT_S3_MAX_PARTS_PENDING_READ=20
+    $ aws s3 cp large-file s3://amzn-s3-demo-bucket/large-file

pyproject.toml

@@ -41,7 +41,7 @@ dependencies = [
     "ruamel_yaml_clib>=0.2.0,<=0.2.15",
     "prompt-toolkit>=3.0.24,<3.0.52",
     "distro>=1.5.0,<1.9.0",
-    "awscrt==0.32.2",
+    "awscrt==0.35.0",
     "python-dateutil>=2.1,<=2.9.0",
     "jmespath>=0.7.1,<1.1.0",
     "urllib3>=1.25.4,<=2.6.3",

requirements-docs-lock.txt

@@ -8,59 +8,55 @@ alabaster==0.7.16 \
     --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \
     --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92
     # via sphinx
-awscrt==0.32.2 \
-    --hash=sha256:023a2f4595804a0f1d61ab49b64dda5612be9bfbe9b13759331e8e31658dda3f \
-    --hash=sha256:0d2e8a13063a3d9b61eee0e2885d133a75b38de3600f6b62437d8559f7d664e3 \
-    --hash=sha256:0fc7e9500c0c44a5ee42ab2784d10468c6c08cbc2f0ccfdafab9fa5a8fc43d6e \
-    --hash=sha256:1752afab4c0c530eae3023e44373914debc1e3b02d76d268aa11d1ebdf42fb07 \
-    --hash=sha256:193cc3ecf03a1dd6f989853b6c21549a0a9750c855520fedc8c3c8fbcd32e1bc \
-    --hash=sha256:25c7e7e6535cb2d2a4d22fd6264f621672d3903491318364dbc59066b63c7186 \
-    --hash=sha256:2792fc2877335673058d0b4e8249ef73dc36b22689fda939506aea6eceb42054 \
-    --hash=sha256:32785f54d0786e07b6491b51f9c2f75ea9e17decd39bb6b66fdc60cd871a49ef \
-    --hash=sha256:3a023e9f6ed7485175c3bd38868020144757c262000bd175a993ef4b9579216d \
-    --hash=sha256:3ef1664588d35dcad2115377120667e689cd7a517da52a481373c9536811ed96 \
-    --hash=sha256:4715bf71a4a5e89a26e63a21524263137385c1e09f967db745acda49a979798b \
-    --hash=sha256:47330f421948207a122092042f235cf82d48fa145c446ff4db12cc8cd3a418b6 \
-    --hash=sha256:4e975223f3af4faa581c733f0fdd316514b987c42ce85ef3bcd6d9c02eb48f77 \
-    --hash=sha256:5197d1e4e780d755632f79f8d32a09a30a9101cccb51ca1694fe25c711b2e801 \
-    --hash=sha256:51d6132e9d70de40da07dfad5f17780a652dd4b351c35ca97c79d0fa0186d645 \
-    --hash=sha256:5ec4d8200eb0158b60e35fbb65faa96ef1eb7763f6ce1192827886ee24c6df99 \
-    --hash=sha256:6cb3c858e96ba023de691a3b6478bed9fb59085433042dff78c42e59eed19cf2 \
-    --hash=sha256:6d153aa979d5d92421847682d6cc10cbbacfe8f5eff4e77b0764a3045674b459 \
-    --hash=sha256:7404cb551046bbe7cd454ea75f88b4a1b26f018d1b9bea83dbe46c174789d835 \
-    --hash=sha256:768e55dd83b8734fb92fdb67eef1a54b9c8af8fbb4ea79f9cba912b3929f17f8 \
-    --hash=sha256:79bb11d5d1dfdcfd867aac4a026bee11afbd2154279e12b66588442d8c14bdf7 \
-    --hash=sha256:7d5ab0bd3ede050ba6053d194e692b4778d485c9091608ea5fcfef7cdb67245e \
-    --hash=sha256:7ed7c209136650fe25659436bb4150e5af6eb43d71a0bf294f0bf414428736ee \
-    --hash=sha256:7f92b8f40a104a2a87ea5f428b3799220666ec1450b3a90665867d3715749e91 \
-    --hash=sha256:80420aa19c074a4c0335f2bd0e4aee3381fa452328d937795a1e0c779f0c052d \
-    --hash=sha256:8d731edda20dce5afc15a04731d91136a31779a244672d1f0a292a8b04aa0fd3 \
-    --hash=sha256:95cdc1bea168d4600126daf901d35101a8b989594d4d061fe6f60f7d020a6c2d \
-    --hash=sha256:979351dbc395be28928e561d35fe836d66ae28037b4673a16ac409fcf9c1d381 \
-    --hash=sha256:9b9e2125a88abafd2c6fd2b97e020ccdff0765df650f0525ec7c6e388f461605 \
-    --hash=sha256:9f8c4428defa80d7ed98b6e942bd374a9734178a2a26b747c42a66d5cbe8de12 \
-    --hash=sha256:9fd2dbca02f4e22fb5c02d1505327e6e6e9320dbe8ca80fc033cbbb29ed8631a \
-    --hash=sha256:a13c0a555bd930c829c72e6b2b2df70442b3b414037fd488984495b5beff5ddc \
-    --hash=sha256:a2f513f0fb3aafdf7cdf29d7f6f0c46bf4cc7880380c86e88635b7818565e76d \
-    --hash=sha256:a4f48805e8a66237923f03b7b692d213994cff42d1ff08125d1d60c74fcaf872 \
-    --hash=sha256:a6782c19a00f354c7b232b675f09cde94d1ca37bcf29009b8779b3f6395b27b4 \
-    --hash=sha256:a81f30a501d2eb6ba52c769cd6ecb3f7005512fba4a533211dc717e1115b0d94 \
-    --hash=sha256:a9ed98e20ca6fcad8ef32e3c6779aaa3526a5ff3f0aa99d59e0deeff59640375 \
-    --hash=sha256:ab06479bcdcb42b2956f59c0e4138049e5b44c885b7584ea05e39cc4d71b1f99 \
-    --hash=sha256:ab76d196b9c8bb429ebff8c8c44a0070760879ce4afd6440cc8c75f3fcd7381d \
-    --hash=sha256:ac90b6f1a268cd3a29daaee1290bdad6f4dfb9b6d4f4aef8f5192b8fa0f236ef \
-    --hash=sha256:acb707df280079d21d20ad1735b38a80d4939133cbaecfc2e4927dd8fc0190bc \
-    --hash=sha256:c116d1f9ea97a23581f6fb0836082162d18f7c70790c506a78cee0eeae7fd92b \
-    --hash=sha256:c68d92261fe192442c388f63f36a3a3f54784ab684f61208a737d2a182d4aeab \
-    --hash=sha256:cfd437122d5a2ec7eb9fffaf2cd8b96543d4a0d7e906b9515b79672005a1607a \
-    --hash=sha256:d2f7aee3bce261ab1ceba1fac404de4d496aa866237161d4257cef92bff9d828 \
-    --hash=sha256:d6a11bc885aa0e8197a6458a5dba16f5d9959a040cb1967a424ff1d8e803b777 \
-    --hash=sha256:d87467931fa54a000f26b2bea302c95b2c938a1b807b7e42fca296a36250d77a \
-    --hash=sha256:d8a4e52f7312e5e435461119aa903f6424e9996d93a040101fb1eb7b9c4e58dc \
-    --hash=sha256:f2a085d0dd4eef974f2ae5467ae3717d2fc08dd8cd508c4fd7a5acb658c68616 \
-    --hash=sha256:f8a586c52f41ff14c2f1b8afeb764e231ad3d66acfd42a6b9fb6c8afd8da8fe2 \
-    --hash=sha256:ff0fff9c2b613d7fabc298b0fd81f0d7056353f3d20271a852a719c5b2f7ccf4 \
-    --hash=sha256:ffb40027e6779138f6cb9b11a85ef76d00ef6b015de6d8ae8e6598659c4af996
+awscrt==0.35.0 \
+    --hash=sha256:0c933fc94db944f4d2504ffd6bd0c2ef2289a475e0f8896cfea6569be2e4796a \
+    --hash=sha256:0d0f49e5740c83bf2c0f0de3bb3eda07bb49205ab075b2f24410ed9b9ab07b0c \
+    --hash=sha256:10a3438c43f4f3ff3eeb4a95b5aec63535c8174e086789fca9d1880d56f68675 \
+    --hash=sha256:14055a74c83548ea369908b122d3470e70a43e4436f095573e285242a76da6c6 \
+    --hash=sha256:1734257417bddcfacfa9985eea110dc8affdb0cee7fc04bef918534a2c4db5a3 \
+    --hash=sha256:177f0f9bbddc3227ede427df0bee9353dbb32fbfdf0d424680400e6d76d11ebc \
+    --hash=sha256:286995476f2f8fd217fbe4c37c7e5bd69953cff03b6ea8d37946537a43208467 \
+    --hash=sha256:2fc7ab1cc6373eb59f439a75cfe837380d3b2c0275ef626b48f8ef75149bc71f \
+    --hash=sha256:341c357843500dee904cbbab799aa1055c959605fb463840b32a343dc9561f23 \
+    --hash=sha256:357ef075dcedfefa6befb8adbcd9f7213b1f906d82083014b668c80ff9f92327 \
+    --hash=sha256:393f4fb3a33d47db24092bf4c0d2170fcb127556a56ea8888e623cf67ad90c53 \
+    --hash=sha256:3d03795144958de79545d02eb5575b71652a4e92de8026c8a75ff5a51e752206 \
+    --hash=sha256:411d15fc50d91655d748e18603a7117450e98a83f5fd2a03a6a74f313de6ac61 \
+    --hash=sha256:42c69743ab4574ca37ad97e52f7fe7b8e70314fececece0dfffe0267fd08aa5d \
+    --hash=sha256:45de0f131bce6d39f2aefc493bb80fd1500b978ce98d867b448add82248ff608 \
+    --hash=sha256:4a7e5f267c494146ca680b1405ad2290b2dcc0821d5f386adcca8aa0bc427c25 \
+    --hash=sha256:4b572c48f473060181bf0544416fde1de803eef0be1252515a01e397bec2fd3c \
+    --hash=sha256:4eb7302dfabb252f1066d7576d6153353e84181de0b8f68d8e5e72cee1caefd4 \
+    --hash=sha256:50d0de009d053c5e875e85d8e78ae480bda8d861de238cf3f81b8696e488a595 \
+    --hash=sha256:59976c6063dd79d117d08f93aaddf02448a5b19086b9444c6bdbedf50feff54c \
+    --hash=sha256:6456ddebbedd701448fda24752bb0d280d5291d0c751944ab643c9c7a28e4bc6 \
+    --hash=sha256:703772a269bf013b3014dccef53cccca61f001f97cf79cea1a22fe494e993fe7 \
+    --hash=sha256:74166c05a58abb4d554bde5dbba73f85137be5291caa27194d1b8ef8e8edad9b \
+    --hash=sha256:761ae0dda17fd9dfaff4bbb2a376e28e44dfd77dc6410b7bc408297a1fd5600e \
+    --hash=sha256:7cacaf7516645e0edef360cbac30ee0d99db57a044ff6c12d35b717516c5315e \
+    --hash=sha256:7cd20c94a6008164c89eeb420892c3d88b165a79bdd22ef0a4ee383bee8b4cdb \
+    --hash=sha256:835db99dd677bc290e455f61f11b406b20daf8ab918ecad185839e1f8bf5e930 \
+    --hash=sha256:9074e27c24ec3382cc2cb2a702c7e8e471d57f58844f58e3305b1343a2882075 \
+    --hash=sha256:9bdefc61eb1cb2227c47e5d965fb3fa2c5e235915997739238a82e3e17c80bc1 \
+    --hash=sha256:9d766a3471f637bf4024fd487a10457e53809e17f3fb2f554bf4b73fad415f58 \
+    --hash=sha256:9f4b6c92827f0c033a32fe9c381f8026964cbafa0359c086726cf36deac2cb5f \
+    --hash=sha256:ab050c01fb3a64c4efc7a12baf49321c10e1635903e8ad05d1fe7a88ef5b0f2a \
+    --hash=sha256:ab54ea45befb2c867fc7cf272f2357eb35fa82ac59d3ae2c35b6dd10e546271a \
+    --hash=sha256:b0cc13d38911d2b1cf4e725d8d79a3ad7a4347a50d94183f593c3b8ccdfe2d41 \
+    --hash=sha256:bc604dd77f61c3b6fef06dd73108b390d01699e5535c6d9cf244775cd0bac2f3 \
+    --hash=sha256:bc78244437be093bc8f82045b2f111daa86eecef1e5d10575a99811a67503b81 \
+    --hash=sha256:c0ead21afa3853498b1d0a04d5af46f25220b1c00123ab04b2552262ceb55dec \
+    --hash=sha256:cb9132a8ef13647c74ad5f39b36fec146d07c975a6ae6050672121842d3a61ad \
+    --hash=sha256:cbdcedd9124faac5b5dfbd1a9ff08bb3268d416e88f065b6ed61e2893e8042d3 \
+    --hash=sha256:cbf5ac647aa06c3cbdd752e69fcd7a4520e786a5e4d8faa96bb0aa1e74b21ce1 \
+    --hash=sha256:cc51bc82cac744723959e3116a7724642bd41c163524c86f591a74dc98cee99f \
+    --hash=sha256:d283e17fc34fdde95c4a631eb18062ebedf9727a80f1768e02826bca1cc36d8f \
+    --hash=sha256:d368ed5ea31e414d726d560cae8b0e04de490a1252255b2059b1e4b58b598972 \
+    --hash=sha256:d42da56a4b900e40e6f6c373b9ba69795ec65ff4429a07491104590a9e6dbc71 \
+    --hash=sha256:d605f33d59707993a41bba37b38b56cbf59d577b05abb5d68aaa7d64fcf425c0 \
+    --hash=sha256:e26cca47c8b84dd968bde8c39385010acd710631e55b686a0e5422573c89a25b \
+    --hash=sha256:f809a697666be47f9883b369f24f8212ecedb7f58cd4318cbffca37f2662c55e \
+    --hash=sha256:fb321b44d98b7273752ef34ed0eb1d8d50757c6e165ecad1b8a3dd9281b1b8bc
     # via awscli (pyproject.toml)
 babel==2.18.0 \
     --hash=sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d \