diff --git a/go.mod b/go.mod index 691a789a..46c3f02f 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,10 @@ require ( github.com/buildkite/go-buildkite/v4 v4.8.0 github.com/google/go-containerregistry v0.20.6 golang.org/x/net v0.46.0 + gopkg.in/inf.v0 v0.9.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 + k8s.io/apimachinery v0.34.0 modernc.org/sqlite v1.39.0 ) @@ -42,8 +44,10 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.18.0 // indirect @@ -58,6 +62,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/vbatts/tar-split v0.12.1 // indirect + github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect @@ -74,4 +79,5 @@ require ( modernc.org/libc v1.66.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect ) diff --git a/go.sum b/go.sum index 8f615edc..0db382f4 100644 --- a/go.sum +++ b/go.sum @@ -65,11 +65,15 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -83,6 +87,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -119,12 +125,18 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= @@ -145,37 +157,63 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b h1:18qgiDvlvH7kk8Ioa8Ov+K6xCi0GMvmGfGW0sgd/SYA= golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= -google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= -google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -183,6 +221,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +k8s.io/apimachinery v0.34.0 h1:eR1WO5fo0HyoQZt1wdISpFDffnWOvFLOOeJ7MgIv4z0= +k8s.io/apimachinery v0.34.0/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= @@ -209,3 +249,7 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= diff --git a/rayapp/compute_config_convert.go b/rayapp/compute_config_convert.go index b0d24a45..3cfd08fd 100644 --- a/rayapp/compute_config_convert.go +++ b/rayapp/compute_config_convert.go @@ -2,9 +2,13 @@ package rayapp import ( "fmt" + "math" "sort" + "strings" + "gopkg.in/inf.v0" yaml "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/api/resource" ) // Convert the new user-facing ComputeConfig schema (head_node/worker_nodes/...) to the @@ -178,7 +182,10 @@ func convertWorkerNode(w map[string]any) (map[string]any, error) { return nil, fmt.Errorf("worker node group 'min_nodes': %w", err) } if minWorkers < 0 { - return nil, fmt.Errorf("worker node group 'min_nodes' (%d) must be non-negative", minWorkers) + return nil, fmt.Errorf( + "worker node group 'min_nodes' (%d) must be non-negative", + minWorkers, + ) } legacy["min_workers"] = minWorkers maxWorkers, err := intOrDefault(w["max_nodes"], 10) @@ -186,10 +193,17 @@ func convertWorkerNode(w map[string]any) (map[string]any, error) { return nil, fmt.Errorf("worker node group 'max_nodes': %w", err) } if maxWorkers < 0 { - return nil, fmt.Errorf("worker node group 'max_nodes' (%d) must be non-negative", maxWorkers) + return nil, fmt.Errorf( + "worker node group 'max_nodes' (%d) must be non-negative", + maxWorkers, + ) } if maxWorkers < minWorkers { - return nil, fmt.Errorf("worker node group max_nodes (%d) must be >= min_nodes (%d)", maxWorkers, minWorkers) + return nil, fmt.Errorf( + "worker node group max_nodes (%d) must be >= min_nodes (%d)", + maxWorkers, + minWorkers, + ) } legacy["max_workers"] = maxWorkers @@ -221,6 +235,13 @@ func convertNodeCommonFields(node map[string]any) (map[string]any, error) { legacy[k] = v } } + if rr, ok := legacy["required_resources"]; ok { + normalized, err := normalizeRequiredResources(rr) + if err != nil { + return nil, err + } + legacy["required_resources"] = normalized + } if adv := node["advanced_instance_config"]; isTruthy(adv) { // Generic key; the launch path prefers it over aws_/gcp_ ones. legacy["advanced_configurations_json"] = adv @@ -237,6 +258,120 @@ func convertNodeCommonFields(node map[string]any) (map[string]any, error) { return legacy, nil } +// normalizeRequiredResources returns required_resources with `memory` as an +// integer number of bytes. +// +// The API takes memory as bytes, but the user-facing schema also allows a +// Kubernetes quantity string ("8Gi"). The SDK converts on the way to the API +// (PhysicalResources.to_dict(for_api=True) -> _parse_memory_string); the +// published bundle never touches the SDK — the console clone path parses it +// straight into the backend's PhysicalResources, whose `memory` is an int — +// so a string that passes `compute-config create -f` 422s at launch unless we +// convert here too. +func normalizeRequiredResources(v any) (any, error) { + rr, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("'required_resources' must be a mapping, got %T", v) + } + mem, ok := rr["memory"] + if !ok || mem == nil { + return rr, nil + } + bytes, err := parseMemoryBytes(mem) + if err != nil { + return nil, fmt.Errorf("'required_resources.memory': %w", err) + } + // Copy rather than mutate: the caller's parsed YAML is not ours to edit. + out := make(map[string]any, len(rr)) + for k, val := range rr { + out[k] = val + } + out["memory"] = bytes + return out, nil +} + +// parseMemoryBytes converts a memory value to bytes, accepting either an +// integer (already bytes) or a Kubernetes quantity string. +func parseMemoryBytes(v any) (int64, error) { + switch t := v.(type) { + case int: + return nonNegativeBytes(int64(t)) + case int64: + return nonNegativeBytes(t) + case float64: + // YAML gives a float only when the value was written as one; bytes are + // whole, so a fraction is a mistake rather than something to round. + if t != math.Trunc(t) { + return 0, fmt.Errorf("expected a whole number of bytes, got %v", t) + } + if t > math.MaxInt64 || t < math.MinInt64 { + return 0, fmt.Errorf("value %v is out of range", t) + } + return nonNegativeBytes(int64(t)) + case string: + return parseMemoryQuantity(t) + default: + return 0, fmt.Errorf( + "expected an integer number of bytes or a quantity string "+ + "like \"8Gi\", got %T", v, + ) + } +} + +// nonNegativeBytes rejects a negative byte count. Nothing downstream does: +// the backend's PhysicalResources only range-checks int64, so it would reach +// launch as a nonsense pod request. +func nonNegativeBytes(b int64) (int64, error) { + if b < 0 { + return 0, fmt.Errorf("must not be negative, got %d bytes", b) + } + return b, nil +} + +// parseMemoryQuantity converts a Kubernetes quantity string to bytes, using +// the same parser Kubernetes itself uses so the accepted grammar cannot drift +// from the one the docs describe. +// +// The value must come out as a whole number of bytes. Kubernetes tolerates a +// fractional one -- "100m" is 100 *milli*bytes -- but, as its docs put it, +// "this isn't useful to specify since you must always assign whole numbers of +// bytes", and the backend's PhysicalResources.memory is an int, so a fraction +// could not survive the trip anyway. Requiring it whole also removes the one +// place this parser and the SDK's disagree: ParseQuantity().Value() rounds up +// where the SDK's int(Decimal) truncates, and that difference only exists for +// values we now reject. +func parseMemoryQuantity(s string) (int64, error) { + // ParseQuantity reads a bare suffix ("Gi", "k", ".") as zero, so a typo + // would silently become no memory at all. The SDK rejects those, and so + // do we. + if !strings.ContainsAny(s, "0123456789") { + return 0, fmt.Errorf("invalid memory quantity %q: no number in it", s) + } + q, err := resource.ParseQuantity(s) + if err != nil { + return 0, fmt.Errorf("invalid memory quantity %q: %w", s, err) + } + if q.Sign() < 0 { + return 0, fmt.Errorf("memory quantity %q must not be negative", s) + } + + dec := q.AsDec() + whole := new(inf.Dec).Round(dec, 0, inf.RoundDown) + if whole.Cmp(dec) != 0 { + return 0, fmt.Errorf( + "memory quantity %q is %s bytes, which is not a whole number of bytes", + s, dec.String(), + ) + } + b, ok := whole.Unscaled() + if !ok || b == math.MaxInt64 { + // Quantity saturates at int64 max rather than reporting an overflow, + // so a value that lands exactly there is treated as out of range. + return 0, fmt.Errorf("memory quantity %q is too large", s) + } + return b, nil +} + func rejectUnknownKeys(m map[string]any, known map[string]bool, context string) error { var unknown []string for k := range m { diff --git a/rayapp/compute_config_convert_test.go b/rayapp/compute_config_convert_test.go index eafe846b..08865804 100644 --- a/rayapp/compute_config_convert_test.go +++ b/rayapp/compute_config_convert_test.go @@ -1,7 +1,11 @@ package rayapp import ( + "os/exec" "reflect" + "sort" + "strconv" + "strings" "testing" yaml "gopkg.in/yaml.v3" @@ -136,6 +140,61 @@ head_node_type: worker_node_types: [] auto_select_worker_config: false flags: {allow-cross-zone-autoscaling: false} +`, + }, + { + name: "declarative nodes: memory quantity strings become bytes", + input: ` +head_node: + required_resources: + CPU: 4 + memory: 8Gi + resources: + CPU: 0 +worker_nodes: +- name: gpu_worker + required_resources: + CPU: 4 + memory: 16Gi + GPU: 1 + required_labels: + ray.io/accelerator-type: T4 + min_nodes: 2 + max_nodes: 2 +`, + want: ` +head_node_type: + name: head-node + required_resources: {CPU: 4, memory: 8589934592} + resources: {CPU: 0} +worker_node_types: +- name: gpu_worker + required_resources: {CPU: 4, memory: 17179869184, GPU: 1} + required_labels: {ray.io/accelerator-type: T4} + min_workers: 2 + max_workers: 2 + use_spot: false + fallback_to_ondemand: false +auto_select_worker_config: false +flags: {allow-cross-zone-autoscaling: false} +`, + }, + { + name: "declarative nodes: integer bytes pass through unchanged", + input: ` +head_node: + required_resources: + CPU: 4 + memory: 8589934592 +worker_nodes: [] +`, + want: ` +head_node_type: + name: head-node + required_resources: {CPU: 4, memory: 8589934592} +worker_node_types: [] +auto_select_worker_config: false +flags: {allow-cross-zone-autoscaling: false} `, }, { @@ -195,22 +254,61 @@ func TestConvertNewComputeConfigToLegacyErrors(t *testing.T) { }{ {"unknown top-level key", "head_node: {instance_type: m5.2xlarge}\nbogus_top: 1\n"}, {"unknown node key", "head_node: {instance_type: m5.2xlarge, bogus_node: 1}\n"}, - {"bad market_type", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, market_type: BOGUS}\n"}, + { + "bad market_type", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, market_type: BOGUS}\n", + }, {"missing head_node", "worker_nodes: []\n"}, - {"non-integer min_nodes", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: five}\n"}, - {"auto_select not a bool", "head_node: {instance_type: m5.2xlarge}\nauto_select_worker_config: \"yes\"\n"}, + { + "non-integer min_nodes", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: five}\n", + }, + { + "auto_select not a bool", + "head_node: {instance_type: m5.2xlarge}\nauto_select_worker_config: \"yes\"\n", + }, {"zones not a list", "head_node: {instance_type: m5.2xlarge}\nzones: us-west-2a\n"}, - {"max_nodes < min_nodes", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: 5, max_nodes: 2}\n"}, + { + "max_nodes < min_nodes", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: 5, max_nodes: 2}\n", + }, {"head_node not a mapping", "head_node: just-a-string\n"}, {"flags not a mapping", "head_node: {instance_type: m5.2xlarge}\nflags: not-a-map\n"}, - {"worker name not a string", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {name: 123, instance_type: x}\n"}, - {"worker instance_type not a string", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: 123}\n"}, - {"worker market_type not a string", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, market_type: true}\n"}, - {"negative min_nodes", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: -1}\n"}, - {"negative max_nodes", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, max_nodes: -1}\n"}, - {"fractional min_nodes", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: 1.5}\n"}, + { + "worker name not a string", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {name: 123, instance_type: x}\n", + }, + { + "worker instance_type not a string", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: 123}\n", + }, + { + "worker market_type not a string", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, market_type: true}\n", + }, + { + "negative min_nodes", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: -1}\n", + }, + { + "negative max_nodes", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, max_nodes: -1}\n", + }, + { + "fractional min_nodes", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, min_nodes: 1.5}\n", + }, {"head flags not a mapping", "head_node: {instance_type: m5.2xlarge, flags: [a, b]}\n"}, - {"worker flags not a mapping", "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, flags: not-a-map}\n"}, + { + "worker flags not a mapping", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, flags: not-a-map}\n", + }, + {"bad head memory quantity", "head_node: {required_resources: {CPU: 4, memory: 8GB}}\n"}, + { + "bad worker memory quantity", + "head_node: {instance_type: m5.2xlarge}\nworker_nodes:\n- {instance_type: x, required_resources: {memory: 100m}}\n", + }, + {"required_resources not a mapping", "head_node: {required_resources: 8Gi}\n"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -220,3 +318,185 @@ func TestConvertNewComputeConfigToLegacyErrors(t *testing.T) { }) } } + +// memoryQuantityGoldens maps a quantity string to its value in bytes. Every +// entry is cross-checked against the SDK's _parse_memory_string by +// TestParseMemoryBytesMatchesSDK, which is what keeps this Go port from +// drifting from the Python one it mirrors. +var memoryQuantityGoldens = map[string]int64{ + // The shapes templates actually use. + "8Gi": 8589934592, + "16Gi": 17179869184, + "32Gi": 34359738368, + "4Gi": 4294967296, + "512Mi": 536870912, + // Every accepted suffix, so a wrong multiplier cannot hide. + "1Ki": 1024, + "1Mi": 1048576, + "1Gi": 1073741824, + "1Ti": 1099511627776, + "1Pi": 1125899906842624, + "1Ei": 1152921504606846976, + "1k": 1000, + "1M": 1000000, + "1G": 1000000000, + "1T": 1000000000000, + "1P": 1000000000000000, + "1E": 1000000000000000000, + // No suffix is already bytes. + "1000": 1000, + "0": 0, + "0Gi": 0, + // A fraction is fine as long as it lands on a whole number of bytes. + "1.5Gi": 1610612736, + "2.25Gi": 2415919104, + "0.5Mi": 524288, + // Exponent notation is part of the quantity grammar. + "1e3": 1000, +} + +func TestParseMemoryBytes(t *testing.T) { + for _, q := range sortedKeys(memoryQuantityGoldens) { + t.Run(q, func(t *testing.T) { + got, err := parseMemoryBytes(q) + if err != nil { + t.Fatalf("parseMemoryBytes(%q): %v", q, err) + } + if want := memoryQuantityGoldens[q]; got != want { + t.Errorf("parseMemoryBytes(%q) = %d, want %d", q, got, want) + } + }) + } + + // Numbers are already bytes and pass through. + nums := []struct { + name string + in any + want int64 + }{ + {"int", 8589934592, 8589934592}, + {"int64", int64(1024), 1024}, + {"float64 whole", float64(1024), 1024}, + {"zero", 0, 0}, + } + for _, tt := range nums { + t.Run(tt.name, func(t *testing.T) { + got, err := parseMemoryBytes(tt.in) + if err != nil { + t.Fatalf("parseMemoryBytes(%v): %v", tt.in, err) + } + if got != tt.want { + t.Errorf("parseMemoryBytes(%v) = %d, want %d", tt.in, got, tt.want) + } + }) + } +} + +// TestParseMemoryBytesErrors covers what we reject. The first group is what +// the SDK rejects too; the second is forms the SDK would convert silently into +// something the author did not mean, which is why this port is stricter. +func TestParseMemoryBytesErrors(t *testing.T) { + tests := []struct { + name string + in any + }{ + {"lowercase suffix", "8gi"}, + {"GB is not a k8s suffix", "8GB"}, + {"uppercase K is not a k8s suffix", "1K"}, + {"empty", ""}, + {"not a number", "abc"}, + {"suffix only", "Gi"}, + {"bare decimal suffix", "k"}, + {"just a dot", "."}, + {"just a sign", "+"}, + {"two dots", "1.2.3"}, + {"trailing junk", "8Gi "}, + {"space before suffix", "1 Gi"}, + {"bool", true}, + {"list", []any{1}}, + {"mapping", map[string]any{"a": 1}}, + {"fractional float", 1.5}, + {"negative int", -1}, + {"negative int64", int64(-1)}, + {"negative float", -1.0}, + {"negative quantity", "-1"}, + {"saturates int64", "16Ei"}, + + // Parse fine as quantities, but are not a whole number of bytes, so + // they could not reach the backend's integer memory field intact: + {"milli is 100 millibytes", "100m"}, + {"fraction of a byte", "1.5"}, + {"fractional mebibytes", "3.7Mi"}, + {"fractional kibibytes", "1.1Ki"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := parseMemoryBytes(tt.in); err == nil { + t.Errorf("parseMemoryBytes(%v) = %d, want an error", tt.in, got) + } + }) + } +} + +// sdkParseMemoryScript prints one parsed value per input line, and exits 97 +// when the anyscale SDK is not importable so the caller can skip. +const sdkParseMemoryScript = ` +import sys +try: + from anyscale.compute_config.models import _parse_memory_string +except Exception: + sys.exit(97) +for line in sys.stdin.read().splitlines(): + print(_parse_memory_string(line)) +` + +// TestParseMemoryBytesMatchesSDK checks every golden against the SDK function +// this port mirrors, so the two cannot drift apart unnoticed. Skipped where +// python3 or the anyscale package is unavailable. +func TestParseMemoryBytesMatchesSDK(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("python3 not available") + } + + quantities := sortedKeys(memoryQuantityGoldens) + cmd := exec.Command(python, "-c", sdkParseMemoryScript) + cmd.Stdin = strings.NewReader(strings.Join(quantities, "\n")) + out, err := cmd.Output() + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 97 { + t.Skip("anyscale SDK not importable") + } + if err != nil { + t.Fatalf("run SDK parser: %v", err) + } + + lines := strings.Fields(string(out)) + if len(lines) != len(quantities) { + t.Fatalf("got %d results for %d quantities: %q", len(lines), len(quantities), out) + } + for i, q := range quantities { + want, err := strconv.ParseInt(lines[i], 10, 64) + if err != nil { + t.Fatalf("parse SDK result %q for %q: %v", lines[i], q, err) + } + if golden := memoryQuantityGoldens[q]; golden != want { + t.Errorf("golden for %q is %d, but the SDK says %d", q, golden, want) + } + got, err := parseMemoryBytes(q) + if err != nil { + t.Fatalf("parseMemoryBytes(%q): %v", q, err) + } + if got != want { + t.Errorf("parseMemoryBytes(%q) = %d, but the SDK says %d", q, got, want) + } + } +} + +func sortedKeys(m map[string]int64) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +}