Prebuilt base images

.github/workflows/check-labels.yaml

@@ -0,0 +1,9 @@
+name: CORE4 label enforcement
+permissions:
+  contents: read
+on:
+  pull_request:
+    types: [opened, labeled, unlabeled, synchronize, reopened, edited]
+jobs:
+  require-label:
+    uses: guardian/.github/.github/workflows/require-label.yaml@4cb5024736632ffcc564b7f4b772c38b8e5ce739 #  v2.0.0

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       pull-requests: write
     services:
       elasticsearch:
-        image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.29
         # Wait for elasticsearch to report healthy before continuing.
         # see https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml#L28
         options: -e "discovery.type=single-node" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10
@@ -36,11 +36,13 @@ jobs:
           --health-timeout 5s
           --health-retries 10
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6.0.2
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v6
         with:
           node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: "cdk/package-lock.json"
       - name: Kahuna (client-side)
         working-directory: ./kahuna
         run: |
@@ -72,7 +74,11 @@ jobs:
           npm install-clean
           npm test
           npm run build
-      - uses: guardian/actions-riff-raff@v4.2.3
+      - name: Image embedder cdk
+        run: ./scripts/embedder-deploy/cdk.sh
+      - name: Image embedder bundling
+        run: ./scripts/embedder-deploy/build.sh
+      - uses: guardian/actions-riff-raff@v4.2.4
         if: "! github.event.pull_request.head.repo.fork"
         with:
           githubToken: ${{ secrets.GITHUB_TOKEN }}
@@ -107,5 +113,8 @@ jobs:
 
             image-counter-lambda:
               - image-counter-lambda/dist/image-counter-lambda.zip
-
-
+
+            cdk.out:
+              - cdk/cdk.out
+            image-embedder-lambda:
+              - image-embedder.zip

.github/workflows/push-pr.yml

@@ -18,7 +18,7 @@ jobs:
       contents: write
       actions: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6.0.2
       - name: push-pr
         env:
           PR: ${{ inputs.prNumber }}

.github/workflows/sbt-dependency-graph.yaml

@@ -10,19 +10,19 @@ jobs:
     steps:
       - name: Checkout branch
         id: checkout
-        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1
       - name: Install Java
         id: java
-        uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.2.0
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
         with:
           distribution: corretto
           java-version: 17
       - name: Install sbt
         id: sbt
-        uses: sbt/setup-sbt@8a071aa780c993c7a204c785d04d3e8eb64ef272 # v1.1.0
+        uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14
       - name: Submit dependencies
         id: submit
-        uses: scalacenter/sbt-dependency-submission@64084844d2b0a9b6c3765f33acde2fbe3f5ae7d3 # v3.1.0
+        uses: scalacenter/sbt-dependency-submission@f43202114d7522a4b233e052f82c2eea8d658134 # v3.2.1
       - name: Log snapshot for user validation
         id: validate
         run: cat ${{ steps.submit.outputs.snapshot-json-path }} | jq

.gitignore

@@ -28,3 +28,8 @@ dev/config/permissions.json
 
 coverage/
 .bsp
+
+image-embedder.zip
+
+# Integration test data (cached S3 downloads and outputs)
+image-embedder-lambda/__tests__/integration/test-data/

auth/app/auth/AuthComponents.scala

@@ -1,6 +1,6 @@
 package auth
 
-import com.gu.mediaservice.lib.management.{InnerServiceStatusCheckController, Management}
+import com.gu.mediaservice.lib.management.Management
 import com.gu.mediaservice.lib.play.GridComponents
 import play.api.ApplicationLoader.Context
 import play.api.{Configuration, Environment}
@@ -14,10 +14,9 @@ class AuthComponents(context: Context) extends GridComponents(context, new AuthC
 
   val controller = new AuthController(auth, providers, config, controllerComponents, authorisation)
   val permissionsAwareManagement = new Management(controllerComponents, buildInfo)
-  val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
 
 
-  override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement, InnerServiceStatusCheckController)
+  override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement)
 }
 
 object AuthHttpConfig {

auth/app/auth/AuthConfig.scala

@@ -1,8 +1,10 @@
 package auth
 
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
+import com.gu.mediaservice.model.Instance
 
 class AuthConfig(resources: GridConfigResources) extends CommonConfig(resources) {
-  val rootUri: String = services.authBaseUri
-  val mediaApiUri: String = services.apiBaseUri
+  val rootUri: Instance => String = services.authBaseUri
+  val rootInstanceUri: Instance => String = services.authBaseInstanceUri
+  val mediaApiUri: Instance => String = services.apiBaseUri
 }

auth/app/auth/AuthController.scala

@@ -6,9 +6,11 @@ import com.gu.mediaservice.lib.auth.Authentication.{InnerServicePrincipal, Machi
 import com.gu.mediaservice.lib.auth.Permissions.{DeleteImage, ShowPaid, UploadImages}
 import com.gu.mediaservice.lib.auth.provider.AuthenticationProviders
 import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, Internal}
+import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.guardian.auth.PandaAuthenticationProvider
+import com.gu.mediaservice.model.Instance
 import play.api.libs.json.Json
-import play.api.mvc.{BaseController, ControllerComponents, Result}
+import play.api.mvc.{AnyContent, BaseController, ControllerComponents, Request, Result}
 
 import java.net.URI
 import java.util.Date
@@ -19,15 +21,15 @@ class AuthController(auth: Authentication, providers: AuthenticationProviders, v
                      override val controllerComponents: ControllerComponents,
                      authorisation: Authorisation)(implicit ec: ExecutionContext)
   extends BaseController
-  with ArgoHelpers {
+  with ArgoHelpers with InstanceForRequest {
 
-  val indexResponse = {
+  def indexResponse()(implicit instance: Instance) = {
     val indexData = Map("description" -> "This is the Auth API")
     val indexLinks = List(
-      Link("root",          config.mediaApiUri),
-      Link("login",         config.services.loginUriTemplate),
-      Link("ui:logout",     s"${config.rootUri}/logout"),
-      Link("session",       s"${config.rootUri}/session")
+      Link("root",          config.mediaApiUri(instance)),
+      Link("login",         config.services.loginUriTemplate(instance)),
+      Link("ui:logout",     s"${config.rootUri(instance)}/logout"),
+      Link("session",       s"${config.rootInstanceUri(instance)}/session")
     )
     respond(indexData, indexLinks)
   }
@@ -42,7 +44,10 @@ class AuthController(auth: Authentication, providers: AuthenticationProviders, v
     }
   }
 
-  def index = auth { indexResponse }
+  def index = auth { request =>
+    implicit val instance: Instance = instanceOf(request)
+    indexResponse()
+  }
 
   def session = auth { request =>
     val showPaid = authorisation.hasPermissionTo(ShowPaid)(request.user)

auth/conf/routes

@@ -16,7 +16,6 @@ GET     /cookieMonster              auth.AuthController.cookieMonster
 # Management
 GET     /management/healthcheck                       com.gu.mediaservice.lib.management.Management.healthCheck
 GET     /management/manifest                          com.gu.mediaservice.lib.management.Management.manifest
-GET     /management/whoAmI                            com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int)
 
 # Shoo robots away
 GET     /robots.txt                                   com.gu.mediaservice.lib.management.Management.disallowRobots

build.sbt

@@ -1,11 +1,10 @@
-import play.sbt.PlayImport.PlayKeys._
+import com.typesafe.sbt.packager.docker.*
+import play.sbt.PlayImport.PlayKeys.*
 import sbt.Package.FixedTimestamp
 
-import scala.sys.process._
+import scala.collection.JavaConverters.*
+import scala.sys.process.*
 import scala.util.control.NonFatal
-import scala.collection.JavaConverters._
-
-import com.typesafe.sbt.packager.debian.JDebPackaging
 
 // We need to keep the timestamps to allow caching headers to work as expected on assets.
 // The below should work, but some problem in one of the plugins (possible the play plugin? or sbt-web?) causes
@@ -21,7 +20,7 @@ ThisBuild / libraryDependencySchemes +=
   "org.scala-lang.modules" %% "scala-java8-compat" % VersionScheme.Always
 
 val commonSettings = Seq(
-  scalaVersion := "2.13.16",
+  scalaVersion := "2.13.18",
   description := "grid",
   organization := "com.gu",
   version := "0.1",
@@ -39,6 +38,7 @@ val commonSettings = Seq(
     "org.scalatestplus" %% "mockito-3-4" % "3.1.4.0" % Test,
     "org.mockito" % "mockito-core" % "2.18.0" % Test,
     "org.scalamock" %% "scalamock" % "5.1.0" % Test,
+    "org.testcontainers" % "localstack" % "1.21.4" % Test
   ),
   dependencyOverrides += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.19.0",
 
@@ -63,7 +63,7 @@ Global / concurrentRestrictions := Seq(
 )
 
 val awsSdkVersion = "1.12.470"
-val awsSdkV2Version = "2.31.12"
+val awsSdkV2Version = "2.32.33"
 val elastic4sVersion = "8.18.2"
 val okHttpVersion = "3.12.1"
 
@@ -84,16 +84,17 @@ lazy val commonLib = project("common-lib").settings(
     "com.amazonaws" % "aws-java-sdk-cloudwatch" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-cloudfront" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion,
+    "software.amazon.awssdk" % "sqs" % awsSdkV2Version,
     "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion,
-    "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion,
     "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion,
     "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion,
     "nl.gn0s1s" %% "elastic4s-domain" % elastic4sVersion,
     "com.gu" %% "thrift-serializer" % "5.0.2",
     "org.scalaz" %% "scalaz-core" % "7.3.8",
     "org.im4java" % "im4java" % "1.4.0",
+    "app.photofox.vips-ffm" % "vips-ffm-core" % "1.9.6",
     "com.gu" % "kinesis-logback-appender" % "1.4.4",
     "net.logstash.logback" % "logstash-logback-encoder" % "5.0",
     logback, // play-logback; needed when running the scripts
@@ -107,8 +108,13 @@ lazy val commonLib = project("common-lib").settings(
     "org.scanamo" %% "scanamo" % "2.0.0",
     // declare explicit dependency on desired version of aws sdk v2 dynamo
     "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version,
+    "software.amazon.awssdk" % "dynamodb-enhanced" % awsSdkV2Version,
+    // declare explicit dependency on desired version of aws sdk v2 bedrock runtime
+    "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version,
+    "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version,
+    "com.adobe.xmp" % "xmpcore" % "6.1.11",
     ws,
-    "org.testcontainers" % "elasticsearch" % "1.19.2" % Test
+    "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test,
   ),
   dependencyOverrides += "ch.qos.logback" % "logback-classic" % "1.2.13" % Test
 )
@@ -125,11 +131,11 @@ lazy val auth = playProject("auth", 9011)
 
 lazy val collections = playProject("collections", 9010)
 
-lazy val cropper = playProject("cropper", 9006)
+lazy val cropper = playImageLoaderProject("cropper", 9006)
 
-lazy val imageLoader = playProject("image-loader", 9003).settings {
+lazy val imageLoader = playImageLoaderProject("image-loader", 9003).settings {
   libraryDependencies ++= Seq(
-    "org.apache.tika" % "tika-core" % "1.28.5",
+    "org.apache.tika" % "tika-core" % "3.2.3",
     "com.drewnoakes" % "metadata-extractor" % "2.19.0"
   )
 }
@@ -164,8 +170,9 @@ lazy val thrall = playProject("thrall", 9002)
       "software.amazon.awssdk" % "kinesis" % awsSdkV2Version,
       "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version,
       "com.gu" %% "kcl-pekko-stream" % "0.1.0",
-      "org.testcontainers" % "elasticsearch" % "1.19.2" % Test,
-      "com.google.protobuf" % "protobuf-java" % "3.19.6"
+      "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test,
+      "com.google.protobuf" % "protobuf-java" % "3.19.6",
+      "software.amazon.awssdk" % "sqs" % awsSdkV2Version
     ),
     // amazon-kinesis-client 2.6.0 brings in a critically vulnerable version of apache avro,
     // but we cannot upgrade amazon-kinesis-client further without performing the v2->v3 upgrade https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html
@@ -177,8 +184,6 @@ lazy val thrall = playProject("thrall", 9002)
 
 lazy val usage = playProject("usage", 9009).settings(
   libraryDependencies ++= Seq(
-    "com.gu" %% "content-api-client-default" % "32.0.0",
-    "com.gu" %% "content-api-client-aws" % "0.7.6",
     "io.reactivex" %% "rxscala" % "0.27.0",
     // amazon-kinesis-client brings in a critical vulnerability warning through apache avro, resolved in versions 1.11.4 and 1.12.0.
     // updating amazon-kinesis-client? check if the override below can be removed
@@ -233,39 +238,53 @@ val buildInfo = Seq(
 )
 
 def playProject(projectName: String, port: Int, path: Option[String] = None): Project = {
-  val commonProject = project(projectName, path)
-    .enablePlugins(PlayScala, JDebPackaging, SystemdPlugin, BuildInfoPlugin)
+  project(projectName, path)
+    .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin)
     .dependsOn(restLib)
     .settings(commonSettings ++ buildInfo ++ Seq(
+      dockerBaseImage := "eclipse-temurin:25",
+      dockerExposedPorts := Seq(port),
       playDefaultPort := port,
-      debianPackageDependencies := Seq("java11-runtime-headless"),
-      Linux / maintainer := "Guardian Developers <dig.dev.software@theguardian.com>",
-      Linux / packageSummary := description.value,
-      packageDescription := description.value,
 
       bashScriptEnvConfigLocation := Some("/etc/environment"),
-      Debian / makeEtcDefault := None,
-      Debian / packageBin := {
-        val originalFileName = (Debian / packageBin).value
-        val (base, ext) = originalFileName.baseAndExt
-        val newBase = base.replace(s"_${version.value}_all","")
-        val newFileName = file(originalFileName.getParent) / s"$newBase.$ext"
-        IO.move(originalFileName, newFileName)
-        println(s"Renamed $originalFileName to $newFileName")
-        newFileName
-      },
       Universal / mappings ++= Seq(
         file("common-lib/src/main/resources/application.conf") -> "conf/application.conf",
         file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml"
       ),
       Universal / javaOptions ++= Seq(
         "-Dpidfile.path=/dev/null",
-        s"-Dconfig.file=/usr/share/$projectName/conf/application.conf",
-        s"-Dlogger.file=/usr/share/$projectName/conf/logback.xml",
-        "-J-Xlog:gc*",
-        s"-J-Xlog:gc:/var/log/$projectName/gc.log"
-      )
-    ))
-  //Add the BBC library dependency if defined
-  maybeBBCLib.fold(commonProject){commonProject.dependsOn(_)}
+        s"-Dconfig.file=/opt/docker/conf/application.conf",
+        s"-Dlogger.file=/opt/docker/conf/logback.xml",
+        "-XX:+PrintCommandLineFlags", "-XX:MaxRAMPercentage=40"
+      ))
+    )
+}
+
+def playImageLoaderProject(projectName: String, port: Int, path: Option[String] = None): Project = {
+  project(projectName, path)
+    .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin)
+    .dependsOn(restLib)
+    .settings(commonSettings ++ buildInfo ++ Seq(
+      dockerBaseImage := "eu.gcr.io/grid-301122/jdk-vips:25-8.18.2",
+      dockerExposedPorts := Seq(port),
+      dockerCommands ++= Seq(
+        Cmd("ENV", "LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so")
+      ),
+      playDefaultPort := port,
+
+      bashScriptEnvConfigLocation := Some("/etc/environment"),
+      Universal / mappings ++= Seq(
+        file("common-lib/src/main/resources/application.conf") -> "conf/application.conf",
+        file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml",
+        file("image-loader/cmyk.icc") -> "cmyk.icc",
+        file("image-loader/facebook-TINYsRGB_c2.icc") -> "facebook-TINYsRGB_c2.icc",
+        file("image-loader/grayscale.icc") -> "grayscale.icc",
+        file("image-loader/srgb.icc") -> "srgb.icc"
+      ),
+      Universal / javaOptions ++= Seq(
+        "-Dpidfile.path=/dev/null",
+        s"-Dconfig.file=/opt/docker/conf/application.conf",
+        s"-Dlogger.file=/opt/docker/conf/logback.xml",
+        "-XX:+PrintCommandLineFlags", "-XX:MaxRAMPercentage=20"
+  )))
 }

cdk/.gitignore

@@ -0,0 +1,11 @@
+*.js
+!jest.config.js
+!jest.setup.js
+!.eslintrc.js
+*.d.ts
+node_modules
+dist
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

cdk/README.md

@@ -0,0 +1,5 @@
+# Infrastructure
+
+This directory defines the components to be deployed to AWS.
+
+See [`package.json`](./package.json) for a list of available scripts.