Image embeddings on GCP

.github/workflows/ci.yml

@@ -77,7 +77,7 @@ jobs:
           npm test
       - name: Image Embedder bundling
         run: ./scripts/embedder-deploy/build.sh
-      - uses: guardian/actions-riff-raff@v4.2.4
+      - uses: guardian/actions-riff-raff@v4.3.3
         if: "! github.event.pull_request.head.repo.fork"
         with:
           githubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sbt-dependency-graph.yaml

@@ -19,7 +19,7 @@ jobs:
           java-version: 17
       - name: Install sbt
         id: sbt
-        uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14
+        uses: sbt/setup-sbt@4ed7b7ec4bfa2074fe48554c09e341267397247c # v1.2.0
       - name: Submit dependencies
         id: submit
         uses: scalacenter/sbt-dependency-submission@f43202114d7522a4b233e052f82c2eea8d658134 # v3.2.1

auth/app/auth/AuthComponents.scala

@@ -1,6 +1,6 @@
 package auth
 
-import com.gu.mediaservice.lib.management.{InnerServiceStatusCheckController, Management}
+import com.gu.mediaservice.lib.management.Management
 import com.gu.mediaservice.lib.play.GridComponents
 import play.api.ApplicationLoader.Context
 import play.api.{Configuration, Environment}
@@ -14,10 +14,9 @@ class AuthComponents(context: Context) extends GridComponents(context, new AuthC
 
   val controller = new AuthController(auth, providers, config, controllerComponents, authorisation)
   val permissionsAwareManagement = new Management(controllerComponents, buildInfo)
-  val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
 
 
-  override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement, InnerServiceStatusCheckController)
+  override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement)
 }
 
 object AuthHttpConfig {

auth/app/auth/AuthConfig.scala

@@ -1,8 +1,10 @@
 package auth
 
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
+import com.gu.mediaservice.model.Instance
 
 class AuthConfig(resources: GridConfigResources) extends CommonConfig(resources) {
-  val rootUri: String = services.authBaseUri
-  val mediaApiUri: String = services.apiBaseUri
+  val rootUri: Instance => String = services.authBaseUri
+  val rootInstanceUri: Instance => String = services.authBaseInstanceUri
+  val mediaApiUri: Instance => String = services.apiBaseUri
 }

auth/app/auth/AuthController.scala

@@ -6,43 +6,51 @@ import com.gu.mediaservice.lib.auth.Authentication.{InnerServicePrincipal, Machi
 import com.gu.mediaservice.lib.auth.Permissions.{DeleteImage, ShowPaid, UploadImages}
 import com.gu.mediaservice.lib.auth.provider.AuthenticationProviders
 import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, Internal}
+import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.guardian.auth.PandaAuthenticationProvider
+import com.gu.mediaservice.model.Instance
 import play.api.libs.json.Json
-import play.api.mvc.{BaseController, ControllerComponents, Result}
+import play.api.mvc.{AnyContent, BaseController, ControllerComponents, Request, Result}
 
 import java.net.URI
-import java.util.Date
+import java.time.Instant
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Try
 
 class AuthController(auth: Authentication, providers: AuthenticationProviders, val config: AuthConfig,
                      override val controllerComponents: ControllerComponents,
                      authorisation: Authorisation)(implicit ec: ExecutionContext)
   extends BaseController
-  with ArgoHelpers {
+  with ArgoHelpers with InstanceForRequest {
 
-  val indexResponse = {
+  def indexResponse()(implicit instance: Instance) = {
     val indexData = Map("description" -> "This is the Auth API")
     val indexLinks = List(
-      Link("root",          config.mediaApiUri),
-      Link("login",         config.services.loginUriTemplate),
-      Link("ui:logout",     s"${config.rootUri}/logout"),
-      Link("session",       s"${config.rootUri}/session")
+      Link("root",          config.mediaApiUri(instance)),
+      Link("login",         config.services.loginUriTemplate(instance)),
+      Link("ui:logout",     s"${config.rootUri(instance)}/logout"),
+      Link("session",       s"${config.rootInstanceUri(instance)}/session")
     )
     respond(indexData, indexLinks)
   }
 
   def cookieMonster = auth { request =>
     providers.userProvider match {
-      case panda: PandaAuthenticationProvider =>{
-        val cookieBatter = panda.readAuthenticatedUser(request).map(user => panda.generateCookie(user.copy(expires = new Date().getTime)))
+      case panda: PandaAuthenticationProvider =>
+        val cookieBatter = panda.readAuthenticatedUser(request)
+          // Note the cookie monster does not expire the cookie itself, but instead expires the panda token stored
+          // by the cookie. The cookie will remain in the browser storage, but if decoded will declare that it expired
+          // at the epoch in 1970.
+          .map(user => panda.generateCookie(user.copy(expires = Instant.ofEpochMilli(0L))))
         cookieBatter.fold(respond("Me want cookie."))(cookie => respond("Cookies are a sometimes food.").withCookies(cookie))
-      }
       case _ => respond("Me want cookie.")
     }
   }
 
-  def index = auth { indexResponse }
+  def index = auth { request =>
+    implicit val instance: Instance = instanceOf(request)
+    indexResponse()
+  }
 
   def session = auth { request =>
     val showPaid = authorisation.hasPermissionTo(ShowPaid)(request.user)

auth/conf/routes

@@ -16,7 +16,6 @@ GET     /cookieMonster              auth.AuthController.cookieMonster
 # Management
 GET     /management/healthcheck                       com.gu.mediaservice.lib.management.Management.healthCheck
 GET     /management/manifest                          com.gu.mediaservice.lib.management.Management.manifest
-GET     /management/whoAmI                            com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int)
 
 # Shoo robots away
 GET     /robots.txt                                   com.gu.mediaservice.lib.management.Management.disallowRobots

build.sbt

@@ -1,11 +1,10 @@
-import play.sbt.PlayImport.PlayKeys._
+import com.typesafe.sbt.packager.docker.*
+import play.sbt.PlayImport.PlayKeys.*
 import sbt.Package.FixedTimestamp
 
-import scala.sys.process._
+import scala.collection.JavaConverters.*
+import scala.sys.process.*
 import scala.util.control.NonFatal
-import scala.collection.JavaConverters._
-
-import com.typesafe.sbt.packager.debian.JDebPackaging
 
 // We need to keep the timestamps to allow caching headers to work as expected on assets.
 // The below should work, but some problem in one of the plugins (possible the play plugin? or sbt-web?) causes
@@ -53,6 +52,7 @@ val commonSettings = Seq(
     "org.scalatestplus" %% "mockito-3-4" % "3.1.4.0" % Test,
     "org.mockito" % "mockito-core" % "2.18.0" % Test,
     "org.scalamock" %% "scalamock" % "5.1.0" % Test,
+    "org.testcontainers" % "localstack" % "1.21.4" % Test
   ),
   dependencyOverrides ++= jacksonOverrides,
 
@@ -76,9 +76,10 @@ Global / concurrentRestrictions := Seq(
   Tags.limitAll(12)
 )
 
-val awsSdkVersion = "1.12.470"
-val awsSdkV2Version = "2.42.25"
-val elastic4sVersion = "8.18.2"
+val awsSdkVersion = "1.12.797"
+val awsSdkV2Version = "2.44.13"
+val elastic4sVersion = "8.19.1"
+val awsKclVersion = "3.4.3"
 val okHttpVersion = "3.12.1"
 
 val bbcBuildProcess: Boolean = System.getenv().asScala.get("BUILD_ORG").contains("bbc")
@@ -90,22 +91,25 @@ val maybeBBCLib: Option[sbt.ProjectReference] = if(bbcBuildProcess) Some(bbcProj
 
 lazy val commonLib = project("common-lib").settings(
   libraryDependencies ++= Seq(
+    "com.google.guava" % "guava" % "33.5.0-jre",
     "com.gu" %% "editorial-permissions-client" % "4.0.0",
-    "com.gu" %% "pan-domain-auth-play_3-0" % "9.0.0",
+    "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0",
     "com.amazonaws" % "aws-java-sdk-iam" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion,
+    "software.amazon.awssdk" % "sqs" % awsSdkV2Version,
     "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion,
-    "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion,
+    "software.amazon.awssdk" % "s3" % awsSdkV2Version,
     "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion,
     "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion,
     "nl.gn0s1s" %% "elastic4s-domain" % elastic4sVersion,
     "com.gu" %% "thrift-serializer" % "5.0.2",
     "org.scalaz" %% "scalaz-core" % "7.3.8",
     "org.im4java" % "im4java" % "1.4.0",
+    "app.photofox.vips-ffm" % "vips-ffm-core" % "1.9.6",
     "com.gu" % "kinesis-logback-appender" % "1.4.4",
     "net.logstash.logback" % "logstash-logback-encoder" % "5.0",
     logback, // play-logback; needed when running the scripts
@@ -124,8 +128,10 @@ lazy val commonLib = project("common-lib").settings(
     // declare explicit dependency on desired version of aws sdk v2 bedrock runtime
     "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version,
     "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version,
+    "com.adobe.xmp" % "xmpcore" % "6.1.11",
     ws,
-    "org.testcontainers" % "elasticsearch" % "1.21.4" % Test
+    "com.google.genai" % "google-genai" % "1.53.0" exclude("com.fasterxml.jackson.module", "jackson-module-kotlin"),
+    "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test,
   ),
   dependencyOverrides += "ch.qos.logback" % "logback-classic" % "1.2.13" % Test
 )
@@ -142,9 +148,9 @@ lazy val auth = playProject("auth", 9011)
 
 lazy val collections = playProject("collections", 9010)
 
-lazy val cropper = playProject("cropper", 9006)
+lazy val cropper = playImageLoaderProject("cropper", 9006)
 
-lazy val imageLoader = playProject("image-loader", 9003).settings {
+lazy val imageLoader = playImageLoaderProject("image-loader", 9003).settings {
   libraryDependencies ++= Seq(
     "org.apache.tika" % "tika-core" % "3.2.3",
     "com.drewnoakes" % "metadata-extractor" % "2.19.0"
@@ -176,39 +182,33 @@ lazy val thrall = playProject("thrall", 9002)
     pipelineStages := Seq(digest, gzip),
     libraryDependencies ++= Seq(
       "org.codehaus.groovy" % "groovy-json" % "3.0.7",
-      // TODO upgrading kcl to v3? check if you can remove avro override below
-      "software.amazon.kinesis" % "amazon-kinesis-client" % "2.6.1",
+      "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion,
       // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl
       "software.amazon.awssdk" % "kinesis" % awsSdkV2Version,
       "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version,
-      "com.gu" %% "kcl-pekko-stream" % "0.1.0",
-      "org.testcontainers" % "elasticsearch" % "1.19.2" % Test,
-      "com.google.protobuf" % "protobuf-java" % "3.19.6"
+      "com.gu" %% "kcl-pekko-stream" % "0.1.2",
+      "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test,
+      "com.google.protobuf" % "protobuf-java" % "3.19.6",
+      "software.amazon.awssdk" % "sqs" % awsSdkV2Version,
+      "org.apache.pekko" %% "pekko-connectors-sqs" % "1.0.2"
     ),
-    // amazon-kinesis-client 2.6.0 brings in a critically vulnerable version of apache avro,
-    // but we cannot upgrade amazon-kinesis-client further without performing the v2->v3 upgrade https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html
     dependencyOverrides ++= Seq(
-      "org.apache.avro" % "avro" % "1.11.4",
-      "org.apache.pekko" %% "pekko-stream" % "1.0.3"
+      "org.apache.pekko" %% "pekko-stream" % "1.0.3",
+      "org.apache.pekko" %% "pekko-http" % "1.0.1",
+      "org.apache.pekko" %% "pekko-http-core" % "1.0.1",
+      "org.apache.pekko" %% "pekko-parsing" % "1.0.1"
     )
   )
 
 lazy val usage = playProject("usage", 9009).settings(
   libraryDependencies ++= Seq(
-    "com.gu" %% "content-api-client-default" % "32.0.0",
-    "com.gu" %% "content-api-client-aws" % "0.7.6",
     "io.reactivex" %% "rxscala" % "0.27.0",
-    // amazon-kinesis-client brings in a critical vulnerability warning through apache avro, resolved in versions 1.11.4 and 1.12.0.
-    // updating amazon-kinesis-client? check if the override below can be removed
-    "software.amazon.kinesis" % "amazon-kinesis-client" % "3.0.2",
+    "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion,
     // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl
     "software.amazon.awssdk" % "kinesis" % awsSdkV2Version,
     "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version,
     "com.google.protobuf" % "protobuf-java" % "3.19.6"
   ),
-  dependencyOverrides ++= Seq(
-    "org.apache.avro" % "avro" % "1.11.4",
-  )
 )
 
 lazy val scripts = project("scripts")
@@ -251,39 +251,53 @@ val buildInfo = Seq(
 )
 
 def playProject(projectName: String, port: Int, path: Option[String] = None): Project = {
-  val commonProject = project(projectName, path)
-    .enablePlugins(PlayScala, JDebPackaging, SystemdPlugin, BuildInfoPlugin)
+  project(projectName, path)
+    .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin)
     .dependsOn(restLib)
     .settings(commonSettings ++ buildInfo ++ Seq(
+      dockerBaseImage := "eclipse-temurin:25",
+      dockerExposedPorts := Seq(port),
       playDefaultPort := port,
-      debianPackageDependencies := Seq("java11-runtime-headless"),
-      Linux / maintainer := "Guardian Developers <dig.dev.software@theguardian.com>",
-      Linux / packageSummary := description.value,
-      packageDescription := description.value,
 
       bashScriptEnvConfigLocation := Some("/etc/environment"),
-      Debian / makeEtcDefault := None,
-      Debian / packageBin := {
-        val originalFileName = (Debian / packageBin).value
-        val (base, ext) = originalFileName.baseAndExt
-        val newBase = base.replace(s"_${version.value}_all","")
-        val newFileName = file(originalFileName.getParent) / s"$newBase.$ext"
-        IO.move(originalFileName, newFileName)
-        println(s"Renamed $originalFileName to $newFileName")
-        newFileName
-      },
       Universal / mappings ++= Seq(
         file("common-lib/src/main/resources/application.conf") -> "conf/application.conf",
         file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml"
       ),
       Universal / javaOptions ++= Seq(
         "-Dpidfile.path=/dev/null",
-        s"-Dconfig.file=/usr/share/$projectName/conf/application.conf",
-        s"-Dlogger.file=/usr/share/$projectName/conf/logback.xml",
-        "-J-Xlog:gc*",
-        s"-J-Xlog:gc:/var/log/$projectName/gc.log"
-      )
-    ))
-  //Add the BBC library dependency if defined
-  maybeBBCLib.fold(commonProject){commonProject.dependsOn(_)}
+        s"-Dconfig.file=/opt/docker/conf/application.conf",
+        s"-Dlogger.file=/opt/docker/conf/logback.xml",
+        "-XX:+PrintCommandLineFlags", "-XX:MaxRAMPercentage=40"
+      ))
+    )
+}
+
+def playImageLoaderProject(projectName: String, port: Int, path: Option[String] = None): Project = {
+  project(projectName, path)
+    .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin)
+    .dependsOn(restLib)
+    .settings(commonSettings ++ buildInfo ++ Seq(
+      dockerBaseImage := "eu.gcr.io/grid-301122/jdk-vips:25-8.18.3",
+      dockerExposedPorts := Seq(port),
+      dockerCommands ++= Seq(
+        Cmd("ENV", "LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so")
+      ),
+      playDefaultPort := port,
+
+      bashScriptEnvConfigLocation := Some("/etc/environment"),
+      Universal / mappings ++= Seq(
+        file("common-lib/src/main/resources/application.conf") -> "conf/application.conf",
+        file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml",
+        file("image-loader/cmyk.icc") -> "cmyk.icc",
+        file("image-loader/facebook-TINYsRGB_c2.icc") -> "facebook-TINYsRGB_c2.icc",
+        file("image-loader/grayscale.icc") -> "grayscale.icc",
+        file("image-loader/srgb.icc") -> "srgb.icc"
+      ),
+      Universal / javaOptions ++= Seq(
+        "-Dpidfile.path=/dev/null",
+        s"-Dconfig.file=/opt/docker/conf/application.conf",
+        s"-Dlogger.file=/opt/docker/conf/logback.xml",
+        "-XX:+PrintCommandLineFlags", "-XX:MaxRAMPercentage=20"
+  )))
 }