feat(s3): support large uploads (#37)

* feat(s3): implement multipart upload

* feat(s3): support tempFilePath and using read stream for uploads

Author: denolfe

dev/src/mocks/fileStub.js

@@ -0,0 +1 @@
+export default 'file-stub'

dev/src/mocks/promisifyMock.js

@@ -0,0 +1 @@
+export const promisify = () => {}

dev/src/payload.config.ts

@@ -9,6 +9,7 @@ import type { Adapter } from '../../src/types'
 import { Media } from './collections/Media'
 
 let adapter: Adapter
+let uploadOptions
 
 if (process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER === 'azure') {
   adapter = azureBlobStorageAdapter({
@@ -20,6 +21,11 @@ if (process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER === 'azure') {
 }
 
 if (process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER === 's3') {
+  // The s3 adapter supports using temp files for uploads
+  uploadOptions = {
+    useTempFiles: true,
+  }
+
   adapter = s3Adapter({
     config: {
       endpoint: process.env.S3_ENDPOINT,
@@ -47,6 +53,7 @@ if (process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER === 'gcs') {
 export default buildConfig({
   serverURL: 'http://localhost:3000',
   collections: [Media, Users],
+  upload: uploadOptions,
   admin: {
     // NOTE - these webpack extensions are only required
     // for development of this plugin.
@@ -59,6 +66,8 @@ export default buildConfig({
           alias: {
             ...(config.resolve.alias || {}),
             react: path.resolve(__dirname, '../node_modules/react'),
+            fs: path.resolve(__dirname, 'mocks/fileStub.js'),
+            util: path.resolve(__dirname, 'mocks/promisifyMock.js'),
             '@azure/storage-blob': path.resolve(__dirname, '../../src/adapters/azure/mock.js'),
             '@aws-sdk/client-s3': path.resolve(__dirname, '../../src/adapters/s3/mock.js'),
             '@google-cloud/storage': path.resolve(__dirname, '../../src/adapters/gcs/mock.js'),
@@ -72,6 +81,7 @@ export default buildConfig({
     outputFile: path.resolve(__dirname, 'payload-types.ts'),
   },
   plugins: [
+    // @ts-expect-error
     cloudStorage({
       collections: {
         media: {
@@ -81,12 +91,19 @@ export default buildConfig({
     }),
   ],
   onInit: async payload => {
-    await payload.create({
+    const users = await payload.find({
       collection: 'users',
-      data: {
-        email: 'dev@payloadcms.com',
-        password: 'test',
-      },
+      limit: 1,
     })
+
+    if (!users.docs.length) {
+      await payload.create({
+        collection: 'users',
+        data: {
+          email: 'dev@payloadcms.com',
+          password: 'test',
+        },
+      })
+    }
   },
 })

src/adapters/s3/handleUpload.ts

@@ -1,6 +1,8 @@
+import fs from 'fs'
 import path from 'path'
 import type * as AWS from '@aws-sdk/client-s3'
 import type { CollectionConfig } from 'payload/types'
+import type stream from 'stream'
 import type { HandleUpload } from '../../types'
 
 interface Args {
@@ -18,10 +20,19 @@ export const getHandleUpload = ({
   prefix = '',
 }: Args): HandleUpload => {
   return async ({ data, file }) => {
+    const fileKey = path.posix.join(prefix, file.filename)
+
+    let fileBufferOrStream: Buffer | stream.Readable
+    if (file.tempFilePath) {
+      fileBufferOrStream = fs.createReadStream(file.tempFilePath)
+    } else {
+      fileBufferOrStream = file.buffer
+    }
+
     await getStorageClient().putObject({
       Bucket: bucket,
-      Key: path.posix.join(prefix, file.filename),
-      Body: file.buffer,
+      Key: fileKey,
+      Body: fileBufferOrStream,
       ACL: acl,
       ContentType: file.mimeType,
     })

src/adapters/s3/index.ts

@@ -18,7 +18,8 @@ export const s3Adapter =
     let storageClient: AWS.S3 | null = null
     const getStorageClient = () => {
       if (storageClient) return storageClient
-      return (storageClient = new AWS.S3(config))
+      storageClient = new AWS.S3(config)
+      return storageClient
     }
 
     return {

src/types.ts

@@ -7,7 +7,9 @@ import type { Configuration as WebpackConfig } from 'webpack'
 export interface File {
   buffer: Buffer
   filename: string
+  filesize: number
   mimeType: string
+  tempFilePath?: string
 }
 
 export type HandleUpload = (args: {

src/utilities/getIncomingFiles.ts

@@ -18,6 +18,8 @@ export function getIncomingFiles({
       filename: data.filename,
       mimeType: data.mimeType,
       buffer: file.data,
+      tempFilePath: file.tempFilePath,
+      filesize: file.size,
     }
 
     files = [mainFile]
@@ -30,6 +32,7 @@ export function getIncomingFiles({
               filename: `${resizedFileData.filename}`,
               mimeType: data.mimeType,
               buffer: req.payloadUploadSizes[key],
+              filesize: req.payloadUploadSizes[key].length,
             },
           ])
         }