Update got to v16

MIGRATION.md

@@ -1,5 +1,21 @@
 # Migration Guide
 
+## From version 6 to 7
+
+#### Node.js
+Minimal supported Node.js version is 22.12.0.
+
+#### options.request
+[got](https://github.com/sindresorhus/got) was updated from v14 to v16. If you're using [request object](https://github.com/website-scraper/node-website-scraper#request) to customize requests, please check got's breaking changes in [v15](https://github.com/sindresorhus/got/releases/tag/v15.0.0) and [v16](https://github.com/sindresorhus/got/releases/tag/v16.0.0). Most notable:
+* `strictContentLength` is `true` by default - request fails if `Content-Length` doesn't match the actual body size
+* 300 and 304 responses are no longer followed as redirects
+* `url` is no longer allowed in options object
+* deprecated `searchParameters`, `followRedirects` and `auth` options now throw
+* `http2-wrapper` and `cacheable-lookup` were replaced with built-in implementations (`agent.http2`, `dnsCache` options changed)
+* credentials are stripped when a hook redirects request to a different origin
+
+`response.body` passed to `afterResponse` action is still a `Buffer` (with default `responseType: 'buffer'`), but `response.rawBody` is now a `Uint8Array`.
+
 ## From version 4 to 5
 
 #### ESM module

lib/request.js

@@ -40,6 +40,10 @@ function throwTypeError (result) {
 	throw new Error(`Wrong response handler result. Expected string or object, but received ${type}`);
 }
 
+function toBuffer (data) {
+	return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+}
+
 function getData (result) {
 	let data = result;
 	if (result && typeof result === 'object' && 'body' in result) {
@@ -60,8 +64,8 @@ function transformResult (result) {
 
 	// Then stringify it.
 	let body = null;
-	if (data instanceof Buffer) {
-		body = data.toString(encoding);
+	if (data instanceof Uint8Array) {
+		body = toBuffer(data).toString(encoding);
 	} else if (typeof data === 'string') {
 		body = data;
 	} else {
@@ -76,7 +80,7 @@ function transformResult (result) {
 }
 
 async function getRequest ({url, referer, options = {}, afterResponse = defaultResponseHandler}) {
-	const requestOptions = extend(options, {url});
+	const requestOptions = extend(options, {});
 
 	if (referer) {
 		requestOptions.headers = requestOptions.headers || {};
@@ -85,7 +89,11 @@ async function getRequest ({url, referer, options = {}, afterResponse = defaultR
 
 	logger.debug(`[request] sending request for url ${url}, referer ${referer}`);
 
-	const response = await got(requestOptions);
+	const response = await got(url, requestOptions);
+	// got returns Uint8Array for responseType 'buffer', keep Buffer for afterResponse handlers
+	if (response.body instanceof Uint8Array) {
+		response.body = toBuffer(response.body);
+	}
 	logger.debug(`[request] received response for ${response.url}, statusCode ${response.statusCode}`);
 	const responseHandlerResult = transformResult(await afterResponse({response}));
 

package.json

@@ -41,7 +41,7 @@
     "css-url-parser": "^1.0.0",
     "debug": "^4.3.1",
     "filenamify": "^7.0.0",
-    "got": "^14.4.7",
+    "got": "^16.1.0",
     "normalize-url": "^10.0.0",
     "p-queue": "^10.0.0",
     "srcset": "^5.0.0"

test/unit/request-test.js

@@ -61,6 +61,18 @@ describe('request', () => {
 		});
 	});
 
+	it('should pass response body to afterResponse as Buffer', () => {
+		const url = 'http://example.com';
+		nock(url).get('/').reply(200, 'TEST BODY');
+		const handlerStub = sinon.stub().resolves('');
+
+		return request.get({url, options: {responseType: 'buffer'}, afterResponse: handlerStub}).then(() => {
+			const { response } = handlerStub.getCall(0).args[0];
+			Buffer.isBuffer(response.body).should.eql(true);
+			response.body.toString().should.eql('TEST BODY');
+		});
+	});
+
 	describe('transformResult from afterResponse', () => {
 		it('should return object with body and metadata properties', () => {
 			const url = 'http://example.com';
@@ -270,6 +282,16 @@ describe('transformResult', () => {
 		result.should.have.property('metadata', null);
 	});
 
+	it('should handle object with Uint8Array body', () => {
+		const result = request.transformResult({
+			body: new TextEncoder().encode('SOME BODY'),
+		});
+
+		result.should.have.property('body', 'SOME BODY');
+		result.should.have.property('encoding', 'binary');
+		result.should.have.property('metadata', null);
+	});
+
 	it('should handle raw string input', () => {
 		const result = request.transformResult('SOME BODY');