Merge pull request #5 from Fr0stM0urne/gzip_decode_fix

Fix for python3 http client when accept-encoding is gzip

Author: pimterry

.gitignore

@@ -1,3 +1,4 @@
 *.log
 node_modules
 coverage*
+.vscode

src/targets/c/libcurl.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const CodeBuilder = require('../../helpers/code-builder')
+const helpers = require('../../helpers/headers')
 
 module.exports = function (source, options) {
   const code = new CodeBuilder()
@@ -26,16 +27,21 @@ module.exports = function (source, options) {
   }
 
   // construct cookies
-  if (source.allHeaders.cookie) {
+  if (helpers.hasHeader(source.allHeaders, 'cookie')) {
     code.blank()
-      .push('curl_easy_setopt(hnd, CURLOPT_COOKIE, "%s");', source.allHeaders.cookie)
+      .push('curl_easy_setopt(hnd, CURLOPT_COOKIE, "%s");', helpers.getHeader(source.allHeaders, 'cookie'))
   }
 
   if (source.postData.text) {
     code.blank()
       .push('curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, %s);', JSON.stringify(source.postData.text))
   }
 
+  if (helpers.hasHeader(source.allHeaders, 'accept-encoding')) {
+    code.blank()
+      .push('curl_easy_setopt(hnd, CURLOPT_ACCEPT_ENCODING, "");')
+  }
+
   code.blank()
     .push('CURLcode ret = curl_easy_perform(hnd);')
 

src/targets/go/native.js

@@ -11,6 +11,7 @@
 'use strict'
 
 const CodeBuilder = require('../../helpers/code-builder')
+const helpers = require('../../helpers/headers')
 
 module.exports = function (source, options) {
   // Let's Go!
@@ -110,6 +111,11 @@ module.exports = function (source, options) {
   errorCheck()
 
   // Add headers
+
+  // Go automatically adds this and handles decompression, as long as we don't try to
+  // manually add it ourselves:
+  delete source.allHeaders[helpers.getHeaderName(source.allHeaders, 'accept-encoding')]
+
   if (Object.keys(source.allHeaders).length) {
     Object.keys(source.allHeaders).forEach(function (key) {
       code.push(indent, 'req.Header.Add("%s", "%qd")', key, source.allHeaders[key])

src/targets/python/python3.js

@@ -11,16 +11,25 @@
 'use strict'
 
 const CodeBuilder = require('../../helpers/code-builder')
+const helpers = require('../../helpers/headers')
 
 module.exports = function (source, options) {
   const code = new CodeBuilder()
+
   // Start Request
   code.push('import http.client')
 
   if (options.insecureSkipVerify) {
     code.push('import ssl')
   }
 
+  const mayBeGzipped = helpers.hasHeader(source.allHeaders, 'accept-encoding') &&
+    helpers.getHeader(source.allHeaders, 'accept-encoding').includes('gzip')
+
+  if (mayBeGzipped) {
+    code.push('import gzip')
+  }
+
   code.blank()
 
   // Check which protocol to be used for the client connection
@@ -90,7 +99,16 @@ module.exports = function (source, options) {
     .push('res = conn.getresponse()')
     .push('data = res.read()')
     .blank()
-    .push('print(data.decode("utf-8"))')
+
+  // Decode response
+  if (mayBeGzipped) {
+    code.push("if res.headers['content-encoding'] == 'gzip':")
+    code.push('    print(gzip.decompress(data).decode("utf-8"))')
+    code.push('else:')
+    code.push('    print(data.decode("utf-8"))')
+  } else {
+    code.push('print(data.decode("utf-8"))')
+  }
 
   return code.join()
 }

test/fixtures/output/c/libcurl/compression.c

@@ -0,0 +1,12 @@
+CURL *hnd = curl_easy_init();
+
+curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "GET");
+curl_easy_setopt(hnd, CURLOPT_URL, "http://mockbin.com/har");
+
+struct curl_slist *headers = NULL;
+headers = curl_slist_append(headers, "accept-encoding: deflate, gzip, br");
+curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);
+
+curl_easy_setopt(hnd, CURLOPT_ACCEPT_ENCODING, "");
+
+CURLcode ret = curl_easy_perform(hnd);

test/fixtures/output/clojure/clj_http/compression.clj

@@ -0,0 +1,3 @@
+(require '[clj-http.client :as client])
+
+(client/get "http://mockbin.com/har" {:headers {:accept-encoding "deflate, gzip, br"}})

test/fixtures/output/csharp/httpclient/compression.cs

@@ -0,0 +1,16 @@
+var clientHandler = new HttpClientHandler
+{
+    AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
+};
+var client = new HttpClient(clientHandler);
+var request = new HttpRequestMessage
+{
+    Method = HttpMethod.Get,
+    RequestUri = new Uri("http://mockbin.com/har"),
+};
+using (var response = await client.SendAsync(request))
+{
+    response.EnsureSuccessStatusCode();
+    var body = await response.Content.ReadAsStringAsync();
+    Console.WriteLine(body);
+}

test/fixtures/output/csharp/restsharp/compression.cs

@@ -0,0 +1,4 @@
+var client = new RestClient("http://mockbin.com/har");
+var request = new RestRequest(Method.GET);
+request.AddHeader("accept-encoding", "deflate, gzip, br");
+IRestResponse response = client.Execute(request);

test/fixtures/output/go/native/compression.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"io"
+)
+
+func main() {
+
+	url := "http://mockbin.com/har"
+
+	req, _ := http.NewRequest("GET", url, nil)
+
+	res, _ := http.DefaultClient.Do(req)
+
+	defer res.Body.Close()
+	body, _ := io.ReadAll(res.Body)
+
+	fmt.Println(res)
+	fmt.Println(string(body))
+
+}

test/fixtures/output/http/1.1/compression

@@ -0,0 +1,3 @@
+GET /har HTTP/1.1
+Accept-Encoding: deflate, gzip, br
+Host: mockbin.com

test/fixtures/output/java/asynchttp/compression.java

@@ -0,0 +1,9 @@
+AsyncHttpClient client = new DefaultAsyncHttpClient();
+client.prepare("GET", "http://mockbin.com/har")
+  .setHeader("accept-encoding", "deflate, gzip, br")
+  .execute()
+  .toCompletableFuture()
+  .thenAccept(System.out::println)
+  .join();
+
+client.close();

test/fixtures/output/java/nethttp/compression.java

@@ -0,0 +1,7 @@
+HttpRequest request = HttpRequest.newBuilder()
+    .uri(URI.create("http://mockbin.com/har"))
+    .header("accept-encoding", "deflate, gzip, br")
+    .method("GET", HttpRequest.BodyPublishers.noBody())
+    .build();
+HttpResponse<String> response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
+System.out.println(response.body());

test/fixtures/output/java/okhttp/compression.java

@@ -0,0 +1,9 @@
+OkHttpClient client = new OkHttpClient();
+
+Request request = new Request.Builder()
+  .url("http://mockbin.com/har")
+  .get()
+  .addHeader("accept-encoding", "deflate, gzip, br")
+  .build();
+
+Response response = client.newCall(request).execute();

test/fixtures/output/java/unirest/compression.java

@@ -0,0 +1,3 @@
+HttpResponse<String> response = Unirest.get("http://mockbin.com/har")
+  .header("accept-encoding", "deflate, gzip, br")
+  .asString();

test/fixtures/output/javascript/axios/compression.js

@@ -0,0 +1,13 @@
+import axios from "axios";
+
+const options = {
+  method: 'GET',
+  url: 'http://mockbin.com/har',
+  headers: {'accept-encoding': 'deflate, gzip, br'}
+};
+
+axios.request(options).then(function (response) {
+  console.log(response.data);
+}).catch(function (error) {
+  console.error(error);
+});

test/fixtures/output/javascript/fetch/compression.js

@@ -0,0 +1,6 @@
+const options = {method: 'GET', headers: {'accept-encoding': 'deflate, gzip, br'}};
+
+fetch('http://mockbin.com/har', options)
+  .then(response => response.json())
+  .then(response => console.log(response))
+  .catch(err => console.error(err));

test/fixtures/output/javascript/jquery/compression.js

@@ -0,0 +1,13 @@
+const settings = {
+  "async": true,
+  "crossDomain": true,
+  "url": "http://mockbin.com/har",
+  "method": "GET",
+  "headers": {
+    "accept-encoding": "deflate, gzip, br"
+  }
+};
+
+$.ajax(settings).done(function (response) {
+  console.log(response);
+});

test/fixtures/output/javascript/xhr/compression.js

@@ -0,0 +1,15 @@
+const data = null;
+
+const xhr = new XMLHttpRequest();
+xhr.withCredentials = true;
+
+xhr.addEventListener("readystatechange", function () {
+  if (this.readyState === this.DONE) {
+    console.log(this.responseText);
+  }
+});
+
+xhr.open("GET", "http://mockbin.com/har");
+xhr.setRequestHeader("accept-encoding", "deflate, gzip, br");
+
+xhr.send(data);

test/fixtures/output/kotlin/okhttp/compression.kt

@@ -0,0 +1,9 @@
+val client = OkHttpClient()
+
+val request = Request.Builder()
+  .url("http://mockbin.com/har")
+  .get()
+  .addHeader("accept-encoding", "deflate, gzip, br")
+  .build()
+
+val response = client.newCall(request).execute()

test/fixtures/output/node/axios/compression.js

@@ -0,0 +1,13 @@
+var axios = require("axios").default;
+
+var options = {
+  method: 'GET',
+  url: 'http://mockbin.com/har',
+  headers: {'accept-encoding': 'deflate, gzip, br'}
+};
+
+axios.request(options).then(function (response) {
+  console.log(response.data);
+}).catch(function (error) {
+  console.error(error);
+});

test/fixtures/output/node/fetch/compression.js

@@ -0,0 +1,10 @@
+const fetch = require('node-fetch');
+
+let url = 'http://mockbin.com/har';
+
+let options = {method: 'GET', headers: {'accept-encoding': 'deflate, gzip, br'}};
+
+fetch(url, options)
+  .then(res => res.json())
+  .then(json => console.log(json))
+  .catch(err => console.error('error:' + err));

test/fixtures/output/node/native/compression.js

@@ -0,0 +1,26 @@
+const http = require("http");
+
+const options = {
+  "method": "GET",
+  "hostname": "mockbin.com",
+  "port": null,
+  "path": "/har",
+  "headers": {
+    "accept-encoding": "deflate, gzip, br"
+  }
+};
+
+const req = http.request(options, function (res) {
+  const chunks = [];
+
+  res.on("data", function (chunk) {
+    chunks.push(chunk);
+  });
+
+  res.on("end", function () {
+    const body = Buffer.concat(chunks);
+    console.log(body.toString());
+  });
+});
+
+req.end();

test/fixtures/output/node/request/compression.js

@@ -0,0 +1,13 @@
+const request = require('request');
+
+const options = {
+  method: 'GET',
+  url: 'http://mockbin.com/har',
+  headers: {'accept-encoding': 'deflate, gzip, br'}
+};
+
+request(options, function (error, response, body) {
+  if (error) throw new Error(error);
+
+  console.log(body);
+});

test/fixtures/output/node/unirest/compression.js

@@ -0,0 +1,13 @@
+const unirest = require("unirest");
+
+const req = unirest("GET", "http://mockbin.com/har");
+
+req.headers({
+  "accept-encoding": "deflate, gzip, br"
+});
+
+req.end(function (res) {
+  if (res.error) throw new Error(res.error);
+
+  console.log(res.body);
+});

test/fixtures/output/objc/nsurlsession/compression.m

@@ -0,0 +1,21 @@
+#import <Foundation/Foundation.h>
+
+NSDictionary *headers = @{ @"accept-encoding": @"deflate, gzip, br" };
+
+NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"http://mockbin.com/har"]
+                                                       cachePolicy:NSURLRequestUseProtocolCachePolicy
+                                                   timeoutInterval:10.0];
+[request setHTTPMethod:@"GET"];
+[request setAllHTTPHeaderFields:headers];
+
+NSURLSession *session = [NSURLSession sharedSession];
+NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
+                                            completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
+                                                if (error) {
+                                                    NSLog(@"%@", error);
+                                                } else {
+                                                    NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
+                                                    NSLog(@"%@", httpResponse);
+                                                }
+                                            }];
+[dataTask resume];

test/fixtures/output/ocaml/cohttp/compression.ml

@@ -0,0 +1,10 @@
+open Cohttp_lwt_unix
+open Cohttp
+open Lwt
+
+let uri = Uri.of_string "http://mockbin.com/har" in
+let headers = Header.add (Header.init ()) "accept-encoding" "deflate, gzip, br" in
+
+Client.call ~headers `GET uri
+>>= fun (res, body_stream) ->
+  (* Do stuff with the result *)