diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 224dc3d3..2449a72e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,16 @@ on: - v2 paths-ignore: - '**.md' + - '**.bazel' + - 'WORKSPACE' pull_request: branches: - main - v2 paths-ignore: - '**.md' + - '**.bazel' + - 'WORKSPACE' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -21,7 +25,7 @@ jobs: name: Build strategy: matrix: - go: [ '1.21.x'] + go: [ 'stable', '1.19.x' ] os: [ ubuntu-latest ] runs-on: ${{ matrix.os }} @@ -43,7 +47,7 @@ jobs: run: diff -u <(echo -n) <(go fmt $(go list ./...)) - name: Test - run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic + run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic -coverpkg=./... - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/.github/workflows/label-actions.yml b/.github/workflows/label-actions.yml index 5c4c8734..a3ab38f2 100644 --- a/.github/workflows/label-actions.yml +++ b/.github/workflows/label-actions.yml @@ -3,12 +3,16 @@ name: 'Label' on: pull_request: types: [labeled] + paths-ignore: + - '**.md' + - '**.bazel' + - 'WORKSPACE' jobs: build: strategy: matrix: - go: [ '1.21.x'] + go: [ 'stable', '1.19.x' ] os: [ ubuntu-latest ] name: Run Build @@ -28,8 +32,13 @@ jobs: cache: true cache-dependency-path: go.sum + - name: Format + run: diff -u <(echo -n) <(go fmt $(go list ./...)) + - name: Test - run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic + run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic -coverpkg=./... - - name: Coverage - run: bash <(curl -s https://codecov.io/bash) + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 9e856bd4..7542ac89 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,6 @@ _testmain.go coverage.out coverage.txt -# Exclude intellij IDE folders +# Exclude IDE folders .idea/* +.vscode/* diff --git a/BUILD.bazel b/BUILD.bazel index 6e730167..0799f53a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,5 +1,5 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") load("@bazel_gazelle//:def.bzl", "gazelle") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") # gazelle:prefix github.com/go-resty/resty/v2 # gazelle:go_naming_convention import_alias @@ -20,10 +20,14 @@ go_library( "transport_js.go", "transport_other.go", "util.go", + "util_curl.go", ], importpath = "github.com/go-resty/resty/v2", visibility = ["//visibility:public"], - deps = ["@org_golang_x_net//publicsuffix:go_default_library"], + deps = [ + "//shellescape", + "@org_golang_x_net//publicsuffix:go_default_library", + ], ) go_test( @@ -32,6 +36,7 @@ go_test( "client_test.go", "context_test.go", "example_test.go", + "middleware_test.go", "request_test.go", "resty_test.go", "retry_test.go", @@ -39,7 +44,10 @@ go_test( ], data = glob([".testdata/*"]), embed = [":resty"], - deps = ["@org_golang_x_net//proxy:go_default_library"], + deps = [ + "@org_golang_x_net//proxy:go_default_library", + "@org_golang_x_time//rate:go_default_library", + ], ) alias( diff --git a/LICENSE b/LICENSE index 0c2d38a3..de30fea8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2023 Jeevanandam M., https://myjeeva.com +Copyright (c) 2015-2024 Jeevanandam M., https://myjeeva.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index ef6ed1d5..c0f387e6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

Features section describes in detail about Resty capabilities

-

Build Status Code Coverage Go Report Card Release Version GoDoc License Mentioned in Awesome Go

+

Build Status Code Coverage Go Report Card Release Version GoDoc License Mentioned in Awesome Go

Resty Communication Channels

@@ -13,7 +13,7 @@ ## News - * v2.13.0 [released](https://github.com/go-resty/resty/releases/tag/v2.13.0) and tagged on May 08, 2024. + * v2.14.0 [released](https://github.com/go-resty/resty/releases/tag/v2.14.0) and tagged on Aug 04, 2024. * v2.0.0 [released](https://github.com/go-resty/resty/releases/tag/v2.0.0) and tagged on Jul 16, 2019. * v1.12.0 [released](https://github.com/go-resty/resty/releases/tag/v1.12.0) and tagged on Feb 27, 2019. * v1.0 released and tagged on Sep 25, 2017. - Resty's first version was released on Sep 15, 2015 then it grew gradually as a very handy and helpful library. Its been a two years since first release. I'm very thankful to Resty users and its [contributors](https://github.com/go-resty/resty/graphs/contributors). @@ -62,6 +62,7 @@ * goroutine concurrent safe * Resty Client trace, see [Client.EnableTrace](https://pkg.go.dev/github.com/go-resty/resty/v2#Client.EnableTrace) and [Request.EnableTrace](https://pkg.go.dev/github.com/go-resty/resty/v2#Request.EnableTrace) * Since v2.4.0, trace info contains a `RequestAttempt` value, and the `Request` object contains an `Attempt` attribute + * Supports `GenerateCurlCommand`(**You should turn on `EnableTrace`**, otherwise the curl command will not contain the body) * Debug mode - clean and informative logging presentation * Gzip - Go does it automatically also resty has fallback handling too * Works fine with `HTTP/2` and `HTTP/1.1` @@ -122,15 +123,21 @@ The following samples will assist you to become as comfortable as possible with import "github.com/go-resty/resty/v2" ``` -#### Simple GET +#### Simple POST +>Refer: [debug_curl_test.go](https://github.com/go-resty/resty/blob/v2/examples/debug_curl_test.go) ```go // Create a Resty Client client := resty.New() resp, err := client.R(). - EnableTrace(). - Get("https://httpbin.org/get") + EnableTrace(). // You should turn on `EnableTrace`, otherwise the curl command will not contain the body + SetBody(map[string]string{"name": "Alex"}). + Post("https://httpbin.org/post") +curlCmdExecuted := resp.Request.GenerateCurlCommand() + +// Explore curl command +fmt.Println("Curl Command:\n ", curlCmdExecuted+"\n") // Explore response object fmt.Println("Response Info:") @@ -160,25 +167,36 @@ fmt.Println(" RequestAttempt:", ti.RequestAttempt) fmt.Println(" RemoteAddr :", ti.RemoteAddr.String()) /* Output +Curl Command: + curl -X POST -H 'Content-Type: application/json' -H 'User-Agent: go-resty/2.14.0 (https://github.com/go-resty/resty)' -d '{"name":"Alex"}' https://httpbin.org/post + Response Info: Error : Status Code: 200 Status : 200 OK Proto : HTTP/2.0 Time : 457.034718ms - Received At: 2020-09-14 15:35:29.784681 -0700 PDT m=+0.458137045 + Received At: 2024-08-09 13:02:57.187544 +0800 CST m=+1.304888501 Body : - { - "args": {}, - "headers": { - "Accept-Encoding": "gzip", - "Host": "httpbin.org", - "User-Agent": "go-resty/2.4.0 (https://github.com/go-resty/resty)", - "X-Amzn-Trace-Id": "Root=1-5f5ff031-000ff6292204aa6898e4de49" - }, - "origin": "0.0.0.0", - "url": "https://httpbin.org/get" - } + { + "args": {}, + "data": "{\"name\":\"Alex\"}", + "files": {}, + "form": {}, + "headers": { + "Accept-Encoding": "gzip", + "Content-Length": "15", + "Content-Type": "application/json", + "Host": "httpbin.org", + "User-Agent": "go-resty/2.14.0 (https://github.com/go-resty/resty)", + "X-Amzn-Trace-Id": "Root=1-66b5a301-567c83c86562abd3092f5e19" + }, + "json": { + "name": "Alex" + }, + "origin": "0.0.0.0", + "url": "https://httpbin.org/post" +} Request Trace Info: DNSLookup : 4.074657ms diff --git a/WORKSPACE b/WORKSPACE index 9ef03e95..504de145 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -4,10 +4,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "io_bazel_rules_go", - sha256 = "69de5c704a05ff37862f7e0f5534d4f479418afc21806c887db544a316f3cb6b", + sha256 = "80a98277ad1311dacd837f9b16db62887702e9f1d1c4c9f796d0121a46c8e184", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz", - "https://github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", ], ) @@ -24,7 +24,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe go_rules_dependencies() -go_register_toolchains(version = "1.16") +go_register_toolchains(version = "1.19") load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") diff --git a/client.go b/client.go index a55fb6b9..5c74ce7a 100644 --- a/client.go +++ b/client.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. @@ -178,6 +178,7 @@ type Client struct { // HeaderAuthorizationKey is used to set/access Request Authorization header // value when `SetAuthToken` option is used. HeaderAuthorizationKey string + ResponseBodyLimit int jsonEscapeHTML bool setContentLength bool @@ -476,11 +477,12 @@ func (c *Client) R() *Request { RawPathParams: map[string]string{}, Debug: c.Debug, - client: c, - multipartFiles: []*File{}, - multipartFields: []*MultipartField{}, - jsonEscapeHTML: c.jsonEscapeHTML, - log: c.log, + client: c, + multipartFiles: []*File{}, + multipartFields: []*MultipartField{}, + jsonEscapeHTML: c.jsonEscapeHTML, + log: c.log, + responseBodyLimit: c.ResponseBodyLimit, } return r } @@ -1123,6 +1125,20 @@ func (c *Client) SetJSONEscapeHTML(b bool) *Client { return c } +// SetResponseBodyLimit set a max body size limit on response, avoid reading too many data to memory. +// +// Client will return [resty.ErrResponseBodyTooLarge] if uncompressed response body size if larger than limit. +// Body size limit will not be enforced in following case: +// - ResponseBodyLimit <= 0, which is the default behavior. +// - [Request.SetOutput] is called to save a response data to file. +// - "DoNotParseResponse" is set for client or request. +// +// this can be overridden at client level with [Request.SetResponseBodyLimit] +func (c *Client) SetResponseBodyLimit(v int) *Client { + c.ResponseBodyLimit = v + return c +} + // EnableTrace method enables the Resty client trace for the requests fired from // the client using `httptrace.ClientTrace` and provides insights. // @@ -1195,9 +1211,7 @@ func (c *Client) Clone() *Client { // Client Unexported methods //_______________________________________________________________________ -// Executes method executes the given `Request` object and returns response -// error. -func (c *Client) execute(req *Request) (*Response, error) { +func (c *Client) executeBefore(req *Request) error { // Lock the user-defined pre-request hooks. c.udBeforeRequestLock.RLock() defer c.udBeforeRequestLock.RUnlock() @@ -1213,7 +1227,7 @@ func (c *Client) execute(req *Request) (*Response, error) { // to modify the *resty.Request object for _, f := range c.udBeforeRequest { if err = f(c, req); err != nil { - return nil, wrapNoRetryErr(err) + return wrapNoRetryErr(err) } } @@ -1221,14 +1235,14 @@ func (c *Client) execute(req *Request) (*Response, error) { // will return an error if the rate limit is exceeded. if req.client.rateLimiter != nil { if !req.client.rateLimiter.Allow() { - return nil, wrapNoRetryErr(ErrRateLimitExceeded) + return wrapNoRetryErr(ErrRateLimitExceeded) } } // resty middlewares for _, f := range c.beforeRequest { if err = f(c, req); err != nil { - return nil, wrapNoRetryErr(err) + return wrapNoRetryErr(err) } } @@ -1239,15 +1253,24 @@ func (c *Client) execute(req *Request) (*Response, error) { // call pre-request if defined if c.preReqHook != nil { if err = c.preReqHook(c, req.RawRequest); err != nil { - return nil, wrapNoRetryErr(err) + return wrapNoRetryErr(err) } } if err = requestLogger(c, req); err != nil { - return nil, wrapNoRetryErr(err) + return wrapNoRetryErr(err) } req.RawRequest.Body = newRequestBodyReleaser(req.RawRequest.Body, req.bodyBuf) + return nil +} + +// Executes method executes the given `Request` object and returns response +// error. +func (c *Client) execute(req *Request) (*Response, error) { + if err := c.executeBefore(req); err != nil { + return nil, err + } req.Time = time.Now() resp, err := c.httpClient.Do(req.RawRequest) @@ -1278,7 +1301,7 @@ func (c *Client) execute(req *Request) (*Response, error) { } } - if response.body, err = io.ReadAll(body); err != nil { + if response.body, err = readAllWithLimit(body, req.responseBodyLimit); err != nil { response.setReceivedAt() return response, err } @@ -1298,6 +1321,39 @@ func (c *Client) execute(req *Request) (*Response, error) { return response, wrapNoRetryErr(err) } +var ErrResponseBodyTooLarge = errors.New("resty: response body too large") + +// https://github.com/golang/go/issues/51115 +// [io.LimitedReader] can only return [io.EOF] +func readAllWithLimit(r io.Reader, maxSize int) ([]byte, error) { + if maxSize <= 0 { + return io.ReadAll(r) + } + + var buf [512]byte // make buf stack allocated + result := make([]byte, 0, 512) + total := 0 + for { + n, err := r.Read(buf[:]) + total += n + if total > maxSize { + return nil, ErrResponseBodyTooLarge + } + + if err != nil { + if err == io.EOF { + result = append(result, buf[:n]...) + break + } + return nil, err + } + + result = append(result, buf[:n]...) + } + + return result, nil +} + // getting TLS client config if not exists then create one func (c *Client) tlsConfig() (*tls.Config, error) { transport, err := c.Transport() diff --git a/client_test.go b/client_test.go index 90789e0f..0a93a54a 100644 --- a/client_test.go +++ b/client_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. @@ -6,6 +6,8 @@ package resty import ( "bytes" + "compress/gzip" + "crypto/rand" "crypto/tls" "errors" "fmt" @@ -1161,3 +1163,56 @@ func TestClone(t *testing.T) { assertEqual(t, "clone", parent.UserInfo.Username) assertEqual(t, "clone", clone.UserInfo.Username) } + +func TestResponseBodyLimit(t *testing.T) { + ts := createTestServer(func(w http.ResponseWriter, r *http.Request) { + io.CopyN(w, rand.Reader, 100*800) + }) + defer ts.Close() + + t.Run("Client body limit", func(t *testing.T) { + c := dc().SetResponseBodyLimit(1024) + + _, err := c.R().Get(ts.URL + "/") + assertNotNil(t, err) + assertEqual(t, err, ErrResponseBodyTooLarge) + }) + + t.Run("request body limit", func(t *testing.T) { + c := dc() + + _, err := c.R().SetResponseBodyLimit(1024).Get(ts.URL + "/") + assertNotNil(t, err) + assertEqual(t, err, ErrResponseBodyTooLarge) + }) + + t.Run("body less than limit", func(t *testing.T) { + c := dc() + + res, err := c.R().SetResponseBodyLimit(800*100 + 10).Get(ts.URL + "/") + assertNil(t, err) + assertEqual(t, 800*100, len(res.body)) + }) + + t.Run("no body limit", func(t *testing.T) { + c := dc() + + res, err := c.R().Get(ts.URL + "/") + assertNil(t, err) + assertEqual(t, 800*100, len(res.body)) + }) + + t.Run("read error", func(t *testing.T) { + tse := createTestServer(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(hdrContentEncodingKey, "gzip") + var buf [1024]byte + w.Write(buf[:]) + }) + defer tse.Close() + + c := dc() + + _, err := c.R().SetResponseBodyLimit(10240).Get(tse.URL + "/") + assertErrorIs(t, err, gzip.ErrHeader) + }) +} diff --git a/context_test.go b/context_test.go index f3c53450..5d60cfff 100644 --- a/context_test.go +++ b/context_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com) +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com) // 2016 Andrew Grigorev (https://github.com/ei-grad) // All rights reserved. // resty source code and usage is governed by a MIT style diff --git a/digest.go b/digest.go index 3cd19637..3a08477d 100644 --- a/digest.go +++ b/digest.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com) +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com) // 2023 Segev Dagan (https://github.com/segevda) // 2024 Philipp Wolfer (https://github.com/phw) // All rights reserved. diff --git a/example_test.go b/example_test.go index d7d91a12..425ea727 100644 --- a/example_test.go +++ b/example_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M. (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M. (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/examples/BUILD.bazel b/examples/BUILD.bazel new file mode 100644 index 00000000..849ea4e6 --- /dev/null +++ b/examples/BUILD.bazel @@ -0,0 +1,10 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "examples_test", + srcs = [ + "debug_curl_test.go", + "server_test.go", + ], + deps = ["//:resty"], +) diff --git a/examples/debug_curl_test.go b/examples/debug_curl_test.go new file mode 100644 index 00000000..68606edd --- /dev/null +++ b/examples/debug_curl_test.go @@ -0,0 +1,132 @@ +package examples + +import ( + "io" + "net/http" + "os" + "strings" + "testing" + + "github.com/go-resty/resty/v3" +) + +// 1. Generate curl for unexecuted request(dry-run) +func TestGenerateUnexcutedCurl(t *testing.T) { + ts := createHttpbinServer(0) + defer ts.Close() + + req := resty.New().R(). + SetBody(map[string]string{ + "name": "Alex", + }). + SetCookies( + []*http.Cookie{ + {Name: "count", Value: "1"}, + }, + ) + + curlCmdUnexecuted := req.GenerateCurlCommand() + + if !strings.Contains(curlCmdUnexecuted, "Cookie: count=1") || + !strings.Contains(curlCmdUnexecuted, "curl -X GET") || + !strings.Contains(curlCmdUnexecuted, `-d '{"name":"Alex"}'`) { + t.Fatal("Incomplete curl:", curlCmdUnexecuted) + } else { + t.Log("curlCmdUnexecuted: \n", curlCmdUnexecuted) + } + +} + +// 2. Generate curl for executed request +func TestGenerateExecutedCurl(t *testing.T) { + ts := createHttpbinServer(0) + defer ts.Close() + + data := map[string]string{ + "name": "Alex", + } + req := resty.New().R(). + SetBody(data). + SetCookies( + []*http.Cookie{ + {Name: "count", Value: "1"}, + }, + ) + + url := ts.URL + "/post" + resp, err := req. + EnableTrace(). + Post(url) + if err != nil { + t.Fatal(err) + } + curlCmdExecuted := resp.Request.GenerateCurlCommand() + if !strings.Contains(curlCmdExecuted, "Cookie: count=1") || + !strings.Contains(curlCmdExecuted, "curl -X POST") || + !strings.Contains(curlCmdExecuted, `-d '{"name":"Alex"}'`) || + !strings.Contains(curlCmdExecuted, url) { + t.Fatal("Incomplete curl:", curlCmdExecuted) + } else { + t.Log("curlCmdExecuted: \n", curlCmdExecuted) + } +} + +// 3. Generate curl in debug mode +func TestDebugModeCurl(t *testing.T) { + ts := createHttpbinServer(0) + defer ts.Close() + + // 1. Capture stderr + getOutput, restore := captureStderr() + defer restore() + + // 2. Build request + req := resty.New().R(). + SetBody(map[string]string{ + "name": "Alex", + }). + SetCookies( + []*http.Cookie{ + {Name: "count", Value: "1"}, + }, + ) + + // 3. Execute request: set debug mode + url := ts.URL + "/post" + _, err := req.SetDebug(true).Post(url) + if err != nil { + t.Fatal(err) + } + + // 4. test output curl + output := getOutput() + if !strings.Contains(output, "Cookie: count=1") || + !strings.Contains(output, `-d '{"name":"Alex"}'`) { + t.Fatal("Incomplete debug curl info:", output) + } else { + t.Log("Normal debug curl info: \n", output) + } +} + +func captureStderr() (getOutput func() string, restore func()) { + old := os.Stderr + r, w, err := os.Pipe() + if err != nil { + panic(err) + } + os.Stderr = w + getOutput = func() string { + w.Close() + buf := make([]byte, 2048) + n, err := r.Read(buf) + if err != nil && err != io.EOF { + panic(err) + } + return string(buf[:n]) + } + restore = func() { + os.Stderr = old + w.Close() + } + return getOutput, restore +} diff --git a/examples/server_test.go b/examples/server_test.go new file mode 100644 index 00000000..285f8b64 --- /dev/null +++ b/examples/server_test.go @@ -0,0 +1,162 @@ +package examples + +import ( + "bytes" + "encoding/json" + "fmt" + ioutil "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" +) + +const maxMultipartMemory = 4 << 30 // 4MB + +// tlsCert: +// +// 0 No certificate +// 1 With self-signed certificate +// 2 With custom certificate from CA(todo) +func createHttpbinServer(tlsCert int) (ts *httptest.Server) { + ts = createTestServer(func(w http.ResponseWriter, r *http.Request) { + httpbinHandler(w, r) + }, tlsCert) + + return ts +} + +func httpbinHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + body, _ := ioutil.ReadAll(r.Body) + r.Body = ioutil.NopCloser(bytes.NewBuffer(body)) // important!! + m := map[string]interface{}{ + "args": parseRequestArgs(r), + "headers": dumpRequestHeader(r), + "data": string(body), + "json": nil, + "form": map[string]string{}, + "files": map[string]string{}, + "method": r.Method, + "origin": r.RemoteAddr, + "url": r.URL.String(), + } + + // 1. parse text/plain + if strings.HasPrefix(r.Header.Get("Content-Type"), "text/plain") { + m["data"] = string(body) + } + + // 2. parse application/json + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + var data interface{} + if err := json.Unmarshal(body, &data); err != nil { + m["err"] = err.Error() + } else { + m["json"] = data + } + } + + // 3. parse application/x-www-form-urlencoded + if strings.HasPrefix(r.Header.Get("Content-Type"), "application/x-www-form-urlencoded") { + m["form"] = parseQueryString(string(body)) + } + + // 4. parse multipart/form-data + if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") { + form, files := readMultipartForm(r) + m["form"] = form + m["files"] = files + } + buf, _ := json.Marshal(m) + _, _ = w.Write(buf) +} + +func readMultipartForm(r *http.Request) (map[string]string, map[string]string) { + if err := r.ParseMultipartForm(maxMultipartMemory); err != nil { + if err != http.ErrNotMultipart { + panic(fmt.Sprintf("error on parse multipart form array: %v", err)) + } + } + // parse form data + formData := make(map[string]string) + for k, vs := range r.PostForm { + for _, v := range vs { + formData[k] = v + } + } + // parse files + files := make(map[string]string) + if r.MultipartForm != nil && r.MultipartForm.File != nil { + for key, fhs := range r.MultipartForm.File { + // if len(fhs)>0 + // f, err := fhs[0].Open() + files[key] = fhs[0].Filename + } + } + return formData, files +} + +func dumpRequestHeader(req *http.Request) string { + var res strings.Builder + headers := sortHeaders(req) + for _, kv := range headers { + res.WriteString(kv[0] + ": " + kv[1] + "\n") + } + return res.String() +} + +// sortHeaders +func sortHeaders(request *http.Request) [][2]string { + headers := [][2]string{} + for k, vs := range request.Header { + for _, v := range vs { + headers = append(headers, [2]string{k, v}) + } + } + n := len(headers) + for i := 0; i < n; i++ { + for j := n - 1; j > i; j-- { + jj := j - 1 + h1, h2 := headers[j], headers[jj] + if h1[0] < h2[0] { + headers[jj], headers[j] = headers[j], headers[jj] + } + } + } + return headers +} + +func parseRequestArgs(request *http.Request) map[string]string { + query := request.URL.RawQuery + return parseQueryString(query) +} + +func parseQueryString(query string) map[string]string { + params := map[string]string{} + paramsList, _ := url.ParseQuery(query) + for key, vals := range paramsList { + // params[key] = vals[len(vals)-1] + params[key] = strings.Join(vals, ",") + } + return params +} + +/* +* + - tlsCert: + 0 no certificate + 1 with self-signed certificate + 2 with custom certificate from CA(todo) +*/ +func createTestServer(fn func(w http.ResponseWriter, r *http.Request), tlsCert int) (ts *httptest.Server) { + if tlsCert == 0 { + // 1. http test server + ts = httptest.NewServer(http.HandlerFunc(fn)) + } else if tlsCert == 1 { + // 2. https test server: https://stackoverflow.com/questions/54899550/create-https-test-server-for-any-client + ts = httptest.NewUnstartedServer(http.HandlerFunc(fn)) + ts.StartTLS() + } + return ts +} diff --git a/go.mod b/go.mod index 2959f044..f2533ee7 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/go-resty/resty/v3 -go 1.18 +go 1.19 require ( - golang.org/x/net v0.25.0 - golang.org/x/time v0.5.0 + golang.org/x/net v0.27.0 + golang.org/x/time v0.6.0 ) diff --git a/go.sum b/go.sum index f0bd8d75..66793eb1 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,4 @@ -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= diff --git a/middleware.go b/middleware.go index d116ae0f..cba51c17 100644 --- a/middleware.go +++ b/middleware.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. @@ -302,6 +302,16 @@ func addCredentials(c *Client, r *Request) error { return nil } +func createCurlCmd(c *Client, r *Request) (err error) { + if r.trace { + if r.resultCurlCmd == nil { + r.resultCurlCmd = new(string) + } + *r.resultCurlCmd = buildCurlRequest(r.RawRequest, c.httpClient.Jar) + } + return nil +} + func requestLogger(c *Client, r *Request) error { if r.Debug { rr := r.RawRequest @@ -324,6 +334,8 @@ func requestLogger(c *Client, r *Request) error { } reqLog := "\n==============================================================================\n" + + "~~~ REQUEST(curl) ~~~\n" + + fmt.Sprintf("CURL:\n %v\n", buildCurlRequest(r.RawRequest, r.client.httpClient.Jar)) + "~~~ REQUEST ~~~\n" + fmt.Sprintf("%s %s %s\n", r.Method, rr.URL.RequestURI(), rr.Proto) + fmt.Sprintf("HOST : %s\n", rr.URL.Host) + @@ -412,6 +424,13 @@ func handleMultipart(c *Client, r *Request) error { r.bodyBuf = acquireBuffer() w := multipart.NewWriter(r.bodyBuf) + // Set boundary if not set by user + if r.multipartBoundary != "" { + if err := w.SetBoundary(r.multipartBoundary); err != nil { + return err + } + } + for k, v := range c.FormData { for _, iv := range v { if err := w.WriteField(k, iv); err != nil { diff --git a/redirect.go b/redirect.go index ed58d735..19bd587d 100644 --- a/redirect.go +++ b/redirect.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/request.go b/request.go index 4e13ff09..cfbe89b4 100644 --- a/request.go +++ b/request.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. @@ -39,6 +39,7 @@ type Request struct { Time time.Time Body interface{} Result interface{} + resultCurlCmd *string Error interface{} RawRequest *http.Request SRV *SRVRecord @@ -68,9 +69,29 @@ type Request struct { bodyBuf *bytes.Buffer clientTrace *clientTrace log Logger + multipartBoundary string multipartFiles []*File multipartFields []*MultipartField retryConditions []RetryConditionFunc + responseBodyLimit int +} + +// Generate curl command for the request. +func (r *Request) GenerateCurlCommand() string { + if r.resultCurlCmd != nil { + return *r.resultCurlCmd + } else { + if r.RawRequest == nil { + r.client.executeBefore(r) // mock with r.Get("/") + } + if r.resultCurlCmd == nil { + r.resultCurlCmd = new(string) + } + if *r.resultCurlCmd == "" { + *r.resultCurlCmd = buildCurlRequest(r.RawRequest, r.client.httpClient.Jar) + } + return *r.resultCurlCmd + } } // Context method returns the Context if its already set in request @@ -439,6 +460,15 @@ func (r *Request) SetMultipartFields(fields ...*MultipartField) *Request { return r } +// SetMultipartBoundary method sets the custom multipart boundary for the multipart request. +// Typically, the `mime/multipart` package generates a random multipart boundary, if not provided. +// +// Since v2.15.0 +func (r *Request) SetMultipartBoundary(boundary string) *Request { + r.multipartBoundary = boundary + return r +} + // SetContentLength method sets the HTTP header `Content-Length` value for current request. // By default Resty won't set `Content-Length`. Also you have an option to enable for every // request. @@ -571,6 +601,20 @@ func (r *Request) SetDoNotParseResponse(parse bool) *Request { return r } +// SetResponseBodyLimit set a max body size limit on response, avoid reading too many data to memory. +// +// Request will return [resty.ErrResponseBodyTooLarge] if uncompressed response body size if larger than limit. +// Body size limit will not be enforced in following case: +// - ResponseBodyLimit <= 0, which is the default behavior. +// - [Request.SetOutput] is called to save a response data to file. +// - "DoNotParseResponse" is set for client or request. +// +// This will override Client config. +func (r *Request) SetResponseBodyLimit(v int) *Request { + r.responseBodyLimit = v + return r +} + // SetPathParam method sets single URL path key-value pair in the // Resty current request instance. // @@ -886,7 +930,7 @@ func (r *Request) Patch(url string) (*Response, error) { // for current `Request`. // // req := client.R() -// req.Method = resty.GET +// req.Method = resty.MethodGet // req.URL = "http://httpbin.org/get" // resp, err := req.Send() func (r *Request) Send() (*Response, error) { @@ -896,7 +940,7 @@ func (r *Request) Send() (*Response, error) { // Execute method performs the HTTP request with given HTTP method and URL // for current `Request`. // -// resp, err := client.R().Execute(resty.GET, "http://httpbin.org/get") +// resp, err := client.R().Execute(resty.MethodGet, "http://httpbin.org/get") func (r *Request) Execute(method, url string) (*Response, error) { var addrs []*net.SRV var resp *Response diff --git a/request_test.go b/request_test.go index 01e07844..bae4a028 100644 --- a/request_test.go +++ b/request_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. @@ -1034,6 +1034,29 @@ func TestMultiPartMultipartFields(t *testing.T) { assertEqual(t, true, strings.Contains(responseStr, "upload-file-2.json")) } +func TestMultiPartCustomBoundary(t *testing.T) { + ts := createFormPostServer(t) + defer ts.Close() + defer cleanupFiles(".testdata/upload") + + _, err := dclr(). + SetMultipartFormData(map[string]string{"first_name": "Jeevanandam", "last_name": "M", "zip_code": "00001"}). + SetMultipartBoundary(`"my-custom-boundary"`). + SetBasicAuth("myuser", "mypass"). + Post(ts.URL + "/profile") + + assertEqual(t, "mime: invalid boundary character", err.Error()) + + resp, err := dclr(). + SetMultipartFormData(map[string]string{"first_name": "Jeevanandam", "last_name": "M", "zip_code": "00001"}). + SetMultipartBoundary("my-custom-boundary"). + Post(ts.URL + "/profile") + + assertError(t, err) + assertEqual(t, http.StatusOK, resp.StatusCode()) + assertEqual(t, "Success", resp.String()) +} + func TestGetWithCookie(t *testing.T) { ts := createGetServer(t) defer ts.Close() diff --git a/response.go b/response.go index 63c95c41..58a8e816 100644 --- a/response.go +++ b/response.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/resty.go b/resty.go index 0abbbd0c..86c2243c 100644 --- a/resty.go +++ b/resty.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/resty_test.go b/resty_test.go index c00ad1fb..95ef0b51 100644 --- a/resty_test.go +++ b/resty_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. @@ -809,6 +809,7 @@ func dclr() *Request { } func assertNil(t *testing.T, v interface{}) { + t.Helper() if !isNil(v) { t.Errorf("[%v] was expected to be nil", v) } @@ -841,6 +842,7 @@ func assertErrorIs(t *testing.T, e, g error) (r bool) { } func assertEqual(t *testing.T, e, g interface{}) (r bool) { + t.Helper() if !equal(e, g) { t.Errorf("Expected [%v], got [%v]", e, g) } diff --git a/retry.go b/retry.go index c5eda26b..932a266d 100644 --- a/retry.go +++ b/retry.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/retry_test.go b/retry_test.go index 8d58cc16..84c12a48 100644 --- a/retry_test.go +++ b/retry_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/shellescape/BUILD.bazel b/shellescape/BUILD.bazel new file mode 100644 index 00000000..fe829e39 --- /dev/null +++ b/shellescape/BUILD.bazel @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "shellescape", + srcs = ["shellescape.go"], + importpath = "github.com/go-resty/resty/v2/shellescape", + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":shellescape", + visibility = ["//visibility:public"], +) diff --git a/shellescape/shellescape.go b/shellescape/shellescape.go new file mode 100644 index 00000000..5e6a3799 --- /dev/null +++ b/shellescape/shellescape.go @@ -0,0 +1,34 @@ +/* +Package shellescape provides the shellescape.Quote to escape arbitrary +strings for a safe use as command line arguments in the most common +POSIX shells. + +The original Python package which this work was inspired by can be found +at https://pypi.python.org/pypi/shellescape. +*/ +package shellescape + +import ( + "regexp" + "strings" +) + +var pattern *regexp.Regexp + +func init() { + pattern = regexp.MustCompile(`[^\w@%+=:,./-]`) +} + +// Quote returns a shell-escaped version of the string s. The returned value +// is a string that can safely be used as one token in a shell command line. +func Quote(s string) string { + if len(s) == 0 { + return "''" + } + + if pattern.MatchString(s) { + return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" + } + + return s +} diff --git a/trace.go b/trace.go index be7555c2..7798a395 100644 --- a/trace.go +++ b/trace.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/util.go b/util.go index 5a69e4fc..7bbba912 100644 --- a/util.go +++ b/util.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file. diff --git a/util_curl.go b/util_curl.go new file mode 100644 index 00000000..6eab3b4b --- /dev/null +++ b/util_curl.go @@ -0,0 +1,76 @@ +package resty + +import ( + "bytes" + "io" + "net/http" + "net/http/cookiejar" + + "net/url" + "strings" + + "github.com/go-resty/resty/v3/shellescape" +) + +func buildCurlRequest(req *http.Request, httpCookiejar http.CookieJar) (curl string) { + // 1. Generate curl raw headers + curl = "curl -X " + req.Method + " " + // req.Host + req.URL.Path + "?" + req.URL.RawQuery + " " + req.Proto + " " + headers := dumpCurlHeaders(req) + for _, kv := range *headers { + curl += `-H ` + shellescape.Quote(kv[0]+": "+kv[1]) + ` ` + } + + // 2. Generate curl cookies + if cookieJar, ok := httpCookiejar.(*cookiejar.Jar); ok { + cookies := cookieJar.Cookies(req.URL) + if len(cookies) > 0 { + curl += ` -H ` + shellescape.Quote(dumpCurlCookies(cookies)) + " " + } + } + + // 3. Generate curl body + if req.Body != nil { + buf, _ := io.ReadAll(req.Body) + req.Body = io.NopCloser(bytes.NewBuffer(buf)) // important!! + curl += `-d ` + shellescape.Quote(string(buf)) + } + + urlString := shellescape.Quote(req.URL.String()) + if urlString == "''" { + urlString = "'http://unexecuted-request'" + } + curl += " " + urlString + return curl +} + +// dumpCurlCookies dumps cookies to curl format +func dumpCurlCookies(cookies []*http.Cookie) string { + sb := strings.Builder{} + sb.WriteString("Cookie: ") + for _, cookie := range cookies { + sb.WriteString(cookie.Name + "=" + url.QueryEscape(cookie.Value) + "&") + } + return strings.TrimRight(sb.String(), "&") +} + +// dumpCurlHeaders dumps headers to curl format +func dumpCurlHeaders(req *http.Request) *[][2]string { + headers := [][2]string{} + for k, vs := range req.Header { + for _, v := range vs { + headers = append(headers, [2]string{k, v}) + } + } + n := len(headers) + for i := 0; i < n; i++ { + for j := n - 1; j > i; j-- { + jj := j - 1 + h1, h2 := headers[j], headers[jj] + if h1[0] < h2[0] { + headers[jj], headers[j] = headers[j], headers[jj] + } + } + } + return &headers +} diff --git a/util_test.go b/util_test.go index 74cf4b1f..6c030fd7 100644 --- a/util_test.go +++ b/util_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. +// Copyright (c) 2015-2024 Jeevanandam M (jeeva@myjeeva.com), All rights reserved. // resty source code and usage is governed by a MIT style // license that can be found in the LICENSE file.