feat: use of retryable http server

Signed-off-by: nabil salah <[email protected]>

go.mod

@@ -62,6 +62,8 @@ require (
 	gotest.tools v2.2.0+incompatible
 )
 
+require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+
 require (
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/Microsoft/hcsshim v0.8.25 // indirect
@@ -94,6 +96,7 @@ require (
 	github.com/hanwen/go-fuse/v2 v2.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/holiman/uint256 v1.2.3 // indirect
 	github.com/jarcoal/httpmock v1.3.1

go.sum

@@ -287,8 +287,12 @@ github.com/hanwen/go-fuse/v2 v2.3.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1r
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=

pkg/environment/config.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 )
 
@@ -51,9 +52,9 @@ func GetConfig() (base Config, err error) {
 
 // GetConfig returns extend config for specific run mode
 func GetConfigForMode(mode RunMode) (Config, error) {
-	httpClient := &http.Client{
-		Timeout: defaultHttpTimeout,
-	}
+	httpClient := retryablehttp.NewClient()
+	httpClient.HTTPClient.Timeout = defaultHttpTimeout
+	httpClient.RetryMax = 3
 
 	return getConfig(mode, baseExtendedURL, httpClient)
 }
@@ -70,7 +71,7 @@ func uniqueStr(slice []string) []string {
 	return list
 }
 
-func getConfig(run RunMode, url string, httpClient *http.Client) (ext Config, err error) {
+func getConfig(run RunMode, url string, httpClient *retryablehttp.Client) (ext Config, err error) {
 	if !strings.HasSuffix(url, "/") {
 		url += "/"
 	}

pkg/flist/flist.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
@@ -126,7 +127,7 @@ type flistModule struct {
 	commander commander
 	system    system
 
-	httpClient *http.Client
+	httpClient *retryablehttp.Client
 }
 
 func newFlister(root string, storage volumeAllocator, commander commander, system system) *flistModule {
@@ -155,6 +156,9 @@ func newFlister(root string, storage volumeAllocator, commander commander, syste
 		}
 	}
 
+	httpClient := retryablehttp.NewClient()
+	httpClient.HTTPClient.Timeout = defaultHubCallTimeout
+	httpClient.RetryMax = 3
 	return &flistModule{
 		root:       root,
 		flist:      filepath.Join(root, "flist"),
@@ -168,9 +172,7 @@ func newFlister(root string, storage volumeAllocator, commander commander, syste
 		commander: commander,
 		system:    system,
 
-		httpClient: &http.Client{
-			Timeout: defaultHubCallTimeout,
-		},
+		httpClient: httpClient,
 	}
 }
 
@@ -731,14 +733,14 @@ func (f *flistModule) downloadInNamespace(name, u string) (resp *http.Response,
 			return errors.Wrap(err, "failed to start tcp connection")
 		}
 
-		cl := http.Client{
-			Transport: &http.Transport{
-				DisableKeepAlives: true,
-				DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-					return con, nil
-				},
+		cl := retryablehttp.NewClient()
+		cl.HTTPClient.Transport = &http.Transport{
+			DisableKeepAlives: true,
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return con, nil
 			},
 		}
+		cl.RetryMax = 3
 
 		resp, err = cl.Get(u)
 		return err

pkg/gateway/metrics.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
@@ -139,14 +140,14 @@ func metrics(rawUrl string) (map[string]*metric, error) {
 
 	defer con.Close()
 
-	cl := http.Client{
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return con, nil
-			},
+	cl := retryablehttp.NewClient()
+	cl.HTTPClient.Transport = &http.Transport{
+		DisableKeepAlives: true,
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return con, nil
 		},
 	}
+	cl.RetryMax = 5
 
 	response, err := cl.Get(rawUrl)
 

pkg/perf/publicip/publicip_task.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/cenkalti/backoff/v3"
 	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/rs/zerolog/log"
 	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/zos/pkg/environment"
@@ -271,14 +272,15 @@ func getRealPublicIP() (net.IP, error) {
 
 	defer con.Close()
 
-	cl := http.Client{
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return con, nil
-			},
+	cl := retryablehttp.NewClient()
+	cl.HTTPClient.Transport = &http.Transport{
+		DisableKeepAlives: true,
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return con, nil
 		},
 	}
+	cl.RetryMax = 5
+
 	response, err := cl.Get("https://api.ipify.org/")
 	if err != nil {
 		return nil, errors.Join(err, errPublicIPLookup)

pkg/provision/engine.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v3"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/joncrlsn/dque"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
@@ -91,6 +92,8 @@ const (
 	opPause
 	// opResume resumes a deployment
 	opResume
+	// servers default timeout
+	defaultHttpTimeout = 10 * time.Second
 )
 
 // engineJob is a persisted job instance that is
@@ -1210,11 +1213,11 @@ func isTwinVerified(twinID uint32) (verified bool, err error) {
 	q.Set("twin_id", fmt.Sprint(twinID))
 	request.URL.RawQuery = q.Encode()
 
-	cl := &http.Client{
-		Timeout: 10 * time.Second,
-	}
+	cl := retryablehttp.NewClient()
+	cl.HTTPClient.Timeout = defaultHttpTimeout
+	cl.RetryMax = 3
 
-	response, err := cl.Do(request)
+	response, err := cl.Do(&retryablehttp.Request{Request: request})
 	if err != nil {
 		return
 	}

pkg/upgrade/hub/hub.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/0-fs/meta"
@@ -65,15 +66,16 @@ func MatchType(typ FListType) FListFilter {
 
 // HubClient API for f-list
 type HubClient struct {
-	httpClient *http.Client
+	httpClient *retryablehttp.Client
 }
 
 // NewHubClient create new hub client with the passed option for the http client
 func NewHubClient(timeout time.Duration) *HubClient {
+	httpClient := retryablehttp.NewClient()
+	httpClient.RetryMax = 5
+	httpClient.HTTPClient.Timeout = timeout
 	return &HubClient{
-		httpClient: &http.Client{
-			Timeout: timeout,
-		},
+		httpClient: httpClient,
 	}
 }
 

pkg/vm/client.go

@@ -8,12 +8,13 @@ import (
 	"net"
 	"net/http"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 )
 
 // Client to a cloud hypervisor instance
 type Client struct {
-	client http.Client
+	client *retryablehttp.Client
 }
 
 type VMData struct {
@@ -24,15 +25,16 @@ type VMData struct {
 
 // NewClient creates a new instance of client
 func NewClient(unix string) *Client {
-	client := Client{
-		client: http.Client{
-			Transport: &http.Transport{
-				Dial: func(network, _ string) (net.Conn, error) {
-					return net.Dial("unix", unix)
-				},
-			},
+	httpClient := retryablehttp.NewClient()
+	httpClient.RetryMax = 5
+	httpClient.HTTPClient.Transport = &http.Transport{
+		Dial: func(network, _ string) (net.Conn, error) {
+			return net.Dial("unix", unix)
 		},
 	}
+	client := Client{
+		client: httpClient,
+	}
 
 	return &client
 }
@@ -43,7 +45,7 @@ func (c *Client) Shutdown(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	response, err := c.client.Do(request)
+	response, err := c.client.Do(&retryablehttp.Request{Request: request})
 	if err != nil {
 		return errors.Wrap(err, "error calling machine shutdown")
 	}
@@ -61,7 +63,7 @@ func (c *Client) Pause(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	response, err := c.client.Do(request)
+	response, err := c.client.Do(&retryablehttp.Request{Request: request})
 	if err != nil {
 		return errors.Wrap(err, "error calling machine pause")
 	}
@@ -79,7 +81,7 @@ func (c *Client) Resume(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	response, err := c.client.Do(request)
+	response, err := c.client.Do(&retryablehttp.Request{Request: request})
 	if err != nil {
 		return errors.Wrap(err, "error calling machine pause")
 	}
@@ -100,7 +102,7 @@ func (c *Client) Inspect(ctx context.Context) (VMData, error) {
 	}
 	request.Header.Add("content-type", "application/json")
 
-	response, err := c.client.Do(request)
+	response, err := c.client.Do(&retryablehttp.Request{Request: request})
 	if err != nil {
 		return VMData{}, errors.Wrap(err, "error calling machine info")
 	}