Black Lives Matter. Support the Equal Justice Initiative.

Source file src/net/http/client.go

Documentation: net/http

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // HTTP client. See RFC 7230 through 7235.
     6  //
     7  // This is the high-level Client interface.
     8  // The low-level implementation is in transport.go.
     9  
    10  package http
    11  
    12  import (
    13  	"context"
    14  	"crypto/tls"
    15  	"encoding/base64"
    16  	"errors"
    17  	"fmt"
    18  	"io"
    19  	"log"
    20  	"net/http/internal/ascii"
    21  	"net/url"
    22  	"reflect"
    23  	"sort"
    24  	"strings"
    25  	"sync"
    26  	"time"
    27  )
    28  
    29  // A Client is an HTTP client. Its zero value (DefaultClient) is a
    30  // usable client that uses DefaultTransport.
    31  //
    32  // The Client's Transport typically has internal state (cached TCP
    33  // connections), so Clients should be reused instead of created as
    34  // needed. Clients are safe for concurrent use by multiple goroutines.
    35  //
    36  // A Client is higher-level than a RoundTripper (such as Transport)
    37  // and additionally handles HTTP details such as cookies and
    38  // redirects.
    39  //
    40  // When following redirects, the Client will forward all headers set on the
    41  // initial Request except:
    42  //
    43  // • when forwarding sensitive headers like "Authorization",
    44  // "WWW-Authenticate", and "Cookie" to untrusted targets.
    45  // These headers will be ignored when following a redirect to a domain
    46  // that is not a subdomain match or exact match of the initial domain.
    47  // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
    48  // will forward the sensitive headers, but a redirect to "bar.com" will not.
    49  //
    50  // • when forwarding the "Cookie" header with a non-nil cookie Jar.
    51  // Since each redirect may mutate the state of the cookie jar,
    52  // a redirect may possibly alter a cookie set in the initial request.
    53  // When forwarding the "Cookie" header, any mutated cookies will be omitted,
    54  // with the expectation that the Jar will insert those mutated cookies
    55  // with the updated values (assuming the origin matches).
    56  // If Jar is nil, the initial cookies are forwarded without change.
    57  //
    58  type Client struct {
    59  	// Transport specifies the mechanism by which individual
    60  	// HTTP requests are made.
    61  	// If nil, DefaultTransport is used.
    62  	Transport RoundTripper
    63  
    64  	// CheckRedirect specifies the policy for handling redirects.
    65  	// If CheckRedirect is not nil, the client calls it before
    66  	// following an HTTP redirect. The arguments req and via are
    67  	// the upcoming request and the requests made already, oldest
    68  	// first. If CheckRedirect returns an error, the Client's Get
    69  	// method returns both the previous Response (with its Body
    70  	// closed) and CheckRedirect's error (wrapped in a url.Error)
    71  	// instead of issuing the Request req.
    72  	// As a special case, if CheckRedirect returns ErrUseLastResponse,
    73  	// then the most recent response is returned with its body
    74  	// unclosed, along with a nil error.
    75  	//
    76  	// If CheckRedirect is nil, the Client uses its default policy,
    77  	// which is to stop after 10 consecutive requests.
    78  	CheckRedirect func(req *Request, via []*Request) error
    79  
    80  	// Jar specifies the cookie jar.
    81  	//
    82  	// The Jar is used to insert relevant cookies into every
    83  	// outbound Request and is updated with the cookie values
    84  	// of every inbound Response. The Jar is consulted for every
    85  	// redirect that the Client follows.
    86  	//
    87  	// If Jar is nil, cookies are only sent if they are explicitly
    88  	// set on the Request.
    89  	Jar CookieJar
    90  
    91  	// Timeout specifies a time limit for requests made by this
    92  	// Client. The timeout includes connection time, any
    93  	// redirects, and reading the response body. The timer remains
    94  	// running after Get, Head, Post, or Do return and will
    95  	// interrupt reading of the Response.Body.
    96  	//
    97  	// A Timeout of zero means no timeout.
    98  	//
    99  	// The Client cancels requests to the underlying Transport
   100  	// as if the Request's Context ended.
   101  	//
   102  	// For compatibility, the Client will also use the deprecated
   103  	// CancelRequest method on Transport if found. New
   104  	// RoundTripper implementations should use the Request's Context
   105  	// for cancellation instead of implementing CancelRequest.
   106  	Timeout time.Duration
   107  }
   108  
   109  // DefaultClient is the default Client and is used by Get, Head, and Post.
   110  var DefaultClient = &Client{}
   111  
   112  // RoundTripper is an interface representing the ability to execute a
   113  // single HTTP transaction, obtaining the Response for a given Request.
   114  //
   115  // A RoundTripper must be safe for concurrent use by multiple
   116  // goroutines.
   117  type RoundTripper interface {
   118  	// RoundTrip executes a single HTTP transaction, returning
   119  	// a Response for the provided Request.
   120  	//
   121  	// RoundTrip should not attempt to interpret the response. In
   122  	// particular, RoundTrip must return err == nil if it obtained
   123  	// a response, regardless of the response's HTTP status code.
   124  	// A non-nil err should be reserved for failure to obtain a
   125  	// response. Similarly, RoundTrip should not attempt to
   126  	// handle higher-level protocol details such as redirects,
   127  	// authentication, or cookies.
   128  	//
   129  	// RoundTrip should not modify the request, except for
   130  	// consuming and closing the Request's Body. RoundTrip may
   131  	// read fields of the request in a separate goroutine. Callers
   132  	// should not mutate or reuse the request until the Response's
   133  	// Body has been closed.
   134  	//
   135  	// RoundTrip must always close the body, including on errors,
   136  	// but depending on the implementation may do so in a separate
   137  	// goroutine even after RoundTrip returns. This means that
   138  	// callers wanting to reuse the body for subsequent requests
   139  	// must arrange to wait for the Close call before doing so.
   140  	//
   141  	// The Request's URL and Header fields must be initialized.
   142  	RoundTrip(*Request) (*Response, error)
   143  }
   144  
   145  // refererForURL returns a referer without any authentication info or
   146  // an empty string if lastReq scheme is https and newReq scheme is http.
   147  func refererForURL(lastReq, newReq *url.URL) string {
   148  	// https://tools.ietf.org/html/rfc7231#section-5.5.2
   149  	//   "Clients SHOULD NOT include a Referer header field in a
   150  	//    (non-secure) HTTP request if the referring page was
   151  	//    transferred with a secure protocol."
   152  	if lastReq.Scheme == "https" && newReq.Scheme == "http" {
   153  		return ""
   154  	}
   155  	referer := lastReq.String()
   156  	if lastReq.User != nil {
   157  		// This is not very efficient, but is the best we can
   158  		// do without:
   159  		// - introducing a new method on URL
   160  		// - creating a race condition
   161  		// - copying the URL struct manually, which would cause
   162  		//   maintenance problems down the line
   163  		auth := lastReq.User.String() + "@"
   164  		referer = strings.Replace(referer, auth, "", 1)
   165  	}
   166  	return referer
   167  }
   168  
   169  // didTimeout is non-nil only if err != nil.
   170  func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
   171  	if c.Jar != nil {
   172  		for _, cookie := range c.Jar.Cookies(req.URL) {
   173  			req.AddCookie(cookie)
   174  		}
   175  	}
   176  	resp, didTimeout, err = send(req, c.transport(), deadline)
   177  	if err != nil {
   178  		return nil, didTimeout, err
   179  	}
   180  	if c.Jar != nil {
   181  		if rc := resp.Cookies(); len(rc) > 0 {
   182  			c.Jar.SetCookies(req.URL, rc)
   183  		}
   184  	}
   185  	return resp, nil, nil
   186  }
   187  
   188  func (c *Client) deadline() time.Time {
   189  	if c.Timeout > 0 {
   190  		return time.Now().Add(c.Timeout)
   191  	}
   192  	return time.Time{}
   193  }
   194  
   195  func (c *Client) transport() RoundTripper {
   196  	if c.Transport != nil {
   197  		return c.Transport
   198  	}
   199  	return DefaultTransport
   200  }
   201  
   202  // send issues an HTTP request.
   203  // Caller should close resp.Body when done reading from it.
   204  func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
   205  	req := ireq // req is either the original request, or a modified fork
   206  
   207  	if rt == nil {
   208  		req.closeBody()
   209  		return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport")
   210  	}
   211  
   212  	if req.URL == nil {
   213  		req.closeBody()
   214  		return nil, alwaysFalse, errors.New("http: nil Request.URL")
   215  	}
   216  
   217  	if req.RequestURI != "" {
   218  		req.closeBody()
   219  		return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests")
   220  	}
   221  
   222  	// forkReq forks req into a shallow clone of ireq the first
   223  	// time it's called.
   224  	forkReq := func() {
   225  		if ireq == req {
   226  			req = new(Request)
   227  			*req = *ireq // shallow clone
   228  		}
   229  	}
   230  
   231  	// Most the callers of send (Get, Post, et al) don't need
   232  	// Headers, leaving it uninitialized. We guarantee to the
   233  	// Transport that this has been initialized, though.
   234  	if req.Header == nil {
   235  		forkReq()
   236  		req.Header = make(Header)
   237  	}
   238  
   239  	if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
   240  		username := u.Username()
   241  		password, _ := u.Password()
   242  		forkReq()
   243  		req.Header = cloneOrMakeHeader(ireq.Header)
   244  		req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
   245  	}
   246  
   247  	if !deadline.IsZero() {
   248  		forkReq()
   249  	}
   250  	stopTimer, didTimeout := setRequestCancel(req, rt, deadline)
   251  
   252  	resp, err = rt.RoundTrip(req)
   253  	if err != nil {
   254  		stopTimer()
   255  		if resp != nil {
   256  			log.Printf("RoundTripper returned a response & error; ignoring response")
   257  		}
   258  		if tlsErr, ok := err.(tls.RecordHeaderError); ok {
   259  			// If we get a bad TLS record header, check to see if the
   260  			// response looks like HTTP and give a more helpful error.
   261  			// See golang.org/issue/11111.
   262  			if string(tlsErr.RecordHeader[:]) == "HTTP/" {
   263  				err = errors.New("http: server gave HTTP response to HTTPS client")
   264  			}
   265  		}
   266  		return nil, didTimeout, err
   267  	}
   268  	if resp == nil {
   269  		return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt)
   270  	}
   271  	if resp.Body == nil {
   272  		// The documentation on the Body field says “The http Client and Transport
   273  		// guarantee that Body is always non-nil, even on responses without a body
   274  		// or responses with a zero-length body.” Unfortunately, we didn't document
   275  		// that same constraint for arbitrary RoundTripper implementations, and
   276  		// RoundTripper implementations in the wild (mostly in tests) assume that
   277  		// they can use a nil Body to mean an empty one (similar to Request.Body).
   278  		// (See https://golang.org/issue/38095.)
   279  		//
   280  		// If the ContentLength allows the Body to be empty, fill in an empty one
   281  		// here to ensure that it is non-nil.
   282  		if resp.ContentLength > 0 && req.Method != "HEAD" {
   283  			return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength)
   284  		}
   285  		resp.Body = io.NopCloser(strings.NewReader(""))
   286  	}
   287  	if !deadline.IsZero() {
   288  		resp.Body = &cancelTimerBody{
   289  			stop:          stopTimer,
   290  			rc:            resp.Body,
   291  			reqDidTimeout: didTimeout,
   292  		}
   293  	}
   294  	return resp, nil, nil
   295  }
   296  
   297  // timeBeforeContextDeadline reports whether the non-zero Time t is
   298  // before ctx's deadline, if any. If ctx does not have a deadline, it
   299  // always reports true (the deadline is considered infinite).
   300  func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool {
   301  	d, ok := ctx.Deadline()
   302  	if !ok {
   303  		return true
   304  	}
   305  	return t.Before(d)
   306  }
   307  
   308  // knownRoundTripperImpl reports whether rt is a RoundTripper that's
   309  // maintained by the Go team and known to implement the latest
   310  // optional semantics (notably contexts). The Request is used
   311  // to check whether this particular request is using an alternate protocol,
   312  // in which case we need to check the RoundTripper for that protocol.
   313  func knownRoundTripperImpl(rt RoundTripper, req *Request) bool {
   314  	switch t := rt.(type) {
   315  	case *Transport:
   316  		if altRT := t.alternateRoundTripper(req); altRT != nil {
   317  			return knownRoundTripperImpl(altRT, req)
   318  		}
   319  		return true
   320  	case *http2Transport, http2noDialH2RoundTripper:
   321  		return true
   322  	}
   323  	// There's a very minor chance of a false positive with this.
   324  	// Instead of detecting our golang.org/x/net/http2.Transport,
   325  	// it might detect a Transport type in a different http2
   326  	// package. But I know of none, and the only problem would be
   327  	// some temporarily leaked goroutines if the transport didn't
   328  	// support contexts. So this is a good enough heuristic:
   329  	if reflect.TypeOf(rt).String() == "*http2.Transport" {
   330  		return true
   331  	}
   332  	return false
   333  }
   334  
   335  // setRequestCancel sets req.Cancel and adds a deadline context to req
   336  // if deadline is non-zero. The RoundTripper's type is used to
   337  // determine whether the legacy CancelRequest behavior should be used.
   338  //
   339  // As background, there are three ways to cancel a request:
   340  // First was Transport.CancelRequest. (deprecated)
   341  // Second was Request.Cancel.
   342  // Third was Request.Context.
   343  // This function populates the second and third, and uses the first if it really needs to.
   344  func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) {
   345  	if deadline.IsZero() {
   346  		return nop, alwaysFalse
   347  	}
   348  	knownTransport := knownRoundTripperImpl(rt, req)
   349  	oldCtx := req.Context()
   350  
   351  	if req.Cancel == nil && knownTransport {
   352  		// If they already had a Request.Context that's
   353  		// expiring sooner, do nothing:
   354  		if !timeBeforeContextDeadline(deadline, oldCtx) {
   355  			return nop, alwaysFalse
   356  		}
   357  
   358  		var cancelCtx func()
   359  		req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline)
   360  		return cancelCtx, func() bool { return time.Now().After(deadline) }
   361  	}
   362  	initialReqCancel := req.Cancel // the user's original Request.Cancel, if any
   363  
   364  	var cancelCtx func()
   365  	if oldCtx := req.Context(); timeBeforeContextDeadline(deadline, oldCtx) {
   366  		req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline)
   367  	}
   368  
   369  	cancel := make(chan struct{})
   370  	req.Cancel = cancel
   371  
   372  	doCancel := func() {
   373  		// The second way in the func comment above:
   374  		close(cancel)
   375  		// The first way, used only for RoundTripper
   376  		// implementations written before Go 1.5 or Go 1.6.
   377  		type canceler interface{ CancelRequest(*Request) }
   378  		if v, ok := rt.(canceler); ok {
   379  			v.CancelRequest(req)
   380  		}
   381  	}
   382  
   383  	stopTimerCh := make(chan struct{})
   384  	var once sync.Once
   385  	stopTimer = func() {
   386  		once.Do(func() {
   387  			close(stopTimerCh)
   388  			if cancelCtx != nil {
   389  				cancelCtx()
   390  			}
   391  		})
   392  	}
   393  
   394  	timer := time.NewTimer(time.Until(deadline))
   395  	var timedOut atomicBool
   396  
   397  	go func() {
   398  		select {
   399  		case <-initialReqCancel:
   400  			doCancel()
   401  			timer.Stop()
   402  		case <-timer.C:
   403  			timedOut.setTrue()
   404  			doCancel()
   405  		case <-stopTimerCh:
   406  			timer.Stop()
   407  		}
   408  	}()
   409  
   410  	return stopTimer, timedOut.isSet
   411  }
   412  
   413  // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt
   414  // "To receive authorization, the client sends the userid and password,
   415  // separated by a single colon (":") character, within a base64
   416  // encoded string in the credentials."
   417  // It is not meant to be urlencoded.
   418  func basicAuth(username, password string) string {
   419  	auth := username + ":" + password
   420  	return base64.StdEncoding.EncodeToString([]byte(auth))
   421  }
   422  
   423  // Get issues a GET to the specified URL. If the response is one of
   424  // the following redirect codes, Get follows the redirect, up to a
   425  // maximum of 10 redirects:
   426  //
   427  //    301 (Moved Permanently)
   428  //    302 (Found)
   429  //    303 (See Other)
   430  //    307 (Temporary Redirect)
   431  //    308 (Permanent Redirect)
   432  //
   433  // An error is returned if there were too many redirects or if there
   434  // was an HTTP protocol error. A non-2xx response doesn't cause an
   435  // error. Any returned error will be of type *url.Error. The url.Error
   436  // value's Timeout method will report true if the request timed out.
   437  //
   438  // When err is nil, resp always contains a non-nil resp.Body.
   439  // Caller should close resp.Body when done reading from it.
   440  //
   441  // Get is a wrapper around DefaultClient.Get.
   442  //
   443  // To make a request with custom headers, use NewRequest and
   444  // DefaultClient.Do.
   445  //
   446  // To make a request with a specified context.Context, use NewRequestWithContext
   447  // and DefaultClient.Do.
   448  func Get(url string) (resp *Response, err error) {
   449  	return DefaultClient.Get(url)
   450  }
   451  
   452  // Get issues a GET to the specified URL. If the response is one of the
   453  // following redirect codes, Get follows the redirect after calling the
   454  // Client's CheckRedirect function:
   455  //
   456  //    301 (Moved Permanently)
   457  //    302 (Found)
   458  //    303 (See Other)
   459  //    307 (Temporary Redirect)
   460  //    308 (Permanent Redirect)
   461  //
   462  // An error is returned if the Client's CheckRedirect function fails
   463  // or if there was an HTTP protocol error. A non-2xx response doesn't
   464  // cause an error. Any returned error will be of type *url.Error. The
   465  // url.Error value's Timeout method will report true if the request
   466  // timed out.
   467  //
   468  // When err is nil, resp always contains a non-nil resp.Body.
   469  // Caller should close resp.Body when done reading from it.
   470  //
   471  // To make a request with custom headers, use NewRequest and Client.Do.
   472  //
   473  // To make a request with a specified context.Context, use NewRequestWithContext
   474  // and Client.Do.
   475  func (c *Client) Get(url string) (resp *Response, err error) {
   476  	req, err := NewRequest("GET", url, nil)
   477  	if err != nil {
   478  		return nil, err
   479  	}
   480  	return c.Do(req)
   481  }
   482  
   483  func alwaysFalse() bool { return false }
   484  
   485  // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to
   486  // control how redirects are processed. If returned, the next request
   487  // is not sent and the most recent response is returned with its body
   488  // unclosed.
   489  var ErrUseLastResponse = errors.New("net/http: use last response")
   490  
   491  // checkRedirect calls either the user's configured CheckRedirect
   492  // function, or the default.
   493  func (c *Client) checkRedirect(req *Request, via []*Request) error {
   494  	fn := c.CheckRedirect
   495  	if fn == nil {
   496  		fn = defaultCheckRedirect
   497  	}
   498  	return fn(req, via)
   499  }
   500  
   501  // redirectBehavior describes what should happen when the
   502  // client encounters a 3xx status code from the server
   503  func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) {
   504  	switch resp.StatusCode {
   505  	case 301, 302, 303:
   506  		redirectMethod = reqMethod
   507  		shouldRedirect = true
   508  		includeBody = false
   509  
   510  		// RFC 2616 allowed automatic redirection only with GET and
   511  		// HEAD requests. RFC 7231 lifts this restriction, but we still
   512  		// restrict other methods to GET to maintain compatibility.
   513  		// See Issue 18570.
   514  		if reqMethod != "GET" && reqMethod != "HEAD" {
   515  			redirectMethod = "GET"
   516  		}
   517  	case 307, 308:
   518  		redirectMethod = reqMethod
   519  		shouldRedirect = true
   520  		includeBody = true
   521  
   522  		// Treat 307 and 308 specially, since they're new in
   523  		// Go 1.8, and they also require re-sending the request body.
   524  		if resp.Header.Get("Location") == "" {
   525  			// 308s have been observed in the wild being served
   526  			// without Location headers. Since Go 1.7 and earlier
   527  			// didn't follow these codes, just stop here instead
   528  			// of returning an error.
   529  			// See Issue 17773.
   530  			shouldRedirect = false
   531  			break
   532  		}
   533  		if ireq.GetBody == nil && ireq.outgoingLength() != 0 {
   534  			// We had a request body, and 307/308 require
   535  			// re-sending it, but GetBody is not defined. So just
   536  			// return this response to the user instead of an
   537  			// error, like we did in Go 1.7 and earlier.
   538  			shouldRedirect = false
   539  		}
   540  	}
   541  	return redirectMethod, shouldRedirect, includeBody
   542  }
   543  
   544  // urlErrorOp returns the (*url.Error).Op value to use for the
   545  // provided (*Request).Method value.
   546  func urlErrorOp(method string) string {
   547  	if method == "" {
   548  		return "Get"
   549  	}
   550  	if lowerMethod, ok := ascii.ToLower(method); ok {
   551  		return method[:1] + lowerMethod[1:]
   552  	}
   553  	return method
   554  }
   555  
   556  // Do sends an HTTP request and returns an HTTP response, following
   557  // policy (such as redirects, cookies, auth) as configured on the
   558  // client.
   559  //
   560  // An error is returned if caused by client policy (such as
   561  // CheckRedirect), or failure to speak HTTP (such as a network
   562  // connectivity problem). A non-2xx status code doesn't cause an
   563  // error.
   564  //
   565  // If the returned error is nil, the Response will contain a non-nil
   566  // Body which the user is expected to close. If the Body is not both
   567  // read to EOF and closed, the Client's underlying RoundTripper
   568  // (typically Transport) may not be able to re-use a persistent TCP
   569  // connection to the server for a subsequent "keep-alive" request.
   570  //
   571  // The request Body, if non-nil, will be closed by the underlying
   572  // Transport, even on errors.
   573  //
   574  // On error, any Response can be ignored. A non-nil Response with a
   575  // non-nil error only occurs when CheckRedirect fails, and even then
   576  // the returned Response.Body is already closed.
   577  //
   578  // Generally Get, Post, or PostForm will be used instead of Do.
   579  //
   580  // If the server replies with a redirect, the Client first uses the
   581  // CheckRedirect function to determine whether the redirect should be
   582  // followed. If permitted, a 301, 302, or 303 redirect causes
   583  // subsequent requests to use HTTP method GET
   584  // (or HEAD if the original request was HEAD), with no body.
   585  // A 307 or 308 redirect preserves the original HTTP method and body,
   586  // provided that the Request.GetBody function is defined.
   587  // The NewRequest function automatically sets GetBody for common
   588  // standard library body types.
   589  //
   590  // Any returned error will be of type *url.Error. The url.Error
   591  // value's Timeout method will report true if the request timed out.
   592  func (c *Client) Do(req *Request) (*Response, error) {
   593  	return c.do(req)
   594  }
   595  
   596  var testHookClientDoResult func(retres *Response, reterr error)
   597  
   598  func (c *Client) do(req *Request) (retres *Response, reterr error) {
   599  	if testHookClientDoResult != nil {
   600  		defer func() { testHookClientDoResult(retres, reterr) }()
   601  	}
   602  	if req.URL == nil {
   603  		req.closeBody()
   604  		return nil, &url.Error{
   605  			Op:  urlErrorOp(req.Method),
   606  			Err: errors.New("http: nil Request.URL"),
   607  		}
   608  	}
   609  
   610  	var (
   611  		deadline      = c.deadline()
   612  		reqs          []*Request
   613  		resp          *Response
   614  		copyHeaders   = c.makeHeadersCopier(req)
   615  		reqBodyClosed = false // have we closed the current req.Body?
   616  
   617  		// Redirect behavior:
   618  		redirectMethod string
   619  		includeBody    bool
   620  	)
   621  	uerr := func(err error) error {
   622  		// the body may have been closed already by c.send()
   623  		if !reqBodyClosed {
   624  			req.closeBody()
   625  		}
   626  		var urlStr string
   627  		if resp != nil && resp.Request != nil {
   628  			urlStr = stripPassword(resp.Request.URL)
   629  		} else {
   630  			urlStr = stripPassword(req.URL)
   631  		}
   632  		return &url.Error{
   633  			Op:  urlErrorOp(reqs[0].Method),
   634  			URL: urlStr,
   635  			Err: err,
   636  		}
   637  	}
   638  	for {
   639  		// For all but the first request, create the next
   640  		// request hop and replace req.
   641  		if len(reqs) > 0 {
   642  			loc := resp.Header.Get("Location")
   643  			if loc == "" {
   644  				resp.closeBody()
   645  				return nil, uerr(fmt.Errorf("%d response missing Location header", resp.StatusCode))
   646  			}
   647  			u, err := req.URL.Parse(loc)
   648  			if err != nil {
   649  				resp.closeBody()
   650  				return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err))
   651  			}
   652  			host := ""
   653  			if req.Host != "" && req.Host != req.URL.Host {
   654  				// If the caller specified a custom Host header and the
   655  				// redirect location is relative, preserve the Host header
   656  				// through the redirect. See issue #22233.
   657  				if u, _ := url.Parse(loc); u != nil && !u.IsAbs() {
   658  					host = req.Host
   659  				}
   660  			}
   661  			ireq := reqs[0]
   662  			req = &Request{
   663  				Method:   redirectMethod,
   664  				Response: resp,
   665  				URL:      u,
   666  				Header:   make(Header),
   667  				Host:     host,
   668  				Cancel:   ireq.Cancel,
   669  				ctx:      ireq.ctx,
   670  			}
   671  			if includeBody && ireq.GetBody != nil {
   672  				req.Body, err = ireq.GetBody()
   673  				if err != nil {
   674  					resp.closeBody()
   675  					return nil, uerr(err)
   676  				}
   677  				req.ContentLength = ireq.ContentLength
   678  			}
   679  
   680  			// Copy original headers before setting the Referer,
   681  			// in case the user set Referer on their first request.
   682  			// If they really want to override, they can do it in
   683  			// their CheckRedirect func.
   684  			copyHeaders(req)
   685  
   686  			// Add the Referer header from the most recent
   687  			// request URL to the new one, if it's not https->http:
   688  			if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" {
   689  				req.Header.Set("Referer", ref)
   690  			}
   691  			err = c.checkRedirect(req, reqs)
   692  
   693  			// Sentinel error to let users select the
   694  			// previous response, without closing its
   695  			// body. See Issue 10069.
   696  			if err == ErrUseLastResponse {
   697  				return resp, nil
   698  			}
   699  
   700  			// Close the previous response's body. But
   701  			// read at least some of the body so if it's
   702  			// small the underlying TCP connection will be
   703  			// re-used. No need to check for errors: if it
   704  			// fails, the Transport won't reuse it anyway.
   705  			const maxBodySlurpSize = 2 << 10
   706  			if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
   707  				io.CopyN(io.Discard, resp.Body, maxBodySlurpSize)
   708  			}
   709  			resp.Body.Close()
   710  
   711  			if err != nil {
   712  				// Special case for Go 1 compatibility: return both the response
   713  				// and an error if the CheckRedirect function failed.
   714  				// See https://golang.org/issue/3795
   715  				// The resp.Body has already been closed.
   716  				ue := uerr(err)
   717  				ue.(*url.Error).URL = loc
   718  				return resp, ue
   719  			}
   720  		}
   721  
   722  		reqs = append(reqs, req)
   723  		var err error
   724  		var didTimeout func() bool
   725  		if resp, didTimeout, err = c.send(req, deadline); err != nil {
   726  			// c.send() always closes req.Body
   727  			reqBodyClosed = true
   728  			if !deadline.IsZero() && didTimeout() {
   729  				err = &httpError{
   730  					err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
   731  					timeout: true,
   732  				}
   733  			}
   734  			return nil, uerr(err)
   735  		}
   736  
   737  		var shouldRedirect bool
   738  		redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0])
   739  		if !shouldRedirect {
   740  			return resp, nil
   741  		}
   742  
   743  		req.closeBody()
   744  	}
   745  }
   746  
   747  // makeHeadersCopier makes a function that copies headers from the
   748  // initial Request, ireq. For every redirect, this function must be called
   749  // so that it can copy headers into the upcoming Request.
   750  func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) {
   751  	// The headers to copy are from the very initial request.
   752  	// We use a closured callback to keep a reference to these original headers.
   753  	var (
   754  		ireqhdr  = cloneOrMakeHeader(ireq.Header)
   755  		icookies map[string][]*Cookie
   756  	)
   757  	if c.Jar != nil && ireq.Header.Get("Cookie") != "" {
   758  		icookies = make(map[string][]*Cookie)
   759  		for _, c := range ireq.Cookies() {
   760  			icookies[c.Name] = append(icookies[c.Name], c)
   761  		}
   762  	}
   763  
   764  	preq := ireq // The previous request
   765  	return func(req *Request) {
   766  		// If Jar is present and there was some initial cookies provided
   767  		// via the request header, then we may need to alter the initial
   768  		// cookies as we follow redirects since each redirect may end up
   769  		// modifying a pre-existing cookie.
   770  		//
   771  		// Since cookies already set in the request header do not contain
   772  		// information about the original domain and path, the logic below
   773  		// assumes any new set cookies override the original cookie
   774  		// regardless of domain or path.
   775  		//
   776  		// See https://golang.org/issue/17494
   777  		if c.Jar != nil && icookies != nil {
   778  			var changed bool
   779  			resp := req.Response // The response that caused the upcoming redirect
   780  			for _, c := range resp.Cookies() {
   781  				if _, ok := icookies[c.Name]; ok {
   782  					delete(icookies, c.Name)
   783  					changed = true
   784  				}
   785  			}
   786  			if changed {
   787  				ireqhdr.Del("Cookie")
   788  				var ss []string
   789  				for _, cs := range icookies {
   790  					for _, c := range cs {
   791  						ss = append(ss, c.Name+"="+c.Value)
   792  					}
   793  				}
   794  				sort.Strings(ss) // Ensure deterministic headers
   795  				ireqhdr.Set("Cookie", strings.Join(ss, "; "))
   796  			}
   797  		}
   798  
   799  		// Copy the initial request's Header values
   800  		// (at least the safe ones).
   801  		for k, vv := range ireqhdr {
   802  			if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) {
   803  				req.Header[k] = vv
   804  			}
   805  		}
   806  
   807  		preq = req // Update previous Request with the current request
   808  	}
   809  }
   810  
   811  func defaultCheckRedirect(req *Request, via []*Request) error {
   812  	if len(via) >= 10 {
   813  		return errors.New("stopped after 10 redirects")
   814  	}
   815  	return nil
   816  }
   817  
   818  // Post issues a POST to the specified URL.
   819  //
   820  // Caller should close resp.Body when done reading from it.
   821  //
   822  // If the provided body is an io.Closer, it is closed after the
   823  // request.
   824  //
   825  // Post is a wrapper around DefaultClient.Post.
   826  //
   827  // To set custom headers, use NewRequest and DefaultClient.Do.
   828  //
   829  // See the Client.Do method documentation for details on how redirects
   830  // are handled.
   831  //
   832  // To make a request with a specified context.Context, use NewRequestWithContext
   833  // and DefaultClient.Do.
   834  func Post(url, contentType string, body io.Reader) (resp *Response, err error) {
   835  	return DefaultClient.Post(url, contentType, body)
   836  }
   837  
   838  // Post issues a POST to the specified URL.
   839  //
   840  // Caller should close resp.Body when done reading from it.
   841  //
   842  // If the provided body is an io.Closer, it is closed after the
   843  // request.
   844  //
   845  // To set custom headers, use NewRequest and Client.Do.
   846  //
   847  // To make a request with a specified context.Context, use NewRequestWithContext
   848  // and Client.Do.
   849  //
   850  // See the Client.Do method documentation for details on how redirects
   851  // are handled.
   852  func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) {
   853  	req, err := NewRequest("POST", url, body)
   854  	if err != nil {
   855  		return nil, err
   856  	}
   857  	req.Header.Set("Content-Type", contentType)
   858  	return c.Do(req)
   859  }
   860  
   861  // PostForm issues a POST to the specified URL, with data's keys and
   862  // values URL-encoded as the request body.
   863  //
   864  // The Content-Type header is set to application/x-www-form-urlencoded.
   865  // To set other headers, use NewRequest and DefaultClient.Do.
   866  //
   867  // When err is nil, resp always contains a non-nil resp.Body.
   868  // Caller should close resp.Body when done reading from it.
   869  //
   870  // PostForm is a wrapper around DefaultClient.PostForm.
   871  //
   872  // See the Client.Do method documentation for details on how redirects
   873  // are handled.
   874  //
   875  // To make a request with a specified context.Context, use NewRequestWithContext
   876  // and DefaultClient.Do.
   877  func PostForm(url string, data url.Values) (resp *Response, err error) {
   878  	return DefaultClient.PostForm(url, data)
   879  }
   880  
   881  // PostForm issues a POST to the specified URL,
   882  // with data's keys and values URL-encoded as the request body.
   883  //
   884  // The Content-Type header is set to application/x-www-form-urlencoded.
   885  // To set other headers, use NewRequest and Client.Do.
   886  //
   887  // When err is nil, resp always contains a non-nil resp.Body.
   888  // Caller should close resp.Body when done reading from it.
   889  //
   890  // See the Client.Do method documentation for details on how redirects
   891  // are handled.
   892  //
   893  // To make a request with a specified context.Context, use NewRequestWithContext
   894  // and Client.Do.
   895  func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
   896  	return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   897  }
   898  
   899  // Head issues a HEAD to the specified URL. If the response is one of
   900  // the following redirect codes, Head follows the redirect, up to a
   901  // maximum of 10 redirects:
   902  //
   903  //    301 (Moved Permanently)
   904  //    302 (Found)
   905  //    303 (See Other)
   906  //    307 (Temporary Redirect)
   907  //    308 (Permanent Redirect)
   908  //
   909  // Head is a wrapper around DefaultClient.Head
   910  //
   911  // To make a request with a specified context.Context, use NewRequestWithContext
   912  // and DefaultClient.Do.
   913  func Head(url string) (resp *Response, err error) {
   914  	return DefaultClient.Head(url)
   915  }
   916  
   917  // Head issues a HEAD to the specified URL. If the response is one of the
   918  // following redirect codes, Head follows the redirect after calling the
   919  // Client's CheckRedirect function:
   920  //
   921  //    301 (Moved Permanently)
   922  //    302 (Found)
   923  //    303 (See Other)
   924  //    307 (Temporary Redirect)
   925  //    308 (Permanent Redirect)
   926  //
   927  // To make a request with a specified context.Context, use NewRequestWithContext
   928  // and Client.Do.
   929  func (c *Client) Head(url string) (resp *Response, err error) {
   930  	req, err := NewRequest("HEAD", url, nil)
   931  	if err != nil {
   932  		return nil, err
   933  	}
   934  	return c.Do(req)
   935  }
   936  
   937  // CloseIdleConnections closes any connections on its Transport which
   938  // were previously connected from previous requests but are now
   939  // sitting idle in a "keep-alive" state. It does not interrupt any
   940  // connections currently in use.
   941  //
   942  // If the Client's Transport does not have a CloseIdleConnections method
   943  // then this method does nothing.
   944  func (c *Client) CloseIdleConnections() {
   945  	type closeIdler interface {
   946  		CloseIdleConnections()
   947  	}
   948  	if tr, ok := c.transport().(closeIdler); ok {
   949  		tr.CloseIdleConnections()
   950  	}
   951  }
   952  
   953  // cancelTimerBody is an io.ReadCloser that wraps rc with two features:
   954  // 1) On Read error or close, the stop func is called.
   955  // 2) On Read failure, if reqDidTimeout is true, the error is wrapped and
   956  //    marked as net.Error that hit its timeout.
   957  type cancelTimerBody struct {
   958  	stop          func() // stops the time.Timer waiting to cancel the request
   959  	rc            io.ReadCloser
   960  	reqDidTimeout func() bool
   961  }
   962  
   963  func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
   964  	n, err = b.rc.Read(p)
   965  	if err == nil {
   966  		return n, nil
   967  	}
   968  	b.stop()
   969  	if err == io.EOF {
   970  		return n, err
   971  	}
   972  	if b.reqDidTimeout() {
   973  		err = &httpError{
   974  			err:     err.Error() + " (Client.Timeout or context cancellation while reading body)",
   975  			timeout: true,
   976  		}
   977  	}
   978  	return n, err
   979  }
   980  
   981  func (b *cancelTimerBody) Close() error {
   982  	err := b.rc.Close()
   983  	b.stop()
   984  	return err
   985  }
   986  
   987  func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool {
   988  	switch CanonicalHeaderKey(headerKey) {
   989  	case "Authorization", "Www-Authenticate", "Cookie", "Cookie2":
   990  		// Permit sending auth/cookie headers from "foo.com"
   991  		// to "sub.foo.com".
   992  
   993  		// Note that we don't send all cookies to subdomains
   994  		// automatically. This function is only used for
   995  		// Cookies set explicitly on the initial outgoing
   996  		// client request. Cookies automatically added via the
   997  		// CookieJar mechanism continue to follow each
   998  		// cookie's scope as set by Set-Cookie. But for
   999  		// outgoing requests with the Cookie header set
  1000  		// directly, we don't know their scope, so we assume
  1001  		// it's for *.domain.com.
  1002  
  1003  		ihost := canonicalAddr(initial)
  1004  		dhost := canonicalAddr(dest)
  1005  		return isDomainOrSubdomain(dhost, ihost)
  1006  	}
  1007  	// All other headers are copied:
  1008  	return true
  1009  }
  1010  
  1011  // isDomainOrSubdomain reports whether sub is a subdomain (or exact
  1012  // match) of the parent domain.
  1013  //
  1014  // Both domains must already be in canonical form.
  1015  func isDomainOrSubdomain(sub, parent string) bool {
  1016  	if sub == parent {
  1017  		return true
  1018  	}
  1019  	// If sub is "foo.example.com" and parent is "example.com",
  1020  	// that means sub must end in "."+parent.
  1021  	// Do it without allocating.
  1022  	if !strings.HasSuffix(sub, parent) {
  1023  		return false
  1024  	}
  1025  	return sub[len(sub)-len(parent)-1] == '.'
  1026  }
  1027  
  1028  func stripPassword(u *url.URL) string {
  1029  	_, passSet := u.User.Password()
  1030  	if passSet {
  1031  		return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1)
  1032  	}
  1033  	return u.String()
  1034  }
  1035  

View as plain text