Put Accept header before OnRequest (#786)

Closes #783

colly.go

@@ -671,6 +671,10 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct
 		ID:        atomic.AddUint32(&c.requestCount, 1),
 	}
 
+	if req.Header.Get("Accept") == "" {
+		req.Header.Set("Accept", "*/*")
+	}
+
 	c.handleOnRequest(request)
 
 	if request.abort {
@@ -681,10 +685,6 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct
 		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 	}
 
-	if req.Header.Get("Accept") == "" {
-		req.Header.Set("Accept", "*/*")
-	}
-
 	var hTrace *HTTPTrace
 	if c.TraceHTTP {
 		hTrace = &HTTPTrace{}

colly_test.go

@@ -147,6 +147,11 @@ func newUnstartedTestServer() *httptest.Server {
 		w.Write([]byte(r.Host))
 	})
 
+	mux.HandleFunc("/accept_header", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Write([]byte(r.Header.Get("Accept")))
+	})
+
 	mux.HandleFunc("/custom_header", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(200)
 		w.Write([]byte(r.Header.Get("Test")))
@@ -424,6 +429,39 @@ var newCollectorTests = map[string]func(*testing.T){
 	},
 }
 
+func TestNoAcceptHeader(t *testing.T) {
+	ts := newTestServer()
+	defer ts.Close()
+
+	var receivedHeader string
+	// checks if Accept is enabled by default
+	func() {
+		c := NewCollector()
+		c.OnResponse(func(resp *Response) {
+			receivedHeader = string(resp.Body)
+		})
+		c.Visit(ts.URL + "/accept_header")
+		if receivedHeader != "*/*" {
+			t.Errorf("default Accept header isn't */*. got: %v", receivedHeader)
+		}
+	}()
+
+	// checks if Accept can be disabled
+	func() {
+		c := NewCollector()
+		c.OnRequest(func(r *Request) {
+			r.Headers.Del("Accept")
+		})
+		c.OnResponse(func(resp *Response) {
+			receivedHeader = string(resp.Body)
+		})
+		c.Visit(ts.URL + "/accept_header")
+		if receivedHeader != "" {
+			t.Errorf("failed to pass request with no Accept header. got: %v", receivedHeader)
+		}
+	}()
+}
+
 func TestNewCollector(t *testing.T) {
 	t.Run("Functional Options", func(t *testing.T) {
 		for name, test := range newCollectorTests {