-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #64 from containscafeine/specify_url
add URL support
- Loading branch information
Showing
4 changed files
with
156 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| package util | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io/ioutil" | ||
| "net/http" | ||
| "net/url" | ||
| "time" | ||
| ) | ||
|
|
||
| // This function validates the URL, then tries to fetch the data with retries | ||
| // and then reads and returns the data as []byte | ||
| // Returns an error if the URL is invalid, or fetching the data failed or | ||
| // if reading the response body fails. | ||
| func GetURLData(urlString string, attempts int) ([]byte, error) { | ||
| // Validate URL | ||
| _, err := url.ParseRequestURI(urlString) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid URL: %v", urlString) | ||
| } | ||
|
|
||
| // Fetch the URL and store the response body | ||
| data, err := FetchURLWithRetries(urlString, attempts, 1*time.Second) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed fetching data from the URL %v: %s", urlString, err) | ||
| } | ||
|
|
||
| return data, nil | ||
| } | ||
|
|
||
| // Try to fetch the given url string, and make <attempts> attempts at it. | ||
| // Wait for <duration> time between each try. | ||
| // This returns the data from the response body []byte upon successful fetch | ||
| // The passed URL is not validated, so validate the URL before passing to this | ||
| // function | ||
| func FetchURLWithRetries(url string, attempts int, duration time.Duration) ([]byte, error) { | ||
| var data []byte | ||
| var err error | ||
|
|
||
| for i := 0; i < attempts; i++ { | ||
| var response *http.Response | ||
|
|
||
| // sleep for <duration> seconds before trying again | ||
| if i > 0 { | ||
| time.Sleep(duration) | ||
| } | ||
|
|
||
| // retry if http.Get fails | ||
| // if all the retries fail, then return statement at the end of the | ||
| // function will return this err received from http.Get | ||
| response, err = http.Get(url) | ||
| if err != nil { | ||
| continue | ||
| } | ||
| defer response.Body.Close() | ||
|
|
||
| // if the status code is not 200 OK, return an error | ||
| if response.StatusCode != http.StatusOK { | ||
| return nil, fmt.Errorf("unable to fetch %v, server returned status code %v", url, response.StatusCode) | ||
| } | ||
|
|
||
| // Read from the response body, ioutil.ReadAll will return []byte | ||
| data, err = ioutil.ReadAll(response.Body) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("reading from response body failed: %s", err) | ||
| } | ||
| break | ||
| } | ||
|
|
||
| return data, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package util | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| const ( | ||
| retryAttempts = 3 | ||
| ) | ||
|
|
||
| func TestGetURLData(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| succeed bool | ||
| url string | ||
| }{ | ||
| {"passing a valid URL", true, "http://example.com"}, | ||
| {"passing an invalid URL", false, "invalid.url.^&*!@#"}, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| _, err := GetURLData(tt.url, retryAttempts) | ||
|
|
||
| // error occurred but test was expected to pass | ||
| if err != nil && tt.succeed { | ||
| t.Fatalf("GetURLData was expected to succeed for the URL: %v, but it failed with the error: %v", tt.url, err) | ||
| } | ||
|
|
||
| // no error occurred but test was expected to fail | ||
| if err == nil && !tt.succeed { | ||
| t.Fatalf("GetURLData was expected to fail for the URL: %v, but it passed", tt.url) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestFetchURLWithRetries(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| succeed bool | ||
| url string | ||
| }{ | ||
| {"passing a valid URL", true, "https://example.com/"}, | ||
| {"passing a URL with no DNS resolution", false, "https://invalid.example/"}, | ||
| {"passing a blank string as URL", false, ""}, | ||
| {"passing a URL which gives a non 200 OK status code (gives 404)", false, "https://google.com/giveme404"}, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
|
|
||
| _, err := FetchURLWithRetries(tt.url, retryAttempts, time.Second) | ||
|
|
||
| // error occurred but tt was expected to pass | ||
| if err != nil && tt.succeed { | ||
| t.Fatalf("FetchURLWithRetries was expected to succeed for the URL: %v, but it failed with the error: %v", tt.url, err) | ||
| } | ||
|
|
||
| // no error occurred but test was expected to fail | ||
| if err == nil && !tt.succeed { | ||
| t.Fatalf("FetchURLWithRetries was expected to fail for the URL: %v, but it passed", tt.url) | ||
| } | ||
| }) | ||
| } | ||
| } |