Last active
May 8, 2024 18:24
-
-
Save girorme/e76995cab4e4406724bf530b98542fdc to your computer and use it in GitHub Desktop.
worker pool using go channels + waitGroup
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"flag" | |
"fmt" | |
"net/http" | |
"sync" | |
"threadsync/httpclient" | |
"time" | |
) | |
type HttpClient struct { | |
} | |
func (h *HttpClient) Get(url string) string { | |
fmt.Printf("Getting url: %s\n", url) | |
resp, err := http.Get(url) | |
if err != nil { | |
return fmt.Sprintf("[-][%s] err: %s\n", url, err.Error()) | |
} | |
return fmt.Sprintf("[+][%s] http_code %d - %s \n", url, resp.StatusCode, http.StatusText(resp.StatusCode)) | |
} | |
func worker(wg *sync.WaitGroup, urlChannel <-chan string, resultChannel chan string) { | |
defer wg.Done() | |
client := new(httpclient.HttpClient) | |
for url := range urlChannel { | |
resultChannel <- client.Get(url) | |
} | |
} | |
func main() { | |
start := time.Now() | |
urls := []string{ | |
"https://google.com.br", | |
"https://twitter.com", | |
"https://mj-go.in/golang/async-http-requests-in-go", | |
"https://wordpress.org", | |
"https://cloudflare.com", | |
"https://docs.google.com", | |
"https://mozilla.org", | |
"https://en.wikipedia.org", | |
"https://maps.google.com", | |
"https://accounts.google.com", | |
"https://googleusercontent.com", | |
"https://drive.google.com", | |
"https://sites.google.com", | |
"https://adobe.com", | |
"https://plus.google.com1", | |
"https://europa.eu", | |
"https://bbc.co.uk", | |
"https://vk.com", | |
"https://es.wikipedia.org", | |
} | |
wg := new(sync.WaitGroup) | |
threads := flag.Int("t", 5, "Number of threads") | |
flag.Parse() | |
wg.Add(*threads) | |
urlChannel := make(chan string) | |
resultChannel := make(chan string, len(urls)) | |
// Start the workers | |
for i := 0; i < *threads; i++ { | |
go worker(wg, urlChannel, resultChannel) | |
} | |
// Send jobs to worker | |
for _, url := range urls { | |
urlChannel <- url | |
} | |
close(urlChannel) | |
wg.Wait() | |
close(resultChannel) | |
var results []string | |
// receive results from workes | |
for result := range resultChannel { | |
results = append(results, result) | |
} | |
fmt.Println(results) | |
fmt.Printf("Took %s\n", time.Since(start)) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment