Skip to content

Instantly share code, notes, and snippets.

@girorme
Last active May 8, 2024 18:24
Show Gist options
  • Save girorme/e76995cab4e4406724bf530b98542fdc to your computer and use it in GitHub Desktop.
Save girorme/e76995cab4e4406724bf530b98542fdc to your computer and use it in GitHub Desktop.
worker pool using go channels + waitGroup
package main
import (
"flag"
"fmt"
"net/http"
"sync"
"threadsync/httpclient"
"time"
)
type HttpClient struct {
}
func (h *HttpClient) Get(url string) string {
fmt.Printf("Getting url: %s\n", url)
resp, err := http.Get(url)
if err != nil {
return fmt.Sprintf("[-][%s] err: %s\n", url, err.Error())
}
return fmt.Sprintf("[+][%s] http_code %d - %s \n", url, resp.StatusCode, http.StatusText(resp.StatusCode))
}
func worker(wg *sync.WaitGroup, urlChannel <-chan string, resultChannel chan string) {
defer wg.Done()
client := new(httpclient.HttpClient)
for url := range urlChannel {
resultChannel <- client.Get(url)
}
}
func main() {
start := time.Now()
urls := []string{
"https://google.com.br",
"https://twitter.com",
"https://mj-go.in/golang/async-http-requests-in-go",
"https://wordpress.org",
"https://cloudflare.com",
"https://docs.google.com",
"https://mozilla.org",
"https://en.wikipedia.org",
"https://maps.google.com",
"https://accounts.google.com",
"https://googleusercontent.com",
"https://drive.google.com",
"https://sites.google.com",
"https://adobe.com",
"https://plus.google.com1",
"https://europa.eu",
"https://bbc.co.uk",
"https://vk.com",
"https://es.wikipedia.org",
}
wg := new(sync.WaitGroup)
threads := flag.Int("t", 5, "Number of threads")
flag.Parse()
wg.Add(*threads)
urlChannel := make(chan string)
resultChannel := make(chan string, len(urls))
// Start the workers
for i := 0; i < *threads; i++ {
go worker(wg, urlChannel, resultChannel)
}
// Send jobs to worker
for _, url := range urls {
urlChannel <- url
}
close(urlChannel)
wg.Wait()
close(resultChannel)
var results []string
// receive results from workes
for result := range resultChannel {
results = append(results, result)
}
fmt.Println(results)
fmt.Printf("Took %s\n", time.Since(start))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment