Extract Subdoamins from Project Sonar dataset

sub-extraction.go

package main

import (
	"bufio"
	"fmt"
	"github.com/schollz/progressbar"
	"github.com/valyala/fastjson"
  "github.com/Cgboal/DomainParser"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

var output_wg sync.WaitGroup

func ParseFile(file *os.File, bar *progressbar.ProgressBar) {
	var wg sync.WaitGroup
        domainParser := parser.NewDomainParser()
	scanner := bufio.NewScanner(file)
	linesChannel := make(chan string, 10000)
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			var p fastjson.Parser
			for line := range linesChannel {
				v, _ := p.Parse(line)
				name := string(v.GetStringBytes("name"))
                                fmt.Println(domainParser.GetSubdomain(name))
			}
		}()
	}

	for scanner.Scan() {
		bar.Write([]byte(scanner.Text()))
		line := strings.TrimSpace(scanner.Text())
		linesChannel <- line
	}
	wg.Wait()

}

func main() {
        filename := os.Args[1]
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}

	fi, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}

	bar := progressbar.NewOptions64(
		fi.Size(),
		progressbar.OptionThrottle(20*time.Second),
		progressbar.OptionSetWriter(os.Stderr),
		progressbar.OptionShowBytes(true),
		progressbar.OptionSetWidth(10),
		progressbar.OptionShowCount(),
		progressbar.OptionOnCompletion(func() {
			fmt.Fprint(os.Stderr, "\n")
		}),
		progressbar.OptionSpinnerType(14),
		progressbar.OptionFullWidth(),
	)

	output_wg.Add(1)
	ParseFile(file, bar)
	output_wg.Wait()
}