Iterating over multiple returns in golang

Question

I am trying to take input from a text file containing domains(unknown amount) to then use each as an argument and get their server type. As expected, this only returns the last domain. How do I iterating multiple return values? Below is the code. // Test package main

import (
    "bufio"
    "time"
    "os"
    "fmt"
    "net/http"
    //"github.com/gocolly/colly"
)

var Domain string
var Target string

func main() {
    Domain := DomainGrab()
    Target := BannerGrab(Domain)
    //CheckDB if not listed then add else skip
    //RiskDB
    //Email
    fmt.Println(Domain)
    fmt.Println(Target)
}

func BannerGrab(s string) string {

    client := &http.Client{}
    req, err := http.NewRequest("GET", s, nil)
    if err != nil {
    log.Fatalln(err)
    }
    req.Header.Set("User-Agent", "Authac/0.1")
    resp, _ := client.Do(req)
    serverEntry := resp.Header.Get("Server")
    return serverEntry

}
func DomainGrab() string {

    //c := colly.NewCollector()
// Open the file.
    f, _ := os.Open("domains.txt")
    defer f.Close()
    // Create a new Scanner for the file.
    scanner := bufio.NewScanner(f)
    // Loop over all lines in the file and print them.
    for scanner.Scan() {
        line := scanner.Text()
        time.Sleep(2 * time.Second)
        //fmt.Println(line)
        return line
    }
    return Domain
}

Answer 1

If you wanted to do it "concurrently", you would return a channel through which you will send the multiple things you want to return:

https://play.golang.org/p/iYBGPwfYLYR

func DomainGrab() <-chan string {
    ch := make(chan string, 1)
    f, _ := os.Open("domains.txt")
    defer f.Close()
    scanner := bufio.NewScanner(f)
    go func() {
    // Loop over all lines in the file and print them.
        for scanner.Scan() {
            line := scanner.Text()
            time.Sleep(2 * time.Second)
            ch <- line
        }
        close(ch)
    }()
    return ch
}

Answer 2

If I understand your question, you want to read the file, somehow detect that file was modified and have a method that will emit these modifications to client code.

That is not how files work.

you have two options:

1. Listen for file changes using some OS specific API - https://www.linuxjournal.com/content/linux-filesystem-events-inotify

2. Read file using infinite loop. Read the file once. Save the copy into memory. Read the same file again and again in loop till new file is different from copy and calculate the delta.

Check if that is possible to use push instead of pull for getting new domains. Is it possible that system that control domain names in file will push data to you directly?

If loop is the only possible option, set up some pause time between file reads to reduce system load.

Use channels as @dave suggested when you managed to get new domains and need to process them concurrently.

Answer 3

Probably not the BEST solution. But, I decided to get rid of a separate function all together just cover more ground. I'll post below the code of what I expect. Now, I need to parse the domains so that Root URL's and Subdomains are only scanned once.

// Main
package main

import (
    "log"
    "fmt"
    "time"
    "net/http"
    "github.com/gocolly/colly"
)

//var Domain string
var Target string


func main() {
    c := colly.NewCollector()

    c.OnError(func(r *colly.Response, err error) {
        fmt.Println("Request URL:", r.Request.URL, "\n Failed with response:", r.StatusCode)
    })

    // Find and visit all links
    c.OnHTML("a", func(e *colly.HTMLElement) {
        e.Request.Visit(e.Attr("href"))
    })

    c.OnRequest(func(r *colly.Request) {
        Domain := r.URL.String()
        Target := BannerGrab(Domain)
        fmt.Println(Domain)
        fmt.Println(Target)
        fmt.Println("Dropping By.. ", r.URL)
        time.Sleep(1000 * time.Millisecond)
    })

    c.Visit("http://www.milliondollarhomepage.com/")
    }

    //CheckDB if not listed else add
    //RiskDB
    //Email


func BannerGrab(s string) string {

    client := &http.Client{}
    req, err := http.NewRequest("GET", s, nil)
        if err != nil {
        log.Fatalln(err)
    }
    req.Header.Set("User-Agent", "Authac/0.1")
    resp, _ := client.Do(req)
    serverEntry := resp.Header.Get("Server")
    return serverEntry

}