Go sync.WaitGroup doesnt work correctly with chromedp

76 views Asked by At

I'm a beginner in Go, and I'm experiencing an issue related to wait groups in the code I provided below. The sync.WaitGroup seems to be causing a deadlock as it enters the Wait function before the go routine function completes. The Go routines print to the screen once, then the subsequent routines do not execute, they simply wait. How can I resolve this issue? Thanks for your answering `

func main() {
    start := time.Now()

    opts := append(chromedp.DefaultExecAllocatorOptions[:],
        chromedp.Flag("headless", true),
        chromedp.Flag("ignore-certificate-errors", true),
        chromedp.Flag("disable-dev-shm-usage", true),
        chromedp.Flag("disable-gpu", true),
        chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3713.125 Safari/537.36"),
    )

    alloctx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
    defer cancel()
    ctx, cancel := chromedp.NewContext(alloctx)
    defer cancel()

    var wg sync.WaitGroup

    for i := 1; i <= 10; i++ {
        i := i
        wg.Add(1)
        go func() {
            defer wg.Done()
            err := chromedp.Run(ctx, visitAndFetch("https://example.com/?q=ka&page="+strconv.Itoa(i)))

            if err != nil {
                fmt.Println("error occurred", err)
            }

        }()

    }
    wg.Wait()
    fmt.Println(time.Since(start))

}

var title string

func visitAndFetch(url string) chromedp.Tasks {

    return chromedp.Tasks{
        chromedp.Navigate(url),
        chromedp.Nodes("article.search-result-item span#htmlContent", &titles, chromedp.ByQueryAll),
        chromedp.Nodes("article.search-result-item a.pdfLink", &links, chromedp.ByQueryAll),
        chromedp.Nodes("article.search-result-item h4.dates", &dates, chromedp.ByQueryAll),

        chromedp.ActionFunc(func(ctx context.Context) error {
            for i := 0; i < 10; i++ {
                title := links[i].AttributeValue("href")
                date := dates[i].Children[0].NodeValue
                strarr := strings.Split(date, " ")

                patent := &Patent{
                    PdfLink:     title,
                    PublishDate: strarr[1],
                }
                patents = append(patents, patent)

            }
            fmt.Printf("Added %s", patents)

            return nil
        }),
    }
}

I'm tried different solution and debug code but i dont handle this issue

0

There are 0 answers