Limit parallelism

This commit is contained in:
Gabriel Augendre 2018-11-20 09:35:31 +01:00
parent 287424055e
commit 0ea086dc43

View file

@ -44,6 +44,8 @@ func scrapeVideos(startId string, fieldsToSend chan common.MapStr, done chan boo
colly.Async(true),
colly.MaxDepth(10),
)
videoCollector.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 5})
videoCollector.AllowURLRevisit = true
videoCollector.OnHTML("body", func(e *colly.HTMLElement) {
url := e.Request.URL.String()
@ -101,6 +103,8 @@ func (bt *Youtubebeat) Run(b *beat.Beat) error {
ticker := time.NewTicker(bt.config.Period)
for {
select {
case <-done:
return nil
case <-bt.done:
return nil
case <-ticker.C: