diff --git a/beater/youtubebeat.go b/beater/youtubebeat.go index 191cc94..3dae92b 100644 --- a/beater/youtubebeat.go +++ b/beater/youtubebeat.go @@ -44,6 +44,8 @@ func scrapeVideos(startId string, fieldsToSend chan common.MapStr, done chan boo colly.Async(true), colly.MaxDepth(10), ) + videoCollector.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 5}) + videoCollector.AllowURLRevisit = true videoCollector.OnHTML("body", func(e *colly.HTMLElement) { url := e.Request.URL.String() @@ -101,6 +103,8 @@ func (bt *Youtubebeat) Run(b *beat.Beat) error { ticker := time.NewTicker(bt.config.Period) for { select { + case <-done: + return nil case <-bt.done: return nil case <-ticker.C: