Limit parallelism

beater/youtubebeat.go

@@ -44,6 +44,8 @@ func scrapeVideos(startId string, fieldsToSend chan common.MapStr, done chan boo
 		colly.Async(true),
 		colly.MaxDepth(10),
 	)
+	videoCollector.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 5})
+	videoCollector.AllowURLRevisit = true
 
 	videoCollector.OnHTML("body", func(e *colly.HTMLElement) {
 		url := e.Request.URL.String()
@@ -101,6 +103,8 @@ func (bt *Youtubebeat) Run(b *beat.Beat) error {
 	ticker := time.NewTicker(bt.config.Period)
 	for {
 		select {
+		case <-done:
+			return nil
 		case <-bt.done:
 			return nil
 		case <-ticker.C: