Browse Source

Limit parallelism

master
Gabriel Augendre 3 years ago
parent
commit
0ea086dc43
  1. 4
      beater/youtubebeat.go

4
beater/youtubebeat.go

@ -44,6 +44,8 @@ func scrapeVideos(startId string, fieldsToSend chan common.MapStr, done chan boo
colly.Async(true),
colly.MaxDepth(10),
)
videoCollector.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 5})
videoCollector.AllowURLRevisit = true
videoCollector.OnHTML("body", func(e *colly.HTMLElement) {
url := e.Request.URL.String()
@ -101,6 +103,8 @@ func (bt *Youtubebeat) Run(b *beat.Beat) error {
ticker := time.NewTicker(bt.config.Period)
for {
select {
case <-done:
return nil
case <-bt.done:
return nil
case <-ticker.C:

Loading…
Cancel
Save