gaugendre/youtubebeat
1
0
Fork 0
Code Issues Pull requests Releases Activity

Limit parallelism

Browse source
This commit is contained in:
Gabriel Augendre 2018-11-20 09:35:31 +01:00
parent 287424055e
commit 0ea086dc43
1 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
beater/youtubebeat.go
View file

@ -44,6 +44,8 @@ func scrapeVideos(startId string, fieldsToSend chan common.MapStr, done chan boo
colly.Async(true), colly.Async(true),
colly.MaxDepth(10), colly.MaxDepth(10),
) )
videoCollector.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 5})
videoCollector.AllowURLRevisit = true
videoCollector.OnHTML("body", func(e *colly.HTMLElement) { videoCollector.OnHTML("body", func(e *colly.HTMLElement) {
url := e.Request.URL.String() url := e.Request.URL.String()
@ -101,6 +103,8 @@ func (bt *Youtubebeat) Run(b *beat.Beat) error {
ticker := time.NewTicker(bt.config.Period) ticker := time.NewTicker(bt.config.Period)
for { for {
select { select {
case <-done:
return nil
case <-bt.done: case <-bt.done:
return nil return nil
case <-ticker.C: case <-ticker.C: