Browse Source

Add genre and start from Youtube home page

master
Gabriel Augendre 3 years ago
parent
commit
27214bb1e0
  1. 6
      beater/youtubebeat.go

6
beater/youtubebeat.go

@ -65,11 +65,13 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
return
}
date := e.ChildAttr("meta[itemprop=\"datePublished\"]", "content")
genre := e.ChildAttr("meta[itemprop=\"genre\"]", "content")
fields := common.MapStr{
"url": url,
"title": title,
"views": views,
"date": date,
"genre": genre,
"type": b.Info.Name,
}
event := beat.Event{
@ -79,7 +81,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
id := strings.Replace(fields["url"].(string), BaseUrl+BaseSuffix, "", -1)
event.SetID(id)
logp.Info(fields.String())
bt.client.Publish(event)
logp.Info("Event sent")
})
@ -91,7 +93,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
}
})
videoCollector.Visit(BaseUrl + BaseSuffix + bt.config.StartId)
videoCollector.Visit(BaseUrl)
videoCollector.Wait()
logp.Info("Done parsing all videos")
done <- true

Loading…
Cancel
Save