gaugendre
/
youtubebeat
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Add genre and start from Youtube home page

This commit is contained in:
Gabriel Augendre 2018-11-20 11:45:41 +01:00
parent f00529bb58
commit 27214bb1e0
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
beater/youtubebeat.go
View File

@ -65,11 +65,13 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
return
}
date := e.ChildAttr("meta[itemprop=\"datePublished\"]", "content")
genre := e.ChildAttr("meta[itemprop=\"genre\"]", "content")
fields := common.MapStr{
"url": url,
"title": title,
"views": views,
"date": date,
"genre": genre,
"type": b.Info.Name,
}
event := beat.Event{
@ -79,7 +81,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
id := strings.Replace(fields["url"].(string), BaseUrl+BaseSuffix, "", -1)
event.SetID(id)
logp.Info(fields.String())
bt.client.Publish(event)
logp.Info("Event sent")
})
@ -91,7 +93,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
}
})
videoCollector.Visit(BaseUrl + BaseSuffix + bt.config.StartId)
videoCollector.Visit(BaseUrl)
videoCollector.Wait()
logp.Info("Done parsing all videos")
done <- true