gaugendre/youtubebeat
1
0
Fork 0
Code Issues Pull requests Releases Activity

Add genre and start from Youtube home page

Browse source
This commit is contained in:
Gabriel Augendre 2018-11-20 11:45:41 +01:00
parent f00529bb58
commit 27214bb1e0
1 changed files with 4 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
beater/youtubebeat.go
View file

@ -65,11 +65,13 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
return return
} }
date := e.ChildAttr("meta[itemprop=\"datePublished\"]", "content") date := e.ChildAttr("meta[itemprop=\"datePublished\"]", "content")
genre := e.ChildAttr("meta[itemprop=\"genre\"]", "content")
fields := common.MapStr{ fields := common.MapStr{
"url": url, "url": url,
"title": title, "title": title,
"views": views, "views": views,
"date": date, "date": date,
"genre": genre,
"type": b.Info.Name, "type": b.Info.Name,
} }
event := beat.Event{ event := beat.Event{
@ -79,7 +81,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
id := strings.Replace(fields["url"].(string), BaseUrl+BaseSuffix, "", -1) id := strings.Replace(fields["url"].(string), BaseUrl+BaseSuffix, "", -1)
event.SetID(id) event.SetID(id)
logp.Info(fields.String())
bt.client.Publish(event) bt.client.Publish(event)
logp.Info("Event sent") logp.Info("Event sent")
}) })
@ -91,7 +93,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
} }
}) })
videoCollector.Visit(BaseUrl + BaseSuffix + bt.config.StartId) videoCollector.Visit(BaseUrl)
videoCollector.Wait() videoCollector.Wait()
logp.Info("Done parsing all videos") logp.Info("Done parsing all videos")
done <- true done <- true