diff --git a/beater/youtubebeat.go b/beater/youtubebeat.go index 6d42a47..5a976c0 100644 --- a/beater/youtubebeat.go +++ b/beater/youtubebeat.go @@ -65,11 +65,13 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) { return } date := e.ChildAttr("meta[itemprop=\"datePublished\"]", "content") + genre := e.ChildAttr("meta[itemprop=\"genre\"]", "content") fields := common.MapStr{ "url": url, "title": title, "views": views, "date": date, + "genre": genre, "type": b.Info.Name, } event := beat.Event{ @@ -79,7 +81,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) { id := strings.Replace(fields["url"].(string), BaseUrl+BaseSuffix, "", -1) event.SetID(id) - + logp.Info(fields.String()) bt.client.Publish(event) logp.Info("Event sent") }) @@ -91,7 +93,7 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) { } }) - videoCollector.Visit(BaseUrl + BaseSuffix + bt.config.StartId) + videoCollector.Visit(BaseUrl) videoCollector.Wait() logp.Info("Done parsing all videos") done <- true