From f76e8bbf423fef040964cb74fbe06d35f5ef0a78 Mon Sep 17 00:00:00 2001 From: Gabriel Augendre Date: Mon, 19 Nov 2018 18:21:56 +0100 Subject: [PATCH] Use an event ID --- beater/youtubebeat.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/beater/youtubebeat.go b/beater/youtubebeat.go index 3a5e0b6..191cc94 100644 --- a/beater/youtubebeat.go +++ b/beater/youtubebeat.go @@ -35,10 +35,10 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) { return bt, nil } -func scrapeVideos(startId string, fieldsToSend chan common.MapStr, done chan bool) { - const BaseUrl = "https://www.youtube.com" - const BaseSuffix = "/watch?v=" +const BaseUrl = "https://www.youtube.com" +const BaseSuffix = "/watch?v=" +func scrapeVideos(startId string, fieldsToSend chan common.MapStr, done chan bool) { videoCollector := colly.NewCollector( colly.AllowedDomains("youtube.com", "www.youtube.com"), colly.Async(true), @@ -117,6 +117,10 @@ func (bt *Youtubebeat) Run(b *beat.Beat) error { Timestamp: time.Now(), Fields: fields, } + + id := strings.Replace(fields["url"].(string), BaseUrl+BaseSuffix, "", -1) + event.SetID(id) + bt.client.Publish(event) logp.Info("Event sent") }