Remove some fields from config as we don't use them anymore + add genre field

This commit is contained in:
Gabriel Augendre 2018-11-20 14:31:38 +01:00
parent 27214bb1e0
commit 9958d50aa9
11 changed files with 27 additions and 18 deletions

View File

@ -4,8 +4,6 @@
youtubebeat:
# Defines how often an event is sent to the output
period: 1s
start_id: "SmBCZgcGlKk"
parallelism: 5
max_depth: 10

View File

@ -12,6 +12,11 @@
required: true
description: >
Video title
- name: genre
type: text
required: true
description: >
Video title
- name: views
type: long
required: true

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -81,7 +81,6 @@ func scrapeVideos(b *beat.Beat, bt *Youtubebeat, done chan bool) {
id := strings.Replace(fields["url"].(string), BaseUrl+BaseSuffix, "", -1)
event.SetID(id)
logp.Info(fields.String())
bt.client.Publish(event)
logp.Info("Event sent")
})

View File

@ -3,18 +3,12 @@
package config
import "time"
type Config struct {
Period time.Duration `config:"period"`
StartId string `config:"start_id"`
Parallelism int `config:"parallelism"`
MaxDepth int `config:"max_depth"`
Parallelism int `config:"parallelism"`
MaxDepth int `config:"max_depth"`
}
var DefaultConfig = Config{
Period: 1 * time.Second,
StartId: "SmBCZgcGlKk",
Parallelism: 5,
MaxDepth: 10,
}

View File

@ -453,6 +453,18 @@ required: True
Video title
--
*`genre`*::
+
--
type: text
required: True
Video title
--
*`views`*::

View File

@ -12,6 +12,11 @@
required: true
description: >
Video title
- name: genre
type: text
required: true
description: >
Video title
- name: views
type: long
required: true

View File

@ -31,5 +31,5 @@ func init() {
// Asset returns asset data
func Asset() string {
return "eJy0WFtv2zYUfu+vOOjLNsAR1mQNBj8M69JhDdZuRdfu1aXJY4kLRao8ZBzv1w+kqFsky07tvQTm7fu+w3OjcgF3uFvCznjn17hG5p4BOOkUPp4USNzKykmjl88ANhKVoPAL4AI0K3EJ3qo4BnC7Cpfg8MGlCYtfvLQoluCsxzTZh4Sf0iTA31Kg+YZg45WCTx/eDjiitrOx9OAagnuJWxoQKKPzE8wIgMCN127AI5gb2tGb+Bqayq+VpKKGeZb8OnToL2NPJqAbox2TmoCbsjQ6nksuBnbPpGJrhSA1MKUA71G7KJqyXiQMrAsAWfg5L/5jgfEAmA24AqNCINRC6jxOKJNDiUQsR8rgtrcrHpPUQhG6IDCsc6M3MveWBTrYSIWLMB8WmYN7pnw4CZ5QREzpwlAb1weLR6Aw5BJT2v/RRKqBjkVYi1Ofw/Bzi2Oixft1ZeNLaxgPX1yrjRFYdN5qFLDeRSpTYaDROdCOHJZgNGwLyYtOeO/urNda6nxCjZMl/mv0EWqanf+nmnu0JI0+LCZtbMIqhnN0fo46SEEBrpBUh3I2DN3nPwdTyLGyev6kBN0YWzI32IcPrKxC6r3yuScHl9eugMvvX1wv4MXl8url8uVVdnV1edztRkmwrQMZUxqGBLHIjRWwZdTZ98gox3KaZ3ll19JZZndxb31bnIVSEOO9Qls7imkRB84yTYy7zh/1PT0irqvD4B7N+h/kTa7Vg1W9coe7rbFiXmhbqzyh7XIqFKia7JECtNbYgYDcGl/Nk/waDjUVkNeMIX6ZEDLsZQqk3piQ2ZxRrF+Rh7ImGHr9sa8mFbN2fqKLzcjqpCWcbETAjRij91rYUegBZAwdsEbQQ58dhV6HSWpRXBkvuh51E4ZQWXMvBQYzHRPMsem29S6twsaaskZqj1LwVVeCmBCruGHVQIadHImM3dvFwtYsnsoa2MeJjfxA9v7Ra29DhRm8N0QyBG7sSQTMYgBcQM5xAcaCkLl0TBmOTGd7tUlNjmmOK3kgdW7TRrh93UgKTQRKxgupH6fuFMPhztRy9Pv6cSxpw6oXZ+09u8usRCF9Oc/+roaIIfY08vTMkUq63arX8loFni6Qkbt4wQ8U0h4QxI4ou24nqZYjqWtzMyEXa2Pr1VZKWrl4OD700pGg5TdjcoV1pu1nt5gfbLUf4p5D9qVEF4bfxfxJmf66GU+A12tAjrlQfpVCHnp2TPN6LeQsFca6Vd0BlrBhioLTmOaFsQ3fRZvlez5aWlkw2R/21fHUE9BmUpxWEz9p+cVjBwhSTFX1lq6cah9PYuzHRYRrXqdJQHhIrL1UDoyek9IrBl+p5KblDFhzXIqtUdGIbfCWgPn3xAEtt/Emap42aEMwdyH7ph5NgNyGx0AvUEOXG5WeLjbD/MHITNxPi8vTffImfVaMvXGmSK8LxESQM8sL6ZA7b89gwwAOvsUsz+Dhx+vV9Q8LYLZcQFXxBZSyou/GUgxllWIuPOlPU/LnX9AAJQ0ctTO0AL/22vkFbKUWZrtHxPCL5+s1JJxJjg0rpdqdTFHDJCMtioK5BQhcS6YXsLGIaxJz1spqJGEwNcP+VpILBe32/QUTwiIR0pigZPw0IxuaglmxZRY7sgV48kypHbx7ddPX0NSRO79Gq9EhddXk9/7cBG233j6Dh2/aDhT6tWS+LXaHDhaggWh4UhmqjDhDe+jdQGVEXdsmqfyppanH9N4I+HT7ekwU/lLF+PmM6hDHZOEL7Kw3GBD3XOGxzfU4ohoNSlaNmZjWxsX/f52Nrgc5zXnOB0uPlw/eLnO0Z3iyTfLWuP8FAAD//7HY8EI="
return "eJy0WN1uGzcTvc9TDHLzfQXkRWM3RqGLoqlTNEaTNkiT3ioUOdplzSU3HNKy+vQFudw/72olR8qNIf6dc0iemeH6Au5wt4Sd8c6vcY3MPQNw0il83CmQuJWVk0YvnwFsJCpB4RfABWhW4hK8VbEN4HYVLsHhg0sdFr94aVEswVmPqbMPCT+lToC/pUDzP4KNVwo+fXg74IjazsbSg2sIctT2mxLcS9zSgEAZnZ9wTgEQuPHaDXgEc8N99Dq+hqbyayWpqGGeJeMMHfPL2CoJ6MZox6Qm4KYsjY7rkoeA3TOp2FohSA1MKcB71C6KpqxntcHuAkAWfs6L/1hgXABmA67AqBAItZA6jx3K5FAiEcuRMrjtzYrLJLVQhC4IDOPc6I3MvWWBDjZS4SL0h0Hm4J4pH1aCJxQRU7rQ1Mb1weISKAy5xJTmfzSRaqBjEcZi1+fQ/NzimLjj/bqy8aE1jIcPrtXGCCw6bzUKWO8ilakw0OgcaEcOSzAatoXkRSe8d3bWay11PqHGyRL/NfoINc3Mb6nmHi1Jow+LSRMbW0U7x8vPUQcpKMAVkmorZ0PrPv85bIUcK6vnTwrQjbElc4N5+MDKKoTeK597cnB57Qq4/P7F9QJeXC6vXi5fXmVXV5fHnW6UBNvayJjCMASIRW6sgC2jbn+PNuVYTvMsr+xaOsvsLs6tT4uzkAqi3yu09UUxLWLDWaaJcdfdR31Oj4jr7DA4R7P+B3kTa3VjVY/c4W5rrJgX2uYqT2i7mAoJqiZ7pACtNXYgILfGV/Mkv4ZFTQbkNWPwLxNChrlMgdQbEyKbM4r5K/JQ1pihV4D7alIya/snqtiMrE5awslGBNyIMXqvhB2FHkDG0AFrBD28s6PQa5ukEsWV8aKrUTehCZU191Jg2KZjgjk2XbbepVHYWFPWSO1SCnfVpSAmxCpOWDWQYSZHImP3VrEwNYursgb2cWAjPxC9f/TK21BhBu8NkQzGjTWJgFkMgAvIOS7AWBAyl44pw5HpbK82qckxzXElD4TObZoIt68bSaGIQMl4IfXj0J1iOFyZWo5+XT+OJU1Y9XzWnrO7zEoU0pfz7O9qiGixp5GnZ45U0u1WvZLXKvB0gYzcxQt+IJH2gCBWRNlVO0m1HEldmZuxXMyN7a22UtLIxcPx1ktLgpbfjMkV1pG2n91ifrDUfohzDu0vBbow/C7GT4r01017ArweA3LMhfSrFPJQs2OY12MhZqkw1q3qCrCEDVMULo1pXhjb8F20Ub7nq6iVBZP1YV8eTzUBbSbFaTnxk5ZfPHaAIMVUVm/pyqny8STGvi8iXPM6TQLCQ2LtpXJg9JyUXjL4SiU3LWfAmuNSbI2KRmyDtwTMvycOaLmNJ1HztKYNZu4s+6ZuTYDchsdAz6ihyo1ST+fN0H/QmYn7ab48/U7epM+K8W2cyel1gpgwObO8kA658/YMexjAwf8xyzN4+PF6df3DApgtF1BVfAGlrOi7sRRDWaWYC0/605T8+Rc0QEkDR+0MLcCvvXZ+AVuphdnuETH84vl6DQlnkmPDSql2J1PUMGmTFkXB3AIEriXTC9hYxDWJud3KaiRh0DXD/laSCwnt9v0FE8IiEdKYoGT8tE02NAWzYsssdmQL8OSZUjt49+qmr6HJI3d+jVajQ+qyye/9vgnabrx9Bg/ftB0o9HPJfFnsFh1MQAPR8KQ0VBlxhvLQO4HKiDq3TVL5U1NTj+m9EfDp9vWYKPylivHzbapDHJOFL7CznmBA3HOExxbX44hqNChZNWZiWhsX//91Nroe5DTnOR8sPV4+eLvM0Z7hyTbJW+P+FwAA//+CrAtt"
}

View File

@ -4,8 +4,6 @@
youtubebeat:
# Defines how often an event is sent to the output
period: 1s
start_id: "SmBCZgcGlKk"
parallelism: 5
max_depth: 10

View File

@ -4,8 +4,6 @@
youtubebeat:
# Defines how often an event is sent to the output
period: 1s
start_id: "SmBCZgcGlKk"
parallelism: 5
max_depth: 10