Browse Source

Add code and dependencies

master
Gabriel Augendre 3 years ago
parent
commit
d0ded4fd02
  1. 2
      _meta/beat.yml
  2. 19
      _meta/fields.yml
  3. 2
      _meta/kibana.generated/5/index-pattern/youtubebeat.json
  4. 2
      _meta/kibana.generated/6/index-pattern/youtubebeat.json
  5. 64
      beater/youtubebeat.go
  6. 8
      config/config.go
  7. 1
      data/meta.json
  8. 40
      docs/fields.asciidoc
  9. 19
      fields.yml
  10. 2
      include/fields.go
  11. 8
      logs/youtubebeat
  12. 12
      vendor/github.com/PuerkitoBio/goquery/LICENSE
  13. 179
      vendor/github.com/PuerkitoBio/goquery/README.md
  14. 124
      vendor/github.com/PuerkitoBio/goquery/array.go
  15. 234
      vendor/github.com/PuerkitoBio/goquery/array_test.go
  16. 120
      vendor/github.com/PuerkitoBio/goquery/bench_array_test.go
  17. 40
      vendor/github.com/PuerkitoBio/goquery/bench_example_test.go
  18. 104
      vendor/github.com/PuerkitoBio/goquery/bench_expand_test.go
  19. 236
      vendor/github.com/PuerkitoBio/goquery/bench_filter_test.go
  20. 68
      vendor/github.com/PuerkitoBio/goquery/bench_iteration_test.go
  21. 51
      vendor/github.com/PuerkitoBio/goquery/bench_property_test.go
  22. 111
      vendor/github.com/PuerkitoBio/goquery/bench_query_test.go
  23. 802
      vendor/github.com/PuerkitoBio/goquery/bench_traversal_test.go
  24. 123
      vendor/github.com/PuerkitoBio/goquery/doc.go
  25. 82
      vendor/github.com/PuerkitoBio/goquery/example_test.go
  26. 70
      vendor/github.com/PuerkitoBio/goquery/expand.go
  27. 118
      vendor/github.com/PuerkitoBio/goquery/expand_test.go
  28. 163
      vendor/github.com/PuerkitoBio/goquery/filter.go
  29. 206
      vendor/github.com/PuerkitoBio/goquery/filter_test.go
  30. 6
      vendor/github.com/PuerkitoBio/goquery/go.mod
  31. 5
      vendor/github.com/PuerkitoBio/goquery/go.sum
  32. 39
      vendor/github.com/PuerkitoBio/goquery/iteration.go
  33. 88
      vendor/github.com/PuerkitoBio/goquery/iteration_test.go
  34. 574
      vendor/github.com/PuerkitoBio/goquery/manipulation.go
  35. 513
      vendor/github.com/PuerkitoBio/goquery/manipulation_test.go
  36. 275
      vendor/github.com/PuerkitoBio/goquery/property.go
  37. 252
      vendor/github.com/PuerkitoBio/goquery/property_test.go
  38. 49
      vendor/github.com/PuerkitoBio/goquery/query.go
  39. 103
      vendor/github.com/PuerkitoBio/goquery/query_test.go
  40. 855
      vendor/github.com/PuerkitoBio/goquery/testdata/gotesting.html
  41. 1214
      vendor/github.com/PuerkitoBio/goquery/testdata/gowiki.html
  42. 413
      vendor/github.com/PuerkitoBio/goquery/testdata/metalreview.html
  43. 102
      vendor/github.com/PuerkitoBio/goquery/testdata/page.html
  44. 24
      vendor/github.com/PuerkitoBio/goquery/testdata/page2.html
  45. 24
      vendor/github.com/PuerkitoBio/goquery/testdata/page3.html
  46. 698
      vendor/github.com/PuerkitoBio/goquery/traversal.go
  47. 793
      vendor/github.com/PuerkitoBio/goquery/traversal_test.go
  48. 141
      vendor/github.com/PuerkitoBio/goquery/type.go
  49. 202
      vendor/github.com/PuerkitoBio/goquery/type_test.go
  50. 161
      vendor/github.com/PuerkitoBio/goquery/utilities.go
  51. 128
      vendor/github.com/PuerkitoBio/goquery/utilities_test.go
  52. 24
      vendor/github.com/andybalholm/cascadia/LICENSE
  53. 9
      vendor/github.com/andybalholm/cascadia/README.md
  54. 53
      vendor/github.com/andybalholm/cascadia/benchmark_test.go
  55. 3
      vendor/github.com/andybalholm/cascadia/go.mod
  56. 835
      vendor/github.com/andybalholm/cascadia/parser.go
  57. 86
      vendor/github.com/andybalholm/cascadia/parser_test.go
  58. 622
      vendor/github.com/andybalholm/cascadia/selector.go
  59. 654
      vendor/github.com/andybalholm/cascadia/selector_test.go
  60. 17
      vendor/github.com/antchfx/htmlquery/LICENSE
  61. 93
      vendor/github.com/antchfx/htmlquery/README.md
  62. 299
      vendor/github.com/antchfx/htmlquery/query.go
  63. 126
      vendor/github.com/antchfx/htmlquery/query_test.go
  64. 17
      vendor/github.com/antchfx/xmlquery/LICENSE
  65. 169
      vendor/github.com/antchfx/xmlquery/README.md
  66. 121
      vendor/github.com/antchfx/xmlquery/books.xml
  67. 281
      vendor/github.com/antchfx/xmlquery/node.go
  68. 308
      vendor/github.com/antchfx/xmlquery/node_test.go
  69. 256
      vendor/github.com/antchfx/xmlquery/query.go
  70. 115
      vendor/github.com/antchfx/xmlquery/query_test.go
  71. 17
      vendor/github.com/antchfx/xpath/LICENSE
  72. 155
      vendor/github.com/antchfx/xpath/README.md
  73. 483
      vendor/github.com/antchfx/xpath/build.go
  74. 33
      vendor/github.com/antchfx/xpath/doc_test.go
  75. 475
      vendor/github.com/antchfx/xpath/func.go
  76. 9
      vendor/github.com/antchfx/xpath/func_go110.go
  77. 15
      vendor/github.com/antchfx/xpath/func_pre_go110.go
  78. 295
      vendor/github.com/antchfx/xpath/operator.go
  79. 1164
      vendor/github.com/antchfx/xpath/parse.go
  80. 791
      vendor/github.com/antchfx/xpath/query.go
  81. 157
      vendor/github.com/antchfx/xpath/xpath.go
  82. 631
      vendor/github.com/antchfx/xpath/xpath_test.go
  83. 21
      vendor/github.com/gobwas/glob/LICENSE
  84. 26
      vendor/github.com/gobwas/glob/bench.sh
  85. 525
      vendor/github.com/gobwas/glob/compiler/compiler.go
  86. 624
      vendor/github.com/gobwas/glob/compiler/compiler_test.go
  87. 80
      vendor/github.com/gobwas/glob/glob.go
  88. 531
      vendor/github.com/gobwas/glob/glob_test.go
  89. 45
      vendor/github.com/gobwas/glob/match/any.go
  90. 82
      vendor/github.com/gobwas/glob/match/any_of.go
  91. 53
      vendor/github.com/gobwas/glob/match/any_of_test.go
  92. 57
      vendor/github.com/gobwas/glob/match/any_test.go
  93. 185
      vendor/github.com/gobwas/glob/match/btree.go
  94. 90
      vendor/github.com/gobwas/glob/match/btree_test.go
  95. 58
      vendor/github.com/gobwas/glob/match/contains.go
  96. 74
      vendor/github.com/gobwas/glob/match/contains_test.go
  97. 55
      vendor/github.com/gobwas/glob/match/debug/debug.go
  98. 99
      vendor/github.com/gobwas/glob/match/every_of.go
  99. 45
      vendor/github.com/gobwas/glob/match/every_of_test.go
  100. 49
      vendor/github.com/gobwas/glob/match/list.go

2
_meta/beat.yml

@ -5,4 +5,4 @@
youtubebeat:
# Defines how often an event is sent to the output
period: 1s
start_url: "https://www.youtube.com/watch?v=SmBCZgcGlKk"
start_id: "SmBCZgcGlKk"

19
_meta/fields.yml

@ -2,8 +2,23 @@
title: youtubebeat
description:
fields:
- name: counter
- name: url
type: string
required: true
description: >
Video's full URL
- name: title
type: string
required: true
description: >
Video title
- name: views
type: long
required: true
description: >
PLEASE UPDATE DOCUMENTATION
Video's view count
- name: date
type: string
required: true
description: >
Video's publish date

2
_meta/kibana.generated/5/index-pattern/youtubebeat.json
File diff suppressed because it is too large
View File

2
_meta/kibana.generated/6/index-pattern/youtubebeat.json
File diff suppressed because it is too large
View File

64
beater/youtubebeat.go

@ -2,6 +2,9 @@ package beater
import (
"fmt"
"github.com/gocolly/colly"
"strconv"
"strings"
"time"
"github.com/elastic/beats/libbeat/beat"
@ -32,6 +35,54 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
return bt, nil
}
func scrapeVideos(startId string, fieldsToSend chan common.MapStr) {
const BaseUrl = "https://www.youtube.com"
const BaseSuffix = "/watch?v="
videoCollector := colly.NewCollector(
colly.AllowedDomains("youtube.com", "www.youtube.com"),
colly.Async(true),
colly.MaxDepth(10),
)
videoCollector.OnHTML("body", func(e *colly.HTMLElement) {
url := e.Request.URL.String()
isPaid := e.ChildAttr("meta[itemprop=\"paid\"]", "content")
if isPaid == "True" {
logp.Err("Not parsing video because of isPaid" + url)
return
}
title := e.ChildAttr("meta[itemprop=\"name\"]", "content")
if title == "YouTube" {
logp.Err("Not parsing video because of title " + url)
return
}
views, err := strconv.ParseInt(e.ChildAttr("meta[itemprop=\"interactionCount\"]", "content"), 10, 64)
if err != nil {
logp.Err("Can't parse view count for URL " + url)
return
}
date := e.ChildAttr("meta[itemprop=\"datePublished\"]", "content")
fields := common.MapStr{
"url": url,
"title": title,
"views": views,
"date": date,
}
fieldsToSend <- fields
})
videoCollector.OnHTML("a[href]", func(e *colly.HTMLElement) {
href := e.Attr("href")
if strings.HasPrefix(href, BaseSuffix) {
e.Request.Visit(BaseUrl + href)
}
})
videoCollector.Visit(BaseUrl + BaseSuffix + startId)
videoCollector.Wait()
}
// Run starts youtubebeat.
func (bt *Youtubebeat) Run(b *beat.Beat) error {
logp.Info("youtubebeat is running! Hit CTRL-C to stop it.")
@ -42,25 +93,24 @@ func (bt *Youtubebeat) Run(b *beat.Beat) error {
return err
}
fieldsToSend := make(chan common.MapStr)
go scrapeVideos(bt.config.StartId, fieldsToSend)
ticker := time.NewTicker(bt.config.Period)
counter := 1
for {
select {
case <-bt.done:
return nil
case <-ticker.C:
}
fields := <-fieldsToSend
fields["type"] = b.Info.Name
event := beat.Event{
Timestamp: time.Now(),
Fields: common.MapStr{
"type": b.Info.Name,
"counter": counter,
},
Fields: fields,
}
bt.client.Publish(event)
logp.Info("Event sent")
counter++
}
}

8
config/config.go

@ -6,11 +6,11 @@ package config
import "time"
type Config struct {
Period time.Duration `config:"period"`
StartUrl string `config:"start_url"`
Period time.Duration `config:"period"`
StartId string `config:"start_id"`
}
var DefaultConfig = Config{
Period: 1 * time.Second,
StartUrl: "https://www.youtube.com/watch?v=SmBCZgcGlKk",
Period: 1 * time.Second,
StartId: "SmBCZgcGlKk",
}

1
data/meta.json

@ -0,0 +1 @@
{"uuid":"a5fddc65-b03a-4cde-b7a9-f884d277841d"}

40
docs/fields.asciidoc

@ -431,14 +431,50 @@ Kubernetes container image
None
*`counter`*::
*`url`*::
+
--
type: string
required: True
Video's full URL
--
*`title`*::
+
--
type: string
required: True
Video title
--
*`views`*::
+
--
type: long
required: True
PLEASE UPDATE DOCUMENTATION
Video's view count
--
*`date`*::
+
--
type: string
required: True
Video's publish date
--

19
fields.yml

@ -2,11 +2,26 @@
title: youtubebeat
description:
fields:
- name: counter
- name: url
type: string
required: true
description: >
Video's full URL
- name: title
type: string
required: true
description: >
Video title
- name: views
type: long
required: true
description: >
PLEASE UPDATE DOCUMENTATION
Video's view count
- name: date
type: string
required: true
description: >
Video's publish date
- key: beat
title: Beat

2
include/fields.go

@ -31,5 +31,5 @@ func init() {
// Asset returns asset data
func Asset() string {
return "eJy0WFtz2zYTffev2MnT983InMZuPB09dOrankbTOPE09rOyAlYkahJgcLGs/voOQPBmUpQcqy8ZE5dzzgK7Z6GcwiNt57BVzroVrQjtCYAVNqeXg5wM06K0Qsn5CcBaUM6N/wvgFCQWNAemnLSkwxiA3ZY0h1zJNA5o+u6EJj4Hqx3FwS4s/BoHAe4+3Vx+vYGHu+vL+xu4/nL1cHvz+f7yfvHl88lJFN1X+/tQZsS7UtKikAaYKgolw76oH/AJRY6rnEBIwDwHeiJpg3aTdMLsxekBEv/ndAz3GYUNoNZgMwoKwZDkQqZhIFcpFGQMpmQSWHRWhW3CNFCGrBfo55mSa5E6jZ4O1iKnmR/3k2jhCXPnd4IzxAOmsP5TKtsFC1sgU8ZGprj+XgWqno6ZnwtD3/zntwZHhYh360qGh1Yz7j+4Rhsa0GSdlsRhtQ1UqiRPI1MwW2OpACVhkwmWtcI7Z6edlEKmI2qsKOgfJQ9QU6/8L9U8kTZCyf1i4sI6rUI6h8tPSXopxMFmwlSpnPRT991vPhRjsSjf9eqUo6WpOl0rXaDtraNnLEpfepcudcbC2YXN4Oyn9xczeH82P/8w/3CenJ+fHXa6QRJsqkSmWIa+QDQxpTls0LTxvQjKYmqmWS71SliNehvWVqfF0FtByPeSdHVRKHn4sBqlQWbb+6jO6QVx5Q69c1Srv4nVtVZ9LKuZR9pulObTQhuvcoZ0W1PeoCqyFwpIa9U33FQrV06T3PhNtQOyitHnL3Iu/FrMQci18pXN0AT/CjwmqZOhY/5dNdHMmvFak6Vn2xncIauVFnGSAQFTfIjeaTEHoXuQIbTHGkD37+wg9CpNYotiuXK87VFX/hNKrZ4EJx+mRY4Wx9vWbZyFtVZFhdRsNf6uWgtCzpdhwbKG9CsZGaP0zi7mlyZhV1LDvixsYnuq93OnvfUVJnCnjBE+cUNPMoCaPOAMUkYzUBq4SIXFXDFCmezUJqSxKBktxZ7SWcSFsLiuJfkmAgWyTMiXpTvGsL8zNRzdvn4YS1yw7ORZc872LCmIC1dMs99WECHFXkcenzkiF3a77LS8RoEzp4TGnr5ne4y0AwShI4q22wlTyRGmbXMTKRe8sbnVRkqcOX0+PPXiFq/lD6XSnKpK282uKd3bav8Ka/bFFwudK/YY6idW+nX9PQJezYGxaL395jkx37NDmVdzvmZNprRdVh1gDmvMjb80lCxTuuY7bap8x4u8kQWj/WGXj8eeQDoR/G2e+CDFd0ctIAg+5uoNXTHWPl7F2M2LAFe/TqMA/5BYOZFbUHJKSscMflDJVcPpsaa4clxRbgZsvbcETL8n9mhZhJOoeJqk9cncpuzH6msEZOEfA51E9V1uYD1tbvrxvZkZuV+Xl2+/k4/xZ8XwNo6U6ZVBjCQ5apYJS8w6fYQYenDwP0rSBJ5/uVhe/DwD1MUMypLNoBCl+f9QijJJmaP1T/q3KfnyFWqgqIGRtMrMwK2ctG4GGyG52uwQ0f/F8+MaIs4oxxoLkW/fTFHBxCA18QztDDitBMoZrDXRyvCpaEU5kNAbmmD/JIz1hra4O0XONRlDZkhQIHtbkDVNhppvUFNLNgNnHOb5Fm4vr7oaah95dCvSkiyZ1k3+7I6N0LbzzTO4/6ZtQaHrJdNtsd2014B6ouFVNlQqfoT20DmBUvHK20ap3FutqcN0pzg8LK6HRP5fUyI7XlAt4pDM/wI76gl6xB1HeGhzPYyoQoMCyyETSqls+P+vo9F1IMc5j/lg6fCy3ttlivYIT7ZR3gr33wAAAP//UV6dfg=="
return "eJy0WEtvGzcX3edXXGST7wOkQWM3RqFF0dQpGqFJG6RJtwpFXs2w5pATXtKy+usLcjgvz+jhSN0Y4uucc3lfHM/hDncL2Bnv/BrXyNwzACedwseTAolbWTlp9OIZwEaiEhR+AcxBsxIX4K2KYwC3q3AB5KzUeZqy+NVLi2IBznpMk31Q+DFNAvwlBZoXBBuvFHz++G7AEtVdkKcH2FDcS9zSgEKZswwJgMCN127AI5i7pCUvCCq/VpKKGvlZ8u7QrT+P/ZmAbo12TGoCbsrS6HguORrYPZOKrRWC1MCUArxH7aJsynrxMLAvAGTh52HxnwqMB8BswBUYFQKhFlLncUKZHEokYjlSBsvernhMUgtF6ILAsM6N3sjcWxboYCMVzsJ8WGQO7pny4SR4QhExpQtDbVwfLB6BwpBLTGn/JxOpBjpmYS1OfQnDLy2OiRbv15WNL61hPH5xrTZGYNF5q1HAehepTIWBRudAO3JYgtGwLSQvOuG9u7Nea6nzCTVOlviP0SeoaXb+l2ru0ZI0+riYtLEJqxjO0fk56iAFBbhCUh3K2TB0n/8UTCHHyur5IEV7OTuZoBtjS+YG+/CBlVVIvdc+9+Tg6sYVcPXdy5sZvLxaXL9avLrOrq+vTrvdKAm2dSBjSsOQIBa5sQK2jDr7HhnlWE6HWV7btXSW2V3cW98WZ6EUxHiv0NaOYlrEgbNME+Ou80d9T4+I6+owuEez/ht5k2v1YFWv3OFua6w4LLStVZ7QdjkVClRN9kgBWmvsQEBuja8Ok/wSDjUVkNeMIX6ZEDLsZQqk3piQ2ZxRrF+Rh7ImGHpdsq8mFbN2vtHk8MH1JvfI6qQlnGxEwI0Yo/ea2EnoAWQMHbBG0EOfnYReh0lqUVwZL7oedRuGUFlzLwUGMx0TzLHptvU+rcLGmrJGao9S8FVXgpgQq7hh1UCGnRyJjN3bxcLWLJ7KGtjHiY38SPb+3mtvQ4UZfDBEMgRu7EkEzGIAnEHOcQbGgpC5dEwZjkxne7VJTY5pjit5JHWWaSMs3zSSQhOBkvFC6sepO8VwvDO1HP2+fhpL2rDqxVl7z+4qK1FIXx5mf19DxBB7Gnl65kgl3W7Va3mtAk9zZOTmL/mRQtoDgtgRZdftJNVyJHVt7kDIxdrYerWVklbmD6eHXjoStPxqTK6wzrT97Bbzo632Y9xzzL6U6MLwu5g/KdPfNOMJ8HoNyDEXyq9SyEPPjmler4WcpcJYt6o7wAI2TFFwGtO8MLbhm7dZvufTpZUFk/1hXx1PPQFtJsV5NfGzll89doAgxVRVb+nKqfbxJMZ+XES45nWaBISHxNpL5cDoQ1J6xeAbldy2nAHrEJdia1Q0Yhu8JeDwe+KIlmW8iZqnDdoQzF3Ivq1HEyDL8BjoBWrocqPS08VmmD8amYn7aXF5vk/eps+KsTcuFOl1gZgIcmZ5IR1y5+0FbBjAwf8wyzN4+OFmdfP9DJgtZ1BVfAalrOj/YymGskoxF5705yn5409ogJIGjtoZmoFfe+38DLZSC7PdI2L4xfPtGhLOJMeGlVLtzqaoYZKRFkXB3AwEriXTM9hYxDWJQ9bKaiRhMHWA/Z0kFwra8sOcCWGRCGlMUDJ+npENTcGs2DKLHdkMPHmm1A7ev77ta2jqyJ1fo9XokLpq8lt/boK2W2+fwcM3bQcK/VpyuC12h44WoIFoeFIZqoy4QHvo3UBlRF3bJqn8uaWpx/TBCPi8fDMmCn+pYvxyRnWIY7LwBXbRGwyIe67w1OZ6GlGNBiWrxkxMa+Pi/78uRteDnOa85IOlx8sHb5dDtBd4sk3y1rj/BgAA///8ffLf"
}

8
logs/youtubebeat

@ -0,0 +1,8 @@
2018-11-18T11:09:17.508+0100 INFO instance/beat.go:616 Home path: [/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat] Config path: [/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat] Data path: [/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat/data] Logs path: [/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat/logs]
2018-11-18T11:09:17.509+0100 INFO instance/beat.go:623 Beat UUID: a5fddc65-b03a-4cde-b7a9-f884d277841d
2018-11-18T11:09:17.509+0100 INFO [beat] instance/beat.go:849 Beat info {"system_info": {"beat": {"path": {"config": "/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat", "data": "/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat/data", "home": "/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat", "logs": "/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat/logs"}, "type": "youtubebeat", "uuid": "a5fddc65-b03a-4cde-b7a9-f884d277841d"}}}
2018-11-18T11:09:17.509+0100 INFO [beat] instance/beat.go:858 Build info {"system_info": {"build": {"commit": "38fd4465358c195b17152b60e7e32e6ed0345528", "libbeat": "6.5.0", "time": "2018-11-18T10:08:58.000Z", "version": "6.5.0"}}}
2018-11-18T11:09:17.509+0100 INFO [beat] instance/beat.go:861 Go runtime info {"system_info": {"go": {"os":"darwin","arch":"amd64","max_procs":8,"version":"go1.11.2"}}}
2018-11-18T11:09:17.511+0100 INFO [beat] instance/beat.go:865 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2018-11-13T17:51:46.564974+01:00","name":"persephone.local","ip":["127.0.0.1/8","::1/128","fe80::1/64","fe80::1cb1:a370:e195:1b14/64","10.0.0.11/24","2a01:e34:ec0f:96a0:1c7d:59bf:1f3b:b4d9/64","2a01:e34:ec0f:96a0:bdde:b4b8:e13:d8da/64","fe80::d891:cff:fe4e:3f99/64","fe80::8aed:bb9d:8b99:8746/64","fe80::e055:800e:12a0:b862/64","fe80::4e1b:e233:d6da:218/64","fe80::aede:48ff:fe00:1122/64"],"kernel_version":"18.2.0","mac":["b6:00:34:c9:44:01","b6:00:34:c9:44:00","b6:00:34:c9:44:05","b6:00:34:c9:44:04","dc:a9:04:7d:4a:e4","0e:a9:04:7d:4a:e4","da:91:0c:4e:3f:99","b6:00:34:c9:44:01","ac:de:48:00:11:22"],"os":{"family":"darwin","platform":"darwin","name":"Mac OS X","version":"10.14.1","major":10,"minor":14,"patch":1,"build":"18B75"},"timezone":"CET","timezone_offset_sec":3600}}}
2018-11-18T11:09:17.511+0100 INFO [beat] instance/beat.go:894 Process info {"system_info": {"process": {"cwd": "/Users/gaugendre/Projects/golang/src/github.com/Crocmagnon/youtubebeat", "exe": "./youtubebeat", "name": "youtubebeat", "pid": 75764, "ppid": 74760, "start_time": "2018-11-18T11:09:17.481+0100"}}}
2018-11-18T11:09:17.511+0100 INFO instance/beat.go:302 Setup Beat: youtubebeat; Version: 6.5.0

12
vendor/github.com/PuerkitoBio/goquery/LICENSE

@ -0,0 +1,12 @@
Copyright (c) 2012-2016, Martin Angers & Contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

179
vendor/github.com/PuerkitoBio/goquery/README.md

@ -0,0 +1,179 @@
# goquery - a little like that j-thing, only in Go
[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.svg?branch=master)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
goquery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on Go's [net/html package][html] and the CSS Selector library [cascadia][]. Since the net/html parser returns nodes, and not a full-featured DOM tree, jQuery's stateful manipulation functions (like height(), css(), detach()) have been left off.
Also, because the net/html parser requires UTF-8 encoding, so does goquery: it is the caller's responsibility to ensure that the source document provides UTF-8 encoded HTML. See the [wiki][] for various options to do this.
Syntax-wise, it is as close as possible to jQuery, with the same function names when possible, and that warm and fuzzy chainable interface. jQuery being the ultra-popular library that it is, I felt that writing a similar HTML-manipulating library was better to follow its API than to start anew (in the same spirit as Go's `fmt` package), even though some of its methods are less than intuitive (looking at you, [index()][index]...).
## Table of Contents
* [Installation](#installation)
* [Changelog](#changelog)
* [API](#api)
* [Examples](#examples)
* [Related Projects](#related-projects)
* [Support](#support)
* [License](#license)
## Installation
Please note that because of the net/html dependency, goquery requires Go1.1+.
$ go get github.com/PuerkitoBio/goquery
(optional) To run unit tests:
$ cd $GOPATH/src/github.com/PuerkitoBio/goquery
$ go test
(optional) To run benchmarks (warning: it runs for a few minutes):
$ cd $GOPATH/src/github.com/PuerkitoBio/goquery
$ go test -bench=".*"
## Changelog
**Note that goquery's API is now stable, and will not break.**
* **2018-11-15 (v1.5.0)** : Go module support (thanks @Zaba505).
* **2018-06-07 (v1.4.1)** : Add `NewDocumentFromReader` examples.
* **2018-03-24 (v1.4.0)** : Deprecate `NewDocument(url)` and `NewDocumentFromResponse(response)`.
* **2018-01-28 (v1.3.0)** : Add `ToEnd` constant to `Slice` until the end of the selection (thanks to @davidjwilkins for raising the issue).
* **2018-01-11 (v1.2.0)** : Add `AddBack*` and deprecate `AndSelf` (thanks to @davidjwilkins).
* **2017-02-12 (v1.1.0)** : Add `SetHtml` and `SetText` (thanks to @glebtv).
* **2016-12-29 (v1.0.2)** : Optimize allocations for `Selection.Text` (thanks to @radovskyb).
* **2016-08-28 (v1.0.1)** : Optimize performance for large documents.
* **2016-07-27 (v1.0.0)** : Tag version 1.0.0.
* **2016-06-15** : Invalid selector strings internally compile to a `Matcher` implementation that never matches any node (instead of a panic). So for example, `doc.Find("~")` returns an empty `*Selection` object.
* **2016-02-02** : Add `NodeName` utility function similar to the DOM's `nodeName` property. It returns the tag name of the first element in a selection, and other relevant values of non-element nodes (see godoc for details). Add `OuterHtml` utility function similar to the DOM's `outerHTML` property (named `OuterHtml` in small caps for consistency with the existing `Html` method on the `Selection`).
* **2015-04-20** : Add `AttrOr` helper method to return the attribute's value or a default value if absent. Thanks to [piotrkowalczuk][piotr].
* **2015-02-04** : Add more manipulation functions - Prepend* - thanks again to [Andrew Stone][thatguystone].
* **2014-11-28** : Add more manipulation functions - ReplaceWith*, Wrap* and Unwrap - thanks again to [Andrew Stone][thatguystone].
* **2014-11-07** : Add manipulation functions (thanks to [Andrew Stone][thatguystone]) and `*Matcher` functions, that receive compiled cascadia selectors instead of selector strings, thus avoiding potential panics thrown by goquery via `cascadia.MustCompile` calls. This results in better performance (selectors can be compiled once and reused) and more idiomatic error handling (you can handle cascadia's compilation errors, instead of recovering from panics, which had been bugging me for a long time). Note that the actual type expected is a `Matcher` interface, that `cascadia.Selector` implements. Other matcher implementations could be used.
* **2014-11-06** : Change import paths of net/html to golang.org/x/net/html (see https://groups.google.com/forum/#!topic/golang-nuts/eD8dh3T9yyA). Make sure to update your code to use the new import path too when you call goquery with `html.Node`s.
* **v0.3.2** : Add `NewDocumentFromReader()` (thanks jweir) which allows creating a goquery document from an io.Reader.
* **v0.3.1** : Add `NewDocumentFromResponse()` (thanks assassingj) which allows creating a goquery document from an http response.
* **v0.3.0** : Add `EachWithBreak()` which allows to break out of an `Each()` loop by returning false. This function was added instead of changing the existing `Each()` to avoid breaking compatibility.
* **v0.2.1** : Make go-getable, now that [go.net/html is Go1.0-compatible][gonet] (thanks to @matrixik for pointing this out).
* **v0.2.0** : Add support for negative indices in Slice(). **BREAKING CHANGE** `Document.Root` is removed, `Document` is now a `Selection` itself (a selection of one, the root element, just like `Document.Root` was before). Add jQuery's Closest() method.
* **v0.1.1** : Add benchmarks to use as baseline for refactorings, refactor Next...() and Prev...() methods to use the new html package's linked list features (Next/PrevSibling, FirstChild). Good performance boost (40+% in some cases).
* **v0.1.0** : Initial release.
## API
goquery exposes two structs, `Document` and `Selection`, and the `Matcher` interface. Unlike jQuery, which is loaded as part of a DOM document, and thus acts on its containing document, goquery doesn't know which HTML document to act upon. So it needs to be told, and that's what the `Document` type is for. It holds the root document node as the initial Selection value to manipulate.
jQuery often has many variants for the same function (no argument, a selector string argument, a jQuery object argument, a DOM element argument, ...). Instead of exposing the same features in goquery as a single method with variadic empty interface arguments, statically-typed signatures are used following this naming convention:
* When the jQuery equivalent can be called with no argument, it has the same name as jQuery for the no argument signature (e.g.: `Prev()`), and the version with a selector string argument is called `XxxFiltered()` (e.g.: `PrevFiltered()`)
* When the jQuery equivalent **requires** one argument, the same name as jQuery is used for the selector string version (e.g.: `Is()`)
* The signatures accepting a jQuery object as argument are defined in goquery as `XxxSelection()` and take a `*Selection` object as argument (e.g.: `FilterSelection()`)
* The signatures accepting a DOM element as argument in jQuery are defined in goquery as `XxxNodes()` and take a variadic argument of type `*html.Node` (e.g.: `FilterNodes()`)
* The signatures accepting a function as argument in jQuery are defined in goquery as `XxxFunction()` and take a function as argument (e.g.: `FilterFunction()`)
* The goquery methods that can be called with a selector string have a corresponding version that take a `Matcher` interface and are defined as `XxxMatcher()` (e.g.: `IsMatcher()`)
Utility functions that are not in jQuery but are useful in Go are implemented as functions (that take a `*Selection` as parameter), to avoid a potential naming clash on the `*Selection`'s methods (reserved for jQuery-equivalent behaviour).
The complete [godoc reference documentation can be found here][doc].
Please note that Cascadia's selectors do not necessarily match all supported selectors of jQuery (Sizzle). See the [cascadia project][cascadia] for details. Invalid selector strings compile to a `Matcher` that fails to match any node. Behaviour of the various functions that take a selector string as argument follows from that fact, e.g. (where `~` is an invalid selector string):
* `Find("~")` returns an empty selection because the selector string doesn't match anything.
* `Add("~")` returns a new selection that holds the same nodes as the original selection, because it didn't add any node (selector string didn't match anything).
* `ParentsFiltered("~")` returns an empty selection because the selector string doesn't match anything.
* `ParentsUntil("~")` returns all parents of the selection because the selector string didn't match any element to stop before the top element.
## Examples
See some tips and tricks in the [wiki][].
Adapted from example_test.go:
```Go
package main
import (
"fmt"
"log"
"net/http"
"github.com/PuerkitoBio/goquery"
)
func ExampleScrape() {
// Request the HTML page.
res, err := http.Get("http://metalsucks.net")
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
log.Fatal(err)
}
// Find the review items
doc.Find(".sidebar-reviews article .content-block").Each(func(i int, s *goquery.Selection) {
// For each item found, get the band and title
band := s.Find("a").Text()
title := s.Find("i").Text()
fmt.Printf("Review %d: %s - %s\n", i, band, title)
})
}
func main() {
ExampleScrape()
}
```
## Related Projects
- [Goq][goq], an HTML deserialization and scraping library based on goquery and struct tags.
- [andybalholm/cascadia][cascadia], the CSS selector library used by goquery.
- [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
- [asciimoo/colly](https://github.com/asciimoo/colly), a lightning fast and elegant Scraping Framework
- [gnulnx/goperf](https://github.com/gnulnx/goperf), a website performance test tool that also fetches static assets.
- [MontFerret/ferret](https://github.com/MontFerret/ferret), declarative web scraping.
## Support
There are a number of ways you can support the project:
* Use it, star it, build something with it, spread the word!
- If you do build something open-source or otherwise publicly-visible, let me know so I can add it to the [Related Projects](#related-projects) section!
* Raise issues to improve the project (note: doc typos and clarifications are issues too!)
- Please search existing issues before opening a new one - it may have already been adressed.
* Pull requests: please discuss new code in an issue first, unless the fix is really trivial.
- Make sure new code is tested.
- Be mindful of existing code - PRs that break existing code have a high probability of being declined, unless it fixes a serious issue.
If you desperately want to send money my way, I have a BuyMeACoffee.com page:
<a href="https://www.buymeacoffee.com/mna" target="_blank"><img src="https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png" alt="Buy Me A Coffee" style="height: 41px !important;width: 174px !important;box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;-webkit-box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;" ></a>
## License
The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia's license is [here][caslic].
[jquery]: http://jquery.com/
[go]: http://golang.org/
[cascadia]: https://github.com/andybalholm/cascadia
[cascadiacli]: https://github.com/suntong/cascadia
[bsd]: http://opensource.org/licenses/BSD-3-Clause
[golic]: http://golang.org/LICENSE
[caslic]: https://github.com/andybalholm/cascadia/blob/master/LICENSE
[doc]: http://godoc.org/github.com/PuerkitoBio/goquery
[index]: http://api.jquery.com/index/
[gonet]: https://github.com/golang/net/
[html]: http://godoc.org/golang.org/x/net/html
[wiki]: https://github.com/PuerkitoBio/goquery/wiki/Tips-and-tricks
[thatguystone]: https://github.com/thatguystone
[piotr]: https://github.com/piotrkowalczuk
[goq]: https://github.com/andrewstuart/goq

124
vendor/github.com/PuerkitoBio/goquery/array.go

@ -0,0 +1,124 @@
package goquery
import (
"golang.org/x/net/html"
)
const (
maxUint = ^uint(0)
maxInt = int(maxUint >> 1)
// ToEnd is a special index value that can be used as end index in a call
// to Slice so that all elements are selected until the end of the Selection.
// It is equivalent to passing (*Selection).Length().
ToEnd = maxInt
)
// First reduces the set of matched elements to the first in the set.
// It returns a new Selection object, and an empty Selection object if the
// the selection is empty.
func (s *Selection) First() *Selection {
return s.Eq(0)
}
// Last reduces the set of matched elements to the last in the set.
// It returns a new Selection object, and an empty Selection object if
// the selection is empty.
func (s *Selection) Last() *Selection {
return s.Eq(-1)
}
// Eq reduces the set of matched elements to the one at the specified index.
// If a negative index is given, it counts backwards starting at the end of the
// set. It returns a new Selection object, and an empty Selection object if the
// index is invalid.
func (s *Selection) Eq(index int) *Selection {
if index < 0 {
index += len(s.Nodes)
}
if index >= len(s.Nodes) || index < 0 {
return newEmptySelection(s.document)
}
return s.Slice(index, index+1)
}
// Slice reduces the set of matched elements to a subset specified by a range
// of indices. The start index is 0-based and indicates the index of the first
// element to select. The end index is 0-based and indicates the index at which
// the elements stop being selected (the end index is not selected).
//
// The indices may be negative, in which case they represent an offset from the
// end of the selection.
//
// The special value ToEnd may be specified as end index, in which case all elements
// until the end are selected. This works both for a positive and negative start
// index.
func (s *Selection) Slice(start, end int) *Selection {
if start < 0 {
start += len(s.Nodes)
}
if end == ToEnd {
end = len(s.Nodes)
} else if end < 0 {
end += len(s.Nodes)
}
return pushStack(s, s.Nodes[start:end])
}
// Get retrieves the underlying node at the specified index.
// Get without parameter is not implemented, since the node array is available
// on the Selection object.
func (s *Selection) Get(index int) *html.Node {
if index < 0 {
index += len(s.Nodes) // Negative index gets from the end
}
return s.Nodes[index]
}
// Index returns the position of the first element within the Selection object
// relative to its sibling elements.
func (s *Selection) Index() int {
if len(s.Nodes) > 0 {
return newSingleSelection(s.Nodes[0], s.document).PrevAll().Length()
}
return -1
}
// IndexSelector returns the position of the first element within the
// Selection object relative to the elements matched by the selector, or -1 if
// not found.
func (s *Selection) IndexSelector(selector string) int {
if len(s.Nodes) > 0 {
sel := s.document.Find(selector)
return indexInSlice(sel.Nodes, s.Nodes[0])
}
return -1
}
// IndexMatcher returns the position of the first element within the
// Selection object relative to the elements matched by the matcher, or -1 if
// not found.
func (s *Selection) IndexMatcher(m Matcher) int {
if len(s.Nodes) > 0 {
sel := s.document.FindMatcher(m)
return indexInSlice(sel.Nodes, s.Nodes[0])
}
return -1
}
// IndexOfNode returns the position of the specified node within the Selection
// object, or -1 if not found.
func (s *Selection) IndexOfNode(node *html.Node) int {
return indexInSlice(s.Nodes, node)
}
// IndexOfSelection returns the position of the first node in the specified
// Selection object within this Selection object, or -1 if not found.
func (s *Selection) IndexOfSelection(sel *Selection) int {
if sel != nil && len(sel.Nodes) > 0 {
return indexInSlice(s.Nodes, sel.Nodes[0])
}
return -1
}

234
vendor/github.com/PuerkitoBio/goquery/array_test.go

@ -0,0 +1,234 @@
package goquery
import (
"testing"
)
func TestFirst(t *testing.T) {
sel := Doc().Find(".pvk-content").First()
assertLength(t, sel.Nodes, 1)
}
func TestFirstEmpty(t *testing.T) {
sel := Doc().Find(".pvk-zzcontentzz").First()
assertLength(t, sel.Nodes, 0)
}
func TestFirstInvalid(t *testing.T) {
sel := Doc().Find("").First()
assertLength(t, sel.Nodes, 0)
}
func TestFirstRollback(t *testing.T) {
sel := Doc().Find(".pvk-content")
sel2 := sel.First().End()
assertEqual(t, sel, sel2)
}
func TestLast(t *testing.T) {
sel := Doc().Find(".pvk-content").Last()
assertLength(t, sel.Nodes, 1)
// Should contain Footer
foot := Doc().Find(".footer")
if !sel.Contains(foot.Nodes[0]) {
t.Error("Last .pvk-content should contain .footer.")
}
}
func TestLastEmpty(t *testing.T) {
sel := Doc().Find(".pvk-zzcontentzz").Last()
assertLength(t, sel.Nodes, 0)
}
func TestLastInvalid(t *testing.T) {
sel := Doc().Find("").Last()
assertLength(t, sel.Nodes, 0)
}
func TestLastRollback(t *testing.T) {
sel := Doc().Find(".pvk-content")
sel2 := sel.Last().End()
assertEqual(t, sel, sel2)
}
func TestEq(t *testing.T) {
sel := Doc().Find(".pvk-content").Eq(1)
assertLength(t, sel.Nodes, 1)
}
func TestEqNegative(t *testing.T) {
sel := Doc().Find(".pvk-content").Eq(-1)
assertLength(t, sel.Nodes, 1)
// Should contain Footer
foot := Doc().Find(".footer")
if !sel.Contains(foot.Nodes[0]) {
t.Error("Index -1 of .pvk-content should contain .footer.")
}
}
func TestEqEmpty(t *testing.T) {
sel := Doc().Find("something_random_that_does_not_exists").Eq(0)
assertLength(t, sel.Nodes, 0)
}
func TestEqInvalid(t *testing.T) {
sel := Doc().Find("").Eq(0)
assertLength(t, sel.Nodes, 0)
}
func TestEqInvalidPositive(t *testing.T) {
sel := Doc().Find(".pvk-content").Eq(3)
assertLength(t, sel.Nodes, 0)
}
func TestEqInvalidNegative(t *testing.T) {
sel := Doc().Find(".pvk-content").Eq(-4)
assertLength(t, sel.Nodes, 0)
}
func TestEqRollback(t *testing.T) {
sel := Doc().Find(".pvk-content")
sel2 := sel.Eq(1).End()
assertEqual(t, sel, sel2)
}
func TestSlice(t *testing.T) {
sel := Doc().Find(".pvk-content").Slice(0, 2)
assertLength(t, sel.Nodes, 2)
assertSelectionIs(t, sel, "#pc1", "#pc2")
}
func TestSliceToEnd(t *testing.T) {
sel := Doc().Find(".pvk-content").Slice(1, ToEnd)
assertLength(t, sel.Nodes, 2)
assertSelectionIs(t, sel.Eq(0), "#pc2")
if _, ok := sel.Eq(1).Attr("id"); ok {
t.Error("Want no attribute ID, got one")
}
}
func TestSliceEmpty(t *testing.T) {
defer assertPanic(t)
Doc().Find("x").Slice(0, 2)
}
func TestSliceInvalid(t *testing.T) {
defer assertPanic(t)
Doc().Find("").Slice(0, 2)
}
func TestSliceInvalidToEnd(t *testing.T) {
defer assertPanic(t)
Doc().Find("").Slice(2, ToEnd)
}
func TestSliceOutOfBounds(t *testing.T) {
defer assertPanic(t)
Doc().Find(".pvk-content").Slice(2, 12)
}
func TestNegativeSliceStart(t *testing.T) {
sel := Doc().Find(".container-fluid").Slice(-2, 3)
assertLength(t, sel.Nodes, 1)
assertSelectionIs(t, sel.Eq(0), "#cf3")
}
func TestNegativeSliceEnd(t *testing.T) {
sel := Doc().Find(".container-fluid").Slice(1, -1)
assertLength(t, sel.Nodes, 2)
assertSelectionIs(t, sel.Eq(0), "#cf2")
assertSelectionIs(t, sel.Eq(1), "#cf3")
}
func TestNegativeSliceBoth(t *testing.T) {
sel := Doc().Find(".container-fluid").Slice(-3, -1)
assertLength(t, sel.Nodes, 2)
assertSelectionIs(t, sel.Eq(0), "#cf2")
assertSelectionIs(t, sel.Eq(1), "#cf3")
}
func TestNegativeSliceToEnd(t *testing.T) {
sel := Doc().Find(".container-fluid").Slice(-3, ToEnd)
assertLength(t, sel.Nodes, 3)
assertSelectionIs(t, sel, "#cf2", "#cf3", "#cf4")
}
func TestNegativeSliceOutOfBounds(t *testing.T) {
defer assertPanic(t)
Doc().Find(".container-fluid").Slice(-12, -7)
}
func TestSliceRollback(t *testing.T) {
sel := Doc().Find(".pvk-content")
sel2 := sel.Slice(0, 2).End()
assertEqual(t, sel, sel2)
}
func TestGet(t *testing.T) {
sel := Doc().Find(".pvk-content")
node := sel.Get(1)
if sel.Nodes[1] != node {
t.Errorf("Expected node %v to be %v.", node, sel.Nodes[1])
}
}
func TestGetNegative(t *testing.T) {
sel := Doc().Find(".pvk-content")
node := sel.Get(-3)
if sel.Nodes[0] != node {
t.Errorf("Expected node %v to be %v.", node, sel.Nodes[0])
}
}
func TestGetInvalid(t *testing.T) {
defer assertPanic(t)
sel := Doc().Find(".pvk-content")
sel.Get(129)
}
func TestIndex(t *testing.T) {
sel := Doc().Find(".pvk-content")
if i := sel.Index(); i != 1 {
t.Errorf("Expected index of 1, got %v.", i)
}
}
func TestIndexSelector(t *testing.T) {
sel := Doc().Find(".hero-unit")
if i := sel.IndexSelector("div"); i != 4 {
t.Errorf("Expected index of 4, got %v.", i)
}
}
func TestIndexSelectorInvalid(t *testing.T) {
sel := Doc().Find(".hero-unit")
if i := sel.IndexSelector(""); i != -1 {
t.Errorf("Expected index of -1, got %v.", i)
}
}
func TestIndexOfNode(t *testing.T) {
sel := Doc().Find("div.pvk-gutter")
if i := sel.IndexOfNode(sel.Nodes[1]); i != 1 {
t.Errorf("Expected index of 1, got %v.", i)
}
}
func TestIndexOfNilNode(t *testing.T) {
sel := Doc().Find("div.pvk-gutter")
if i := sel.IndexOfNode(nil); i != -1 {
t.Errorf("Expected index of -1, got %v.", i)
}
}
func TestIndexOfSelection(t *testing.T) {
sel := Doc().Find("div")
sel2 := Doc().Find(".hero-unit")
if i := sel.IndexOfSelection(sel2); i != 4 {
t.Errorf("Expected index of 4, got %v.", i)
}
}

120
vendor/github.com/PuerkitoBio/goquery/bench_array_test.go

@ -0,0 +1,120 @@
package goquery
import (
"testing"
)
func BenchmarkFirst(b *testing.B) {
b.StopTimer()
sel := DocB().Find("dd")
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.First()
}
}
func BenchmarkLast(b *testing.B) {
b.StopTimer()
sel := DocB().Find("dd")
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Last()
}
}
func BenchmarkEq(b *testing.B) {
b.StopTimer()
sel := DocB().Find("dd")
j := 0
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Eq(j)
if j++; j >= sel.Length() {
j = 0
}
}
}
func BenchmarkSlice(b *testing.B) {
b.StopTimer()
sel := DocB().Find("dd")
j := 0
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Slice(j, j+4)
if j++; j >= (sel.Length() - 4) {
j = 0
}
}
}
func BenchmarkGet(b *testing.B) {
b.StopTimer()
sel := DocB().Find("dd")
j := 0
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Get(j)
if j++; j >= sel.Length() {
j = 0
}
}
}
func BenchmarkIndex(b *testing.B) {
var j int
b.StopTimer()
sel := DocB().Find("#Main")
b.StartTimer()
for i := 0; i < b.N; i++ {
j = sel.Index()
}
if j != 3 {
b.Fatalf("want 3, got %d", j)
}
}
func BenchmarkIndexSelector(b *testing.B) {
var j int
b.StopTimer()
sel := DocB().Find("#manual-nav dl dd:nth-child(1)")
b.StartTimer()
for i := 0; i < b.N; i++ {
j = sel.IndexSelector("dd")
}
if j != 4 {
b.Fatalf("want 4, got %d", j)
}
}
func BenchmarkIndexOfNode(b *testing.B) {
var j int
b.StopTimer()
sel := DocB().Find("span a")
sel2 := DocB().Find("span a:nth-child(3)")
n := sel2.Get(0)
b.StartTimer()
for i := 0; i < b.N; i++ {
j = sel.IndexOfNode(n)
}
if j != 2 {
b.Fatalf("want 2, got %d", j)
}
}
func BenchmarkIndexOfSelection(b *testing.B) {
var j int
b.StopTimer()
sel := DocB().Find("span a")
sel2 := DocB().Find("span a:nth-child(3)")
b.StartTimer()
for i := 0; i < b.N; i++ {
j = sel.IndexOfSelection(sel2)
}
if j != 2 {
b.Fatalf("want 2, got %d", j)
}
}

40
vendor/github.com/PuerkitoBio/goquery/bench_example_test.go

@ -0,0 +1,40 @@
package goquery
import (
"bytes"
"fmt"
"strconv"
"testing"
)
func BenchmarkMetalReviewExample(b *testing.B) {
var n int
var buf bytes.Buffer
b.StopTimer()
doc := loadDoc("metalreview.html")
b.StartTimer()
for i := 0; i < b.N; i++ {
doc.Find(".slider-row:nth-child(1) .slider-item").Each(func(i int, s *Selection) {
var band, title string
var score float64
var e error
n++
// For each item found, get the band, title and score, and print it
band = s.Find("strong").Text()
title = s.Find("em").Text()
if score, e = strconv.ParseFloat(s.Find(".score").Text(), 64); e != nil {
// Not a valid float, ignore score
if n <= 4 {
buf.WriteString(fmt.Sprintf("Review %d: %s - %s.\n", i, band, title))
}
} else {
// Print all, including score
if n <= 4 {
buf.WriteString(fmt.Sprintf("Review %d: %s - %s (%2.1f).\n", i, band, title, score))
}
}
})
}
}

104
vendor/github.com/PuerkitoBio/goquery/bench_expand_test.go

@ -0,0 +1,104 @@
package goquery
import (
"testing"
)
func BenchmarkAdd(b *testing.B) {
var n int
b.StopTimer()
sel := DocB().Find("dd")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.Add("h2[title]").Length()
} else {
sel.Add("h2[title]")
}
}
if n != 43 {
b.Fatalf("want 43, got %d", n)
}
}
func BenchmarkAddSelection(b *testing.B) {
var n int
b.StopTimer()
sel := DocB().Find("dd")
sel2 := DocB().Find("h2[title]")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.AddSelection(sel2).Length()
} else {
sel.AddSelection(sel2)
}
}
if n != 43 {
b.Fatalf("want 43, got %d", n)
}
}
func BenchmarkAddNodes(b *testing.B) {
var n int
b.StopTimer()
sel := DocB().Find("dd")
sel2 := DocB().Find("h2[title]")
nodes := sel2.Nodes
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.AddNodes(nodes...).Length()
} else {
sel.AddNodes(nodes...)
}
}
if n != 43 {
b.Fatalf("want 43, got %d", n)
}
}
func BenchmarkAddNodesBig(b *testing.B) {
var n int
doc := DocW()
sel := doc.Find("li")
// make nodes > 1000
nodes := sel.Nodes
nodes = append(nodes, nodes...)
nodes = append(nodes, nodes...)
sel = doc.Find("xyz")
b.ResetTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.AddNodes(nodes...).Length()
} else {
sel.AddNodes(nodes...)
}
}
if n != 373 {
b.Fatalf("want 373, got %d", n)
}
}
func BenchmarkAndSelf(b *testing.B) {
var n int
b.StopTimer()
sel := DocB().Find("dd").Parent()
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.AndSelf().Length()
} else {
sel.AndSelf()
}
}
if n != 44 {
b.Fatalf("want 44, got %d", n)
}
}

236
vendor/github.com/PuerkitoBio/goquery/bench_filter_test.go

@ -0,0 +1,236 @@
package goquery
import (
"testing"
)
func BenchmarkFilter(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.Filter(".toclevel-1").Length()
} else {
sel.Filter(".toclevel-1")
}
}
if n != 13 {
b.Fatalf("want 13, got %d", n)
}
}
func BenchmarkNot(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.Not(".toclevel-2").Length()
} else {
sel.Filter(".toclevel-2")
}
}
if n != 371 {
b.Fatalf("want 371, got %d", n)
}
}
func BenchmarkFilterFunction(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
f := func(i int, s *Selection) bool {
return len(s.Get(0).Attr) > 0
}
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.FilterFunction(f).Length()
} else {
sel.FilterFunction(f)
}
}
if n != 112 {
b.Fatalf("want 112, got %d", n)
}
}
func BenchmarkNotFunction(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
f := func(i int, s *Selection) bool {
return len(s.Get(0).Attr) > 0
}
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.NotFunction(f).Length()
} else {
sel.NotFunction(f)
}
}
if n != 261 {
b.Fatalf("want 261, got %d", n)
}
}
func BenchmarkFilterNodes(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".toclevel-2")
nodes := sel2.Nodes
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.FilterNodes(nodes...).Length()
} else {
sel.FilterNodes(nodes...)
}
}
if n != 2 {
b.Fatalf("want 2, got %d", n)
}
}
func BenchmarkNotNodes(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".toclevel-1")
nodes := sel2.Nodes
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.NotNodes(nodes...).Length()
} else {
sel.NotNodes(nodes...)
}
}
if n != 360 {
b.Fatalf("want 360, got %d", n)
}
}
func BenchmarkFilterSelection(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".toclevel-2")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.FilterSelection(sel2).Length()
} else {
sel.FilterSelection(sel2)
}
}
if n != 2 {
b.Fatalf("want 2, got %d", n)
}
}
func BenchmarkNotSelection(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".toclevel-1")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.NotSelection(sel2).Length()
} else {
sel.NotSelection(sel2)
}
}
if n != 360 {
b.Fatalf("want 360, got %d", n)
}
}
func BenchmarkHas(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("h2")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.Has(".editsection").Length()
} else {
sel.Has(".editsection")
}
}
if n != 13 {
b.Fatalf("want 13, got %d", n)
}
}
func BenchmarkHasNodes(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".tocnumber")
nodes := sel2.Nodes
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.HasNodes(nodes...).Length()
} else {
sel.HasNodes(nodes...)
}
}
if n != 15 {
b.Fatalf("want 15, got %d", n)
}
}
func BenchmarkHasSelection(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".tocnumber")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.HasSelection(sel2).Length()
} else {
sel.HasSelection(sel2)
}
}
if n != 15 {
b.Fatalf("want 15, got %d", n)
}
}
func BenchmarkEnd(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("li").Has(".tocnumber")
b.StartTimer()
for i := 0; i < b.N; i++ {
if n == 0 {
n = sel.End().Length()
} else {
sel.End()
}
}
if n != 373 {
b.Fatalf("want 373, got %d", n)
}
}

68
vendor/github.com/PuerkitoBio/goquery/bench_iteration_test.go

@ -0,0 +1,68 @@
package goquery
import (
"testing"
)
func BenchmarkEach(b *testing.B) {
var tmp, n int
b.StopTimer()
sel := DocW().Find("td")
f := func(i int, s *Selection) {
tmp++
}
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Each(f)
if n == 0 {
n = tmp
}
}
if n != 59 {
b.Fatalf("want 59, got %d", n)
}
}
func BenchmarkMap(b *testing.B) {
var tmp, n int
b.StopTimer()
sel := DocW().Find("td")
f := func(i int, s *Selection) string {
tmp++
return string(tmp)
}
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Map(f)
if n == 0 {
n = tmp
}
}
if n != 59 {
b.Fatalf("want 59, got %d", n)
}
}
func BenchmarkEachWithBreak(b *testing.B) {
var tmp, n int
b.StopTimer()
sel := DocW().Find("td")
f := func(i int, s *Selection) bool {
tmp++
return tmp < 10
}
b.StartTimer()
for i := 0; i < b.N; i++ {
tmp = 0
sel.EachWithBreak(f)
if n == 0 {
n = tmp
}
}
if n != 10 {
b.Fatalf("want 10, got %d", n)
}
}

51
vendor/github.com/PuerkitoBio/goquery/bench_property_test.go

@ -0,0 +1,51 @@
package goquery
import (
"testing"
)
func BenchmarkAttr(b *testing.B) {
var s string
b.StopTimer()
sel := DocW().Find("h1")
b.StartTimer()
for i := 0; i < b.N; i++ {
s, _ = sel.Attr("id")
}
if s != "firstHeading" {
b.Fatalf("want firstHeading, got %q", s)
}
}
func BenchmarkText(b *testing.B) {
b.StopTimer()
sel := DocW().Find("h2")
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Text()
}
}
func BenchmarkLength(b *testing.B) {
var n int
b.StopTimer()
sel := DocW().Find("h2")
b.StartTimer()
for i := 0; i < b.N; i++ {
n = sel.Length()
}
if n != 14 {
b.Fatalf("want 14, got %d", n)
}
}
func BenchmarkHtml(b *testing.B) {
b.StopTimer()
sel := DocW().Find("h2")
b.StartTimer()
for i := 0; i < b.N; i++ {
sel.Html()
}
}

111
vendor/github.com/PuerkitoBio/goquery/bench_query_test.go

@ -0,0 +1,111 @@
package goquery
import (
"testing"
)
func BenchmarkIs(b *testing.B) {
var y bool
b.StopTimer()
sel := DocW().Find("li")
b.StartTimer()
for i := 0; i < b.N; i++ {
y = sel.Is(".toclevel-2")
}
if !y {
b.Fatal("want true")
}
}
func BenchmarkIsPositional(b *testing.B) {
var y bool
b.StopTimer()
sel := DocW().Find("li")
b.StartTimer()
for i := 0; i < b.N; i++ {
y = sel.Is("li:nth-child(2)")
}
if !y {
b.Fatal("want true")
}
}
func BenchmarkIsFunction(b *testing.B) {
var y bool
b.StopTimer()
sel := DocW().Find(".toclevel-1")
f := func(i int, s *Selection) bool {
return i == 8
}
b.StartTimer()
for i := 0; i < b.N; i++ {
y = sel.IsFunction(f)
}
if !y {
b.Fatal("want true")
}
}
func BenchmarkIsSelection(b *testing.B) {
var y bool
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".toclevel-2")
b.StartTimer()
for i := 0; i < b.N; i++ {
y = sel.IsSelection(sel2)
}
if !y {
b.Fatal("want true")
}
}
func BenchmarkIsNodes(b *testing.B) {
var y bool
b.StopTimer()
sel := DocW().Find("li")
sel2 := DocW().Find(".toclevel-2")
nodes := sel2.Nodes
b.StartTimer()
for i := 0; i < b.N; i++ {
y = sel.IsNodes(nodes...)
}
if !y {
b.Fatal("want true")
}
}
func BenchmarkHasClass(b *testing.B) {
var y bool
b.StopTimer()
sel := DocW().Find("span")
b.StartTimer()
for i := 0; i < b.N; i++ {
y = sel.HasClass("official")